diff --git a/.gitignore b/.gitignore index c7aa228d3..aba662bbf 100644 --- a/.gitignore +++ b/.gitignore @@ -105,4 +105,4 @@ install_manifest_platform.txt install_manifest_storage-engine.txt _CPack_Packages columnstoreversion.h - +.idea/ diff --git a/README b/README index 0fc563153..8da8e257d 100644 --- a/README +++ b/README @@ -9,3 +9,4 @@ Additional features will be pushed in future releases. A few things to notice: - Do not use pre-releases on production systems. - The building of the ColumnStore engine needs a special build environment. We're working on making it available for everyone to build. + diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 71d5e1f26..1a3527086 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -180,7 +180,6 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/bin/resourceReport.sh" "/usr/local/mariadb/columnstore/bin/hadoopReport.sh" "/usr/local/mariadb/columnstore/bin/alarmReport.sh" -"/usr/local/mariadb/columnstore/bin/amazonInstaller" "/usr/local/mariadb/columnstore/bin/remote_command_verify.sh" "/usr/local/mariadb/columnstore/bin/disable-rep-columnstore.sh" "/usr/local/mariadb/columnstore/bin/columnstore.service" @@ -216,6 +215,9 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/bin/os_detect.sh" "/usr/local/mariadb/columnstore/bin/columnstoreClusterTester.sh" "/usr/local/mariadb/columnstore/bin/mariadb-command-line.sh" +"/usr/local/mariadb/columnstore/bin/quick_installer_single_server.sh" +"/usr/local/mariadb/columnstore/bin/quick_installer_multi_server.sh" +"/usr/local/mariadb/columnstore/bin/quick_installer_amazon.sh" ${ignored}) SET(CPACK_RPM_libs_USER_FILELIST diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index f4ccd5594..30bc97124 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -1,4 +1,3 @@ - INCLUDE_DIRECTORIES( ${ENGINE_COMMON_INCLUDES} ) ADD_CUSTOM_COMMAND( @@ -13,7 +12,7 @@ ADD_CUSTOM_TARGET(ddl-lexer DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ddl-scan.cpp) ADD_CUSTOM_TARGET(ddl-parser DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ddl-gram.cpp) # Parser puts extra info to stderr. INCLUDE(../../check_compiler_flag.cmake) -MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG" DEBUG) +MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG=1" DEBUG) ########### next target ############### diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 9e3b70672..179e5c14b 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -33,6 +33,9 @@ using namespace ddlpackage; typedef enum { NOOP, STRIP_QUOTES } copy_action_t; +#if YYDEBUG == 0 +int ddldebug = 0; +#endif int lineno = 1; void ddlerror(struct pass_to_bison* x, char const *s); diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 7c2c58efe..37ab49425 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -63,7 +63,6 @@ char* copy_string(const char *str); %pure-parser %lex-param {void * scanner} %parse-param {struct ddlpackage::pass_to_bison * x} -%debug /* Bison uses this to generate a C union definition. This is used to store the application created values associated with syntactic diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index c2f44f2b3..aab6c9265 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -368,11 +368,7 @@ bool ArithmeticColumn::operator==(const ArithmeticColumn& t) const else if (fExpression != NULL || t.fExpression != NULL) return false; - if (fAlias != t.fAlias) - return false; - if (fTableAlias != t.fTableAlias) - return false; if (fData != t.fData) return false; diff --git a/dbcon/execplan/clientrotator.cpp b/dbcon/execplan/clientrotator.cpp index 55634af7c..0a289f5f6 100644 --- a/dbcon/execplan/clientrotator.cpp +++ b/dbcon/execplan/clientrotator.cpp @@ -49,6 +49,15 @@ using namespace logging; #include "clientrotator.h" +//#include "idb_mysql.h" + +/** Debug macro */ +#ifdef INFINIDB_DEBUG +#define IDEBUG(x) {x;} +#else +#define IDEBUG(x) {} +#endif + #define LOG_TO_CERR namespace execplan @@ -60,14 +69,36 @@ const uint64_t LOCAL_EXEMGR_PORT = 8601; string ClientRotator::getModule() { string installDir = startup::StartUp::installDir(); + + //Log to debug.log + LoggingID logid( 24, 0, 0); + string fileName = installDir + "/local/module"; + string module; ifstream moduleFile (fileName.c_str()); if (moduleFile.is_open()) + { getline (moduleFile, module); + } + else + { + { + logging::Message::Args args1; + logging::Message msg(1); + std::ostringstream oss; + oss << "ClientRotator::getModule open status2 =" << strerror(errno); + args1.add(oss.str()); + args1.add(fileName); + msg.format( args1 ); + Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + } + } moduleFile.close(); + return module; } diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index d292ee984..dd112600e 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -328,8 +328,6 @@ bool ConstantColumn::operator==(const ConstantColumn& t) const if (fType != t.fType) return false; - if (fAlias != t.fAlias) - return false; if (fData != t.fData) return false; diff --git a/dbcon/execplan/functioncolumn.cpp b/dbcon/execplan/functioncolumn.cpp index 108026484..487d42c0b 100644 --- a/dbcon/execplan/functioncolumn.cpp +++ b/dbcon/execplan/functioncolumn.cpp @@ -332,8 +332,6 @@ bool FunctionColumn::operator==(const FunctionColumn& t) const if (**it != **it2) return false; -// if (fAlias != t.fAlias) -// return false; if (fTableAlias != t.fTableAlias) return false; diff --git a/dbcon/execplan/returnedcolumn.cpp b/dbcon/execplan/returnedcolumn.cpp index 8feb7025b..44278424f 100644 --- a/dbcon/execplan/returnedcolumn.cpp +++ b/dbcon/execplan/returnedcolumn.cpp @@ -173,14 +173,13 @@ void ReturnedColumn::unserialize(messageqcpp::ByteStream& b) bool ReturnedColumn::operator==(const ReturnedColumn& t) const { + // Not all fields are considered for a positive equality. if (fData != t.fData) return false; if (fCardinality != t.fCardinality) return false; - //if (fAlias != t.fAlias) - // return false; if (fDistinct != t.fDistinct) return false; @@ -193,24 +192,18 @@ bool ReturnedColumn::operator==(const ReturnedColumn& t) const if (fNullsFirst != t.fNullsFirst) return false; - //if (fOrderPos != t.fOrderPos) - // return false; if (fInputIndex != t.fInputIndex) return false; if (fOutputIndex != t.fOutputIndex) return false; - //if (fSequence != t.fSequence) - // return false; if (fResultType != t.fResultType) return false; if (fOperationType != t.fOperationType) return false; - //if (fExpressionId != t.fExpressionId) - // return false; return true; } diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 64955401e..1d7780e33 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -346,7 +346,6 @@ void SimpleColumn::serialize(messageqcpp::ByteStream& b) const b << fViewName; b << (uint32_t) fOid; b << fData; - //b << fAlias; b << fTableAlias; b << (uint32_t) fSequence; b << static_cast(fIsInfiniDB); @@ -363,7 +362,6 @@ void SimpleColumn::unserialize(messageqcpp::ByteStream& b) b >> fViewName; b >> (uint32_t&) fOid; b >> fData; - //b >> fAlias; b >> fTableAlias; b >> (uint32_t&) fSequence; b >> reinterpret_cast< ByteStream::doublebyte&>(fIsInfiniDB); @@ -388,8 +386,6 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const if (fColumnName != t.fColumnName) return false; -// if (fIndexName != t.fIndexName) -// return false; if (fViewName != t.fViewName) return false; @@ -399,8 +395,6 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const if (data() != t.data()) return false; -// if (fAlias != t.fAlias) -// return false; if (fTableAlias != t.fTableAlias) return false; diff --git a/dbcon/joblist/fifo.h b/dbcon/joblist/fifo.h index dbdf1eca8..429572ed5 100644 --- a/dbcon/joblist/fifo.h +++ b/dbcon/joblist/fifo.h @@ -398,20 +398,29 @@ void FIFO::signalPs() template inline bool FIFO::next(uint64_t id, element_t* out) { + base::mutex.lock(); fConsumptionStarted = true; if (cpos[id] >= fMaxElements) + { + base::mutex.unlock(); if (!waitForSwap(id)) return false; + base::mutex.lock(); + } *out = cBuffer[cpos[id]++]; #ifndef ONE_CS if (cpos[id] == fMaxElements) + { + base::mutex.unlock(); signalPs(); - + return true; + } #endif + base::mutex.unlock(); return true; } diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index f2b52a9d9..43bdc8cd6 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -1820,8 +1820,11 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ } else if (ci.columnTypes[colpos].colWidth < 16777216) { - dataLength = *(uint32_t*) buf; - buf = buf + 3 ; + dataLength = *(uint16_t*) buf; + buf = buf + 2 ; + if (*(uint8_t*)buf) + dataLength += 256*256*(*(uint8_t*)buf) ; + buf++; } else { diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 596fdb806..91c7f79c5 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -1548,8 +1548,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) ifp->functype() == Item_func::ISNOTNULL_FUNC) { ReturnedColumn* rhs = NULL; - - if (!gwip->rcWorkStack.empty()) + if (!gwip->rcWorkStack.empty() && !gwip->inCaseStmt) { rhs = gwip->rcWorkStack.top(); gwip->rcWorkStack.pop(); @@ -1650,8 +1649,49 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) idbassert(ifp->argument_count() == 1); ParseTree* ptp = 0; + if (((Item_func*)(ifp->arguments()[0]))->functype() == Item_func::EQUAL_FUNC) + { + // negate it in place + // Note that an EQUAL_FUNC ( a <=> b) was converted to + // ( a = b OR ( a is null AND b is null) ) + // NOT of the above expression is: ( a != b AND (a is not null OR b is not null ) - if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM) + if (!gwip->ptWorkStack.empty()) + ptp = gwip->ptWorkStack.top(); + + if (ptp) + { + ParseTree* or_ptp = ptp; + ParseTree* and_ptp = or_ptp->right(); + ParseTree* equal_ptp = or_ptp->left(); + ParseTree* nullck_left_ptp = and_ptp->left(); + ParseTree* nullck_right_ptp = and_ptp->right(); + SimpleFilter *sf_left_nullck = dynamic_cast(nullck_left_ptp->data()); + SimpleFilter *sf_right_nullck = dynamic_cast(nullck_right_ptp->data()); + SimpleFilter *sf_equal = dynamic_cast(equal_ptp->data()); + + if (sf_left_nullck && sf_right_nullck && sf_equal) { + // Negate the null checks + sf_left_nullck->op()->reverseOp(); + sf_right_nullck->op()->reverseOp(); + sf_equal->op()->reverseOp(); + // Rehook the nodes + ptp = and_ptp; + ptp->left(equal_ptp); + ptp->right(or_ptp); + or_ptp->left(nullck_left_ptp); + or_ptp->right(nullck_right_ptp); + gwip->ptWorkStack.pop(); + gwip->ptWorkStack.push(ptp); + } + else { + gwip->fatalParseError = true; + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ASSERTION_FAILURE); + return false; + } + } + } + else if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM) { // negate it in place if (!gwip->ptWorkStack.empty()) @@ -1725,7 +1765,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) } else if (ifp->functype() == Item_func::EQUAL_FUNC) { - // a = b OR (a IS NULL AND b IS NULL) + // Convert "a <=> b" to (a = b OR (a IS NULL AND b IS NULL))" idbassert (gwip->rcWorkStack.size() >= 2); ReturnedColumn* rhs = gwip->rcWorkStack.top(); gwip->rcWorkStack.pop(); @@ -1737,7 +1777,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) // b IS NULL ConstantColumn* nlhs1 = new ConstantColumn("", ConstantColumn::NULLDATA); sop.reset(new PredicateOperator("isnull")); - sop->setOpType(lhs->resultType(), rhs->resultType()); + sop->setOpType(lhs->resultType(), rhs->resultType()); sfn1 = new SimpleFilter(sop, rhs, nlhs1); ParseTree* ptpl = new ParseTree(sfn1); // a IS NULL @@ -1752,7 +1792,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) ptpn->right(ptpr); // a = b sop.reset(new PredicateOperator("=")); - sop->setOpType(lhs->resultType(), lhs->resultType()); + sop->setOpType(lhs->resultType(), rhs->resultType()); sfo = new SimpleFilter(sop, lhs->clone(), rhs->clone()); // OR with the NULL comparison tree ParseTree* ptp = new ParseTree(new LogicOperator("or")); @@ -3772,8 +3812,12 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS if (funcName == "case_searched" && (i < arg_offset)) { + // MCOL-1472 Nested CASE with an ISNULL predicate. We don't want the predicate + // to pull off of rcWorkStack, so we set this inCaseStmt flag to tell it + // not to. + gwi.inCaseStmt = true; sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - + gwi.inCaseStmt = false; if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data()) { gwi.ptWorkStack.pop(); @@ -10228,3 +10272,4 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro } +// vim:ts=4 sw=4: diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 8c790e8d4..c2ff80950 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1964,7 +1964,7 @@ uint32_t doUpdateDelete(THD* thd) } else { - thd->set_row_count_func(dmlRowCount); + thd->set_row_count_func(dmlRowCount+thd->get_row_count_func()); } push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, errorMsg.c_str()); @@ -1972,7 +1972,7 @@ uint32_t doUpdateDelete(THD* thd) else { // if (dmlRowCount != 0) //Bug 5117. Handling self join. - thd->set_row_count_func(dmlRowCount); + thd->set_row_count_func(dmlRowCount+thd->get_row_count_func()); //cout << " error status " << ci->rc << " and rowcount = " << dmlRowCount << endl; diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index b7e668b2d..77b30d988 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -149,6 +149,9 @@ struct gp_walk_info int32_t recursionHWM; std::stack rcBookMarkStack; + // Kludge for MCOL-1472 + bool inCaseStmt; + gp_walk_info() : sessionid(0), fatalParseError(false), condPush(false), @@ -164,7 +167,8 @@ struct gp_walk_info lastSub(0), derivedTbCnt(0), recursionLevel(-1), - recursionHWM(0) + recursionHWM(0), + inCaseStmt(false) {} ~gp_walk_info() {} diff --git a/exemgr/main.cpp b/exemgr/main.cpp index ca0573788..8742188a0 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1443,7 +1443,9 @@ int main(int argc, char* argv[]) #endif setupSignalHandlers(); - int err = setupResources(); + int err = 0; + if (!gDebug) + err = setupResources(); string errMsg; switch (err) diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 70a0add6d..36618e763 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -438,7 +438,7 @@ n n n - 2 + 1 n n internal diff --git a/oam/etc/ProcessConfig.xml b/oam/etc/ProcessConfig.xml index 8088135bb..0bedf10de 100644 --- a/oam/etc/ProcessConfig.xml +++ b/oam/etc/ProcessConfig.xml @@ -97,7 +97,7 @@ WriteEngineServer pm* DBRMWorkerNode - * + @ ExeMgr * SIMPLEX @@ -112,7 +112,7 @@ WriteEngineServer pm* DBRMWorkerNode - * + @ DDLProc @ SIMPLEX diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index 038ad0c43..15c00cea5 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -94,13 +94,9 @@ start() { CoreFileFlag=`$InstallDir/bin/getConfig -c $InstallDir/etc/Columnstore.xml Installation CoreFileFlag` if [ $CoreFileFlag = "y" ]; then - SUDO= - if [ "$user" != "root" ]; then - SUDO="$SUDO" - fi #columnstore core files - $SUDO ulimit -c unlimited > /dev/null 2>&1 + ulimit -c unlimited > /dev/null 2>&1 $SUDO sysctl -q -w kernel.core_uses_pid=1 > /dev/null 2>&1 $SUDO sysctl -q -w kernel.core_pattern=/var/log/mariadb/columnstore/corefiles/core.%e.%p > /dev/null 2>&1 fi diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 6ca39b70f..8910cca7f 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -186,7 +186,6 @@ if [ $module = "um" ]; then echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE echo "done" >> $RCFILE else - sudo chmod 666 $RCFILE sudo echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE sudo echo "done" >> $RCFILE @@ -201,7 +200,6 @@ else echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE echo "done" >> $RCFILE else - sudo chmod 666 $RCFILE sudo echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE sudo echo "done" >> $RCFILE @@ -214,7 +212,7 @@ fi if [ $user != "root" ]; then echo "uncomment runuser in rc.local" - sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 + sudo sed -i -e 's/#sudo runuser/sudo runuser/g' $RCFILE >/dev/null 2>&1 fi echo "!!!Module Installation Successfully Completed!!!" diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 1766a37c2..5a61a9074 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -164,7 +164,7 @@ mkdir -p /tmp/columnstore_tmp_files >/dev/null 2>&1 #setup core file directory and link mkdir /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1 -chmod 755 /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1 +chmod 777 /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1 #create mount directories mkdir /mnt/tmp > /dev/null 2>&1 diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 5c63ffd20..d53dd66de 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -5615,6 +5615,7 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s dbrootList dbroot1; dbroot1.push_back(*pt1); + bool returnDbRoot = false; //send msg to unmount dbroot if module is not offline int opState; @@ -5629,7 +5630,6 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s if (opState != oam::AUTO_OFFLINE || opState != oam::AUTO_DISABLED) { -// bool unmountPass = true; try { mountDBRoot(dbroot1, false); @@ -5639,13 +5639,8 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s writeLog("ERROR: dbroot failed to unmount", LOG_TYPE_ERROR ); cout << endl << "ERROR: umountDBRoot api failure" << endl; exceptionControl("manualMovePmDbroot", API_FAILURE); -// unmountPass = false; } -// if ( !unmountPass) { -// dbrootlist.erase(pt1); -// break; -// } } //check for amazon moving required @@ -5663,40 +5658,79 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s //if Gluster, do the assign command if ( DataRedundancyConfig == "y") { - try - { + try + { string errmsg; int ret = glusterctl(oam::GLUSTER_ASSIGN, *pt1, toPM, errmsg); - - if ( ret != 0 ) + if ( ret == 0 ) + { + todbrootConfigList.push_back(*pt2); + residedbrootConfigList.erase(pt2); + } + else { cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID + ", error: " + errmsg << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID + ", error: " + errmsg, LOG_TYPE_ERROR ); + returnDbRoot = true; } } catch (exception& e) { cout << endl << "**** glusterctl API exception: " << e.what() << endl; cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID, LOG_TYPE_ERROR ); + returnDbRoot = true; } catch (...) { cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID, LOG_TYPE_ERROR ); + returnDbRoot = true; } } - todbrootConfigList.push_back(*pt2); - - residedbrootConfigList.erase(pt2); - + if (returnDbRoot) + { + // something went wrong return it back to original owner + try + { + string errmsg; + writeLog("reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR ); + int ret = glusterctl(oam::GLUSTER_ASSIGN, *pt1, residePM, errmsg); + if ( ret != 0 ) + { + cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID + ", error: " + errmsg << endl; + writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID + ", error: " + errmsg, LOG_TYPE_ERROR ); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); + } + mountDBRoot(dbroot1); + //get updated Columnstore.xml distributed + distributeConfigFile("system"); + return; + } + catch (exception& e) + { + cout << endl << "**** glusterctl API exception: " << e.what() << endl; + cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID << endl; + writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR ); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); + } + catch (...) + { + cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; + cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID << endl; + writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR ); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); + } + } break; } } } + + //set the 2 pms dbroot config try { @@ -5706,7 +5740,7 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s { writeLog("ERROR: setPmDbrootConfig api failure for pm" + residePMID, LOG_TYPE_ERROR ); cout << endl << "ERROR: setPmDbrootConfig api failure for pm" + residePMID << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); } try @@ -5717,7 +5751,7 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s { writeLog("ERROR: setPmDbrootConfig api failure for pm" + toPMID, LOG_TYPE_ERROR ); cout << endl << "ERROR: setPmDbrootConfig api failure for pm" + toPMID << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); } //send msg to mount dbroot @@ -6361,7 +6395,7 @@ bool Oam::autoUnMovePmDbroot(std::string toPM) if (!found) { - writeLog("No dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_DEBUG ); + writeLog("No dbroots found in " + InstallDir + "/moveDbrootTransactionLog", LOG_TYPE_DEBUG ); cout << "No dbroots found in " << fileName << endl; } @@ -6950,32 +6984,6 @@ void Oam::assignDbroot(std::string toPM, DBRootConfigList& dbrootlist) for ( ; pt3 != dbrootlist.end() ; pt3++) { todbrootConfigList.push_back(*pt3); - - /* if ( DataRedundancyConfig == "y") - { - try { - string errmsg; - int ret = glusterctl(oam::GLUSTER_ASSIGN, itoa(*pt3), toPM, errmsg); - if ( ret != 0 ) - { - cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID + ", error: " + errmsg << endl; - exceptionControl("assignPmDbrootConfig", API_FAILURE); - } - } - catch (exception& e) - { - cout << endl << "**** glusterctl API exception: " << e.what() << endl; - cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID << endl; - exceptionControl("assignPmDbrootConfig", API_FAILURE); - } - catch (...) - { - cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; - cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID << endl; - exceptionControl("assignPmDbrootConfig", API_FAILURE); - } - } - */ } try @@ -7435,12 +7443,14 @@ void Oam::removeDbroot(DBRootConfigList& dbrootlist) { cout << endl << "**** glusterctl API exception: " << e.what() << endl; cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl; + writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); exceptionControl("removeDbroot", API_FAILURE); } catch (...) { cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl; + writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); exceptionControl("removeDbroot", API_FAILURE); } } diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index 6cea84596..37acc7c1e 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -463,7 +463,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st int pid = getpid(); int tid = gettid(); - // get reporting Pprocess Name + // get reporting Process Name string processName; if ( repProcessName.empty()) @@ -514,7 +514,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st args.add("sendAlarmReport error:"); args.add(e.what()); msg.format(args); - ml.logErrorMessage(msg); + ml.logDebugMessage(msg); } catch (std::exception& e) { @@ -525,7 +525,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st args.add("sendAlarmReport error:"); args.add(e.what()); msg.format(args); - ml.logErrorMessage(msg); + ml.logDebugMessage(msg); } catch (...) { @@ -536,7 +536,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st args.add("sendAlarmReport error:"); args.add("general failure"); msg.format(args); - ml.logErrorMessage(msg); + ml.logDebugMessage(msg); } return; diff --git a/oamapps/postConfigure/CMakeLists.txt b/oamapps/postConfigure/CMakeLists.txt index ebe7d7f3f..4bdbadd1a 100644 --- a/oamapps/postConfigure/CMakeLists.txt +++ b/oamapps/postConfigure/CMakeLists.txt @@ -37,13 +37,13 @@ install(TARGETS getMySQLpw DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### -set(amazonInstaller_SRCS amazonInstaller.cpp helpers.cpp) +#set(amazonInstaller_SRCS amazonInstaller.cpp helpers.cpp) -add_executable(amazonInstaller ${amazonInstaller_SRCS}) +#add_executable(amazonInstaller ${amazonInstaller_SRCS}) -target_link_libraries(amazonInstaller ${ENGINE_LDFLAGS} readline ncurses ${SNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +#target_link_libraries(amazonInstaller ${ENGINE_LDFLAGS} readline ncurses ${SNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS amazonInstaller DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +#install(TARGETS amazonInstaller DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### @@ -56,3 +56,8 @@ target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_ install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT platform) + +########### next target ############### + +install(PROGRAMS quick_installer_single_server.sh quick_installer_multi_server.sh quick_installer_amazon.sh + DESTINATION ${ENGINE_BINDIR} COMPONENT platform) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 3bc02c6ae..aefeb6d99 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -170,6 +170,12 @@ int main(int argc, char* argv[]) if (p && *p) USER = p; + // setup to start on reboot, for non-root amazon installs + if ( !rootUser ) + { + system("sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); + } + //copy Columnstore.xml.rpmsave if upgrade option is selected if ( installType == "upgrade" ) { @@ -896,7 +902,10 @@ int main(int argc, char* argv[]) cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; - cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl; + if ( !rootUser ) + cout << ". /etc/profile.d/columnstoreEnv.sh" << endl; + + cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl; cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; @@ -912,7 +921,10 @@ int main(int argc, char* argv[]) cout << endl << "ERROR: MariaDB ColumnStore Process failed to start, check log files in /var/log/mariadb/columnstore" << endl; cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; - cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl; + if ( !rootUser ) + cout << ". /etc/profile.d/columnstoreEnv.sh" << endl; + + cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl; cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9cc3bf5b2..2a737861c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + MA 02110-1301, USA. */ /****************************************************************************************** * $Id: postConfigure.cpp 64 2006-10-12 22:21:51Z dhill $ @@ -99,6 +99,14 @@ typedef struct Performance_Module_struct typedef std::vector PerformanceModuleList; +typedef struct ModuleIP_struct +{ + std::string IPaddress; + std::string moduleName; +} ModuleIP; + +typedef std::vector ModuleIpList; + void offLineAppCheck(); bool setOSFiles(string parentOAMModuleName, int serverTypeInstall); bool checkSaveConfigFile(); @@ -121,12 +129,6 @@ void remoteInstallThread(void*); bool glusterSetup(string password); -typedef struct ModuleIP_struct -{ - std::string IPaddress; - std::string moduleName; -} ModuleIP; - std::string launchInstance(ModuleIP moduleip); string columnstorePackage; @@ -150,7 +152,6 @@ string PMVolumeType = "standard"; string PMVolumeIOPS = oam::UnassignedName; string UMVolumeIOPS = oam::UnassignedName; - int DBRootCount; string deviceName; @@ -185,6 +186,10 @@ string MySQLRep = "y"; string PMwithUM = "n"; bool amazonInstall = false; bool nonDistribute = false; +bool nonDistributeFlag = false; +bool single_server_quick_install = false; +bool multi_server_quick_install = false; +bool amazon_quick_install = false; string DataFileEnvFile; @@ -220,6 +225,9 @@ int main(int argc, char* argv[]) noPrompting = false; string password; string cmd; + string pmIpAddrs = ""; + string umIpAddrs = ""; + // struct sysinfo myinfo; // hidden options @@ -278,74 +286,88 @@ int main(int argc, char* argv[]) if (p && *p) HOME = p; } + for ( int i = 1; i < argc; i++ ) { - if ( string("-h") == argv[i] ) - { + if( string("-h") == argv[i] ) + { cout << endl; cout << "This is the MariaDB ColumnStore System Configuration and Installation tool." << endl; cout << "It will Configure the MariaDB ColumnStore System based on Operator inputs and" << endl; cout << "will perform a Package Installation of all of the Modules within the" << endl; cout << "System that is being configured." << endl; cout << endl; - cout << "IMPORTANT: This tool should only be run on a Performance Module Server," << endl; - cout << " preferably Module #1" << endl; + cout << "IMPORTANT: This tool is required to run on a Performance Module #1 (pm1) Server." << endl; cout << endl; cout << "Instructions:" << endl << endl; cout << " Press 'enter' to accept a value in (), if available or" << endl; cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-s][-port][-i][-n]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; cout << " If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl; cout << " -p Unix Password, used with no-prompting option" << endl; - cout << " -s Single Threaded Remote Install" << endl; + cout << " -qs Quick Install - Single Server" << endl; + cout << " -qm Quick Install - Multi Server" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; cout << " -i Non-root Install directory, Only use for non-root installs" << endl; - cout << " -n Non-distributed install, meaning it will not install the remote nodes" << endl; + cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; + cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; + cout << " -sn System Name" << endl; + cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; + cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; exit (0); } - else if ( string("-s") == argv[i] ) - thread_remote_installer = false; + else if( string("-qs") == argv[i] ) + { + single_server_quick_install = true; + noPrompting = true; + } + else if( string("-qm") == argv[i] ) + { + multi_server_quick_install = true; + noPrompting = true; + } + else if( string("-qa") == argv[i] ) + { + amazon_quick_install = true; + noPrompting = true; + } else if ( string("-f") == argv[i] ) nodeps = "--nodeps"; else if ( string("-o") == argv[i] ) startOfflinePrompt = true; - else if ( string("-c") == argv[i] ) - { + else if( string("-c") == argv[i] ) + { i++; - - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Config File not provided" << endl; exit (1); } oldFileName = argv[i]; - - if ( oldFileName.find("Columnstore.xml") == string::npos ) + if ( oldFileName.find("Columnstore.xml") == string::npos ) { cout << " ERROR: Config File is not a Columnstore.xml file name" << endl; exit (1); } } - else if ( string("-p") == argv[i] ) - { + else if( string("-p") == argv[i] ) + { i++; - - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Password not provided" << endl; exit (1); } password = argv[i]; - - if ( password.find("-") != string::npos ) + if ( password.find("-") != string::npos ) { cout << " ERROR: Valid Password not provided" << endl; exit (1); @@ -355,12 +377,19 @@ int main(int argc, char* argv[]) noPrompting = true; // for backward compatibility else if ( string("-n") == argv[i] ) + { nonDistribute = true; - else if ( string("-port") == argv[i] ) - { + nonDistributeFlag = true; + } + else if( string("-d") == argv[i] ) + { + nonDistribute = false; + nonDistributeFlag = true; + } + else if( string("-port") == argv[i] ) + { i++; - - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: MariaDB ColumnStore Port ID not supplied" << endl; exit (1); @@ -374,11 +403,10 @@ int main(int argc, char* argv[]) exit (1); } } - else if ( string("-i") == argv[i] ) + else if( string("-i") == argv[i] ) { i++; - - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Path not provided" << endl; exit (1); @@ -386,11 +414,83 @@ int main(int argc, char* argv[]) installDir = argv[i]; } + else if( string("-sn") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: System-name not provided" << endl; + exit (1); + } + systemName = argv[i]; + } + else if( string("-pm-ip-addrs") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: PM-IP-ADRESSES not provided" << endl; + exit (1); + } + pmIpAddrs = argv[i]; + } + else if( string("-um-ip-addrs") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: UM-IP-ADRESSES not provided" << endl; + exit (1); + } + umIpAddrs = argv[i]; + } + else if( string("-pm-count") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: PM-COUNT not provided" << endl; + exit (1); + } + pmNumber = atoi(argv[i]); + } + else if( string("-um-count") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: UM-COUNT not provided" << endl; + exit (1); + } + umNumber = atoi(argv[i]); + } else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-s][-port][-i]" << endl; - exit (1); + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + exit (1); + } + } + + //check if quick install multi-server has been given ip address + if (multi_server_quick_install) + { + if ( ( umIpAddrs.empty() && pmIpAddrs.empty() ) || + ( !umIpAddrs.empty() && pmIpAddrs.empty() )) + { + cout << " ERROR: Multi-Server option entered, but invalid UM/PM IP addresses were provided, exiting" << endl; + exit(1); + } + } + + //check if quick install multi-server has been given ip address + if (amazon_quick_install) + { + if ( ( umNumber == 0 && pmNumber == 0 ) || + ( umNumber != 0 && pmNumber == 0 ) ) + { + cout << " ERROR: Amazon option entered, but invalid UM/PM Counts were provided, exiting" << endl; + exit(1); } } @@ -409,42 +509,43 @@ int main(int argc, char* argv[]) cout << "Installation of all of the Servers within the System that is being configured." << endl; cout << endl; - cout << "IMPORTANT: This tool should only be run on the Parent OAM Module" << endl; - cout << " which is a Performance Module, preferred Module #1" << endl; + cout << "IMPORTANT: This tool requires to run on the Performance Module #1" << endl; cout << endl; - if (!noPrompting) - { - cout << "Prompting instructions:" << endl << endl; - cout << " Press 'enter' to accept a value in (), if available or" << endl; - cout << " Enter one of the options within [], if available, or" << endl; - cout << " Enter a new value" << endl << endl; - } - else - { - //get current time and date - time_t now; - now = time(NULL); - struct tm tm; - localtime_r(&now, &tm); - char timestamp[200]; - strftime (timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm); - string currentDate = timestamp; + if (!single_server_quick_install || !multi_server_quick_install || !amazon_quick_install) + { + if (!noPrompting) { + cout << "Prompting instructions:" << endl << endl; + cout << " Press 'enter' to accept a value in (), if available or" << endl; + cout << " Enter one of the options within [], if available, or" << endl; + cout << " Enter a new value" << endl << endl; + } + else + { + //get current time and date + time_t now; + now = time(NULL); + struct tm tm; + localtime_r(&now, &tm); + char timestamp[200]; + strftime (timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm); + string currentDate = timestamp; - string postConfigureLog = "/var/log/columnstore-postconfigure-" + currentDate; + string postConfigureLog = "/var/log/columnstore-postconfigure-" + currentDate; - cout << "With the no-Prompting Option being specified, you will be required to have the following:" << endl; - cout << endl; - cout << " 1. Root user ssh keys setup between all nodes in the system or" << endl; - cout << " use the password command line option." << endl; - cout << " 2. A Configure File to use to retrieve configure data, default to Columnstore.xml.rpmsave" << endl; - cout << " or use the '-c' option to point to a configuration file." << endl; - cout << endl; -// cout << " Output if being redirected to " << postConfigureLog << endl; - -// redirectStandardOutputToFile(postConfigureLog, false ); - } + cout << "With the no-Prompting Option being specified, you will be required to have the following:" << endl; + cout << endl; + cout << " 1. Root user ssh keys setup between all nodes in the system or" << endl; + cout << " use the password command line option." << endl; + cout << " 2. A Configure File to use to retrieve configure data, default to Columnstore.xml.rpmsave" << endl; + cout << " or use the '-c' option to point to a configuration file." << endl; + cout << endl; + // cout << " Output if being redirected to " << postConfigureLog << endl; + // redirectStandardOutputToFile(postConfigureLog, false ); + } + } + //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { @@ -458,45 +559,43 @@ int main(int argc, char* argv[]) cout << "ERROR: Configuration File not setup" << endl; exit(1); } - + //determine package type string EEPackageType; - if (!rootUser) - EEPackageType = "binary"; - else - { - int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + if (!rootUser) + EEPackageType = "binary"; + else + { + int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "rpm"; - else - { - rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "rpm"; + else { + rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "deb"; - else - EEPackageType = "binary"; - } - } + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "deb"; + else + EEPackageType = "binary"; + } + } - try - { - sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); - } - catch (...) - { - cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + try { + sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); + } + catch(...) + { + cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + //check for local ip address as pm1 ModuleConfig moduleconfig; @@ -598,7 +697,6 @@ int main(int argc, char* argv[]) catch (...) {} - // run my.cnf upgrade script if ( reuseConfig == "y" ) { @@ -649,16 +747,25 @@ int main(int argc, char* argv[]) } //check for non-Distributed Install - if ( nonDistribute ) - { - try - { - oam.setSystemConfig("DistributedInstall", "n"); - } - catch (...) {} - } - else - { + if ( nonDistributeFlag ) + { + if ( nonDistribute ) + { + try { + oam.setSystemConfig("DistributedInstall", "n"); + } + catch(...) {} + } + else + { + try { + oam.setSystemConfig("DistributedInstall", "y"); + } + catch(...) {} + } + } + else + { //get Distributed Install try { @@ -672,163 +779,290 @@ int main(int argc, char* argv[]) } cout << endl; + + if (single_server_quick_install) + { + cout << "===== Quick Install Single-Server Configuration =====" << endl << endl; - cout << "===== Setup System Server Type Configuration =====" << endl << endl; + cout << "Single-Server install is used when there will only be 1 server configured" << endl; + cout << "on the system. It can also be used for production systems, if the plan is" << endl; + cout << "to stay single-server." << endl; + + singleServerInstall = "1"; + } + else if (multi_server_quick_install) + { + cout << "===== Quick Install Multi-Server Configuration =====" << endl << endl; + + singleServerInstall = "2"; + } + else if (amazon_quick_install) + { + cout << "===== Quick Install Amazon Configuration =====" << endl << endl; + + singleServerInstall = "2"; + } + else + { + cout << "===== Setup System Server Type Configuration =====" << endl << endl; - cout << "There are 2 options when configuring the System Server Type: single and multi" << endl << endl; - cout << " 'single' - Single-Server install is used when there will only be 1 server configured" << endl; - cout << " on the system. It can also be used for production systems, if the plan is" << endl; - cout << " to stay single-server." << endl << endl; - cout << " 'multi' - Multi-Server install is used when you want to configure multiple servers now or" << endl; - cout << " in the future. With Multi-Server install, you can still configure just 1 server" << endl; - cout << " now and add on addition servers/modules in the future." << endl << endl; + cout << "There are 2 options when configuring the System Server Type: single and multi" << endl << endl; + cout << " 'single' - Single-Server install is used when there will only be 1 server configured" << endl; + cout << " on the system. It can also be used for production systems, if the plan is" << endl; + cout << " to stay single-server." << endl << endl; + cout << " 'multi' - Multi-Server install is used when you want to configure multiple servers now or" << endl; + cout << " in the future. With Multi-Server install, you can still configure just 1 server" << endl; + cout << " now and add on addition servers/modules in the future." << endl << endl; + + string temp; - string temp; try { - temp = sysConfig->getConfig(InstallSection, "SingleServerInstall"); - } - catch (...) - {} + temp = sysConfig->getConfig(InstallSection, "SingleServerInstall"); + } + catch(...) + {} - if ( temp == "y" ) - singleServerInstall = "1"; - else - singleServerInstall = "2"; + if ( temp == "y" ) + singleServerInstall = "1"; + else + singleServerInstall = "2"; - while (true) - { - prompt = "Select the type of System Server install [1=single, 2=multi] (" + singleServerInstall + ") > "; - pcommand = callReadline(prompt.c_str()); - string temp; + while(true) { + string temp = singleServerInstall; + prompt = "Select the type of System Server install [1=single, 2=multi] (" + singleServerInstall + ") > "; + pcommand = callReadline(prompt.c_str()); if (pcommand) { - if (strlen(pcommand) > 0) - temp = pcommand; - else - temp = singleServerInstall; + if (strlen(pcommand) > 0) + temp = pcommand; + else + temp = singleServerInstall; + + callFree(pcommand); - callFree(pcommand); + if (temp == "1") { + singleServerInstall = temp; + + cout << endl << "Performing the Single Server Install." << endl; - if (temp == "1") - { - singleServerInstall = temp; - cout << endl << "Performing the Single Server Install." << endl; + break; + } + else + { + if (temp == "2") { + singleServerInstall = temp; + break; + } + } - if ( reuseConfig == "n" ) - { - //setup to use the single server Columnstore.xml file + cout << "Invalid Entry, please re-enter (1 or 2)" << endl; + if ( noPrompting ) + exit(1); + } + } + } + + if (singleServerInstall == "1") + { + if ( reuseConfig == "n" ) { + //setup to use the single server Columnstore.xml file - // we know that our Config instance just timestamped itself in the getConfig - // call above. if postConfigure is running non-interactively we may get here - // within the same second which means the changes that are about to happen - // when Columnstore.xml gets overwritten will be ignored because of the Config - // instance won't know to reload - sleep(2); + // we know that our Config instance just timestamped itself in the getConfig + // call above. if postConfigure is running non-interactively we may get here + // within the same second which means the changes that are about to happen + // when Columnstore.xml gets overwritten will be ignored because of the Config + // instance won't know to reload + sleep(1); - cmd = "rm -f " + installDir + "/etc/Columnstore.xml.installSave > /dev/null 2>&1"; - system(cmd.c_str()); - cmd = "mv -f " + installDir + "/etc/Columnstore.xml " + installDir + "/etc/Columnstore.xml.installSave > /dev/null 2>&1"; - system(cmd.c_str()); - cmd = "/bin/cp -f " + installDir + "/etc/Columnstore.xml.singleserver " + installDir + "/etc/Columnstore.xml > /dev/null 2>&1"; - system(cmd.c_str()); - } + cmd = "rm -f " + installDir + "/etc/Columnstore.xml.installSave > /dev/null 2>&1"; + system(cmd.c_str()); + cmd = "mv -f " + installDir + "/etc/Columnstore.xml " + installDir + "/etc/Columnstore.xml.installSave > /dev/null 2>&1"; + system(cmd.c_str()); + cmd = "/bin/cp -f " + installDir + "/etc/Columnstore.xml.singleserver " + installDir + "/etc/Columnstore.xml > /dev/null 2>&1"; + system(cmd.c_str()); + } - setSystemName(); - cout << endl; + setSystemName(); - system(cmd.c_str()); + if (!single_server_quick_install) + { + cout << endl; - // setup storage - if (!storageSetup(false)) - { - cout << "ERROR: Problem setting up storage" << endl; - exit(1); - } + // setup storage + if (!storageSetup(false)) + { + cout << "ERROR: Problem setting up storage" << endl; + exit(1); + } - if (hdfs || !rootUser) - if ( !updateBash() ) - cout << "updateBash error" << endl; + // setup storage + if (!singleServerDBrootSetup()) + { + cout << "ERROR: Problem setting up DBRoot IDs" << endl; + exit(1); + } - // setup storage - if (!singleServerDBrootSetup()) - { - cout << "ERROR: Problem setting up DBRoot IDs" << endl; - exit(1); - } + //set system DBRoot count and check 'files per parition' with number of dbroots + try { + sysConfig->setConfig(SystemSection, "DBRootCount", oam.itoa(DBRootCount)); + } + catch(...) + { + cout << "ERROR: Problem setting DBRoot Count in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - //set system DBRoot count and check 'files per parition' with number of dbroots - try - { - sysConfig->setConfig(SystemSection, "DBRootCount", oam.itoa(DBRootCount)); - } - catch (...) - { - cout << "ERROR: Problem setting DBRoot Count in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + //check if dbrm data resides in older directory path and inform user if it does + dbrmDirCheck(); - //check if dbrm data resides in older directory path and inform user if it does - dbrmDirCheck(); + if (startOfflinePrompt) + offLineAppCheck(); + } - if (startOfflinePrompt) - offLineAppCheck(); + checkMysqlPort(mysqlPort, sysConfig); - checkMysqlPort(mysqlPort, sysConfig); + if (hdfs || !rootUser) + if( !updateBash() ) + cout << "updateBash error" << endl; - if ( !writeConfig(sysConfig) ) - { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; + cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; - system(cmd.c_str()); - exit(0); - } - else - { - if (temp == "2") - { - singleServerInstall = temp; - break; - } - } + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; + system(cmd.c_str()); + exit(0); + } - cout << "Invalid Entry, please re-enter" << endl; + try { + sysConfig->setConfig(InstallSection, "SingleServerInstall", "n"); + } + catch(...) + { + cout << "ERROR: Problem setting SingleServerInstall from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - if ( noPrompting ) - exit(1); + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - continue; - } + // + // Multi-server install + // + + ModuleIP InputModuleIP; + ModuleIpList InputModuleIPList; - break; - } + int MaxNicID = oam::MAX_NIC; - try - { - sysConfig->setConfig(InstallSection, "SingleServerInstall", "n"); - } - catch (...) - { - cout << "ERROR: Problem setting SingleServerInstall from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + if (multi_server_quick_install) + { + //set configuarion settings for default setup + try { + sysConfig->setConfig(InstallSection, "MySQLRep", "y"); + } + catch(...) + {} + + if (umIpAddrs == "" ) + { + // set Server Type Installation to combined + try { + sysConfig->setConfig(InstallSection, "ServerTypeInstall", "2"); + } + catch(...) + {} + } + else + { + int id = 1; + boost::char_separator sep(","); + boost::tokenizer< boost::char_separator > tokens(umIpAddrs, sep); + for ( boost::tokenizer< boost::char_separator >::iterator it = tokens.begin(); + it != tokens.end(); + ++it, ++id) + { + string module = "um" + oam.itoa(id); - if ( !writeConfig(sysConfig) ) - { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + InputModuleIP.IPaddress = *it; + InputModuleIP.moduleName = module; + InputModuleIPList.push_back(InputModuleIP); + } + + umNumber = id-1; + } + + if (pmIpAddrs != "" ) + { + int id = 1; + boost::char_separator sep(","); + boost::tokenizer< boost::char_separator > tokens(pmIpAddrs, sep); + for ( boost::tokenizer< boost::char_separator >::iterator it = tokens.begin(); + it != tokens.end(); + ++it, ++id) + { + string module = "pm" + oam.itoa(id); - // - // Multi-server install - // + InputModuleIP.IPaddress = *it; + InputModuleIP.moduleName = module; + InputModuleIPList.push_back(InputModuleIP); + } + + pmNumber = id-1; + } + + if ( !writeConfig(sysConfig) ) + { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + MaxNicID = 1; + } + else + { + if (amazon_quick_install) + { + //set configuarion settings for default setup + try { + sysConfig->setConfig(InstallSection, "MySQLRep", "y"); + } + catch(...) + {} + + try { + sysConfig->setConfig(InstallSection, "Cloud", "amazon-vpc"); + } + catch(...) + {} + + if (umNumber == 0 ) + { + // set Server Type Installation to combined + try { + sysConfig->setConfig(InstallSection, "ServerTypeInstall", "2"); + } + catch(...) + {} + } + + if ( !writeConfig(sysConfig) ) + { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + MaxNicID = 1; + } + } cout << endl; //cleanup/create local/etc directory @@ -890,9 +1124,8 @@ int main(int argc, char* argv[]) exit(1); } - switch ( IserverTypeInstall ) - { - case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server + switch ( IserverTypeInstall ) { + case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - um/pm on a single server { cout << "Combined Server Installation will be performed." << endl; cout << "The Server will be configured as a Performance Module." << endl; @@ -1048,8 +1281,8 @@ int main(int argc, char* argv[]) if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; + mysqlRep = true; + MySQLRep = "y"; } else { @@ -1087,24 +1320,45 @@ int main(int argc, char* argv[]) //amazon install setup check bool amazonInstall = false; string cloud = oam::UnassignedName; - system("aws --version > /tmp/amazon.log 2>&1"); + + if (!multi_server_quick_install) + { + system("aws --version > /tmp/amazon.log 2>&1"); - ifstream in("/tmp/amazon.log"); + ifstream in("/tmp/amazon.log"); - in.seekg(0, std::ios::end); - int size = in.tellg(); + in.seekg(0, std::ios::end); + int size = in.tellg(); - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) - // not running on amazon with ec2-api-tools - amazonInstall = false; - else - { - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) - amazonInstall = false; - else - amazonInstall = true; - } + if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) + { + // not running on amazon with ec2-api-tools + if (amazon_quick_install) + { + cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + exit(1); + } + amazonInstall = false; + } + else + { + if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) + { + // not running on amazon with ec2-api-tools + if (amazon_quick_install) + { + cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + exit(1); + } + + amazonInstall = false; + } + else + amazonInstall = true; + } + } + try { cloud = sysConfig->getConfig(InstallSection, "Cloud"); @@ -1142,7 +1396,7 @@ int main(int argc, char* argv[]) try { - sysConfig->setConfig(InstallSection, "Cloud", "disable"); + sysConfig->setConfig(InstallSection, "Cloud", "disable"); } catch (...) {}; @@ -1586,6 +1840,14 @@ int main(int argc, char* argv[]) catch (...) {} } + + if ( moduleType == "um") + if ( umNumber != 0 ) + moduleCount = umNumber; + + if ( moduleType == "pm") + if ( pmNumber != 0 ) + moduleCount = pmNumber; //verify and setup of modules count switch ( IserverTypeInstall ) @@ -1690,7 +1952,7 @@ int main(int argc, char* argv[]) //clear any Equipped Module IP addresses that aren't in current ID range for ( int j = 0 ; j < listSize ; j++ ) { - for ( unsigned int k = 1 ; k < MAX_NIC + 1 ; k++) + for ( unsigned int k = 1 ; k < MaxNicID+1 ; k++) { string ModuleIPAddr = "ModuleIPAddr" + oam.itoa(j + 1) + "-" + oam.itoa(k) + "-" + oam.itoa(i + 1); @@ -1745,43 +2007,68 @@ int main(int argc, char* argv[]) moduleDisableState = oam::ENABLEDSTATE; //setup HostName/IPAddress for each NIC - for ( unsigned int nicID = 1 ; nicID < MAX_NIC + 1 ; nicID++ ) - { - string moduleHostName = oam::UnassignedName; - string moduleIPAddr = oam::UnassignedIpAddr; + + string moduleHostName = oam::UnassignedName; + string moduleIPAddr = oam::UnassignedIpAddr; - DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); + bool found = false; + if (multi_server_quick_install) + { + ModuleIpList::iterator pt2 = InputModuleIPList.begin(); + for( ; pt2 != InputModuleIPList.end() ; pt2++) + { + if ( (*pt2).moduleName == newModuleName ) + { + moduleHostName = (*pt2).IPaddress; + moduleIPAddr = (*pt2).IPaddress; + found = true; + break; + } + } + } + + unsigned int nicID=1; + for( ; nicID < MaxNicID +1 ; nicID++ ) + { + if ( !found ) + { + moduleHostName = oam::UnassignedName; + moduleIPAddr = oam::UnassignedIpAddr; - for ( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) - { - if (newModuleName == (*listPT).DeviceName) - { - if ( nicID == 1 ) - { - moduleDisableState = (*listPT).DisableState; + DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); - if ( moduleDisableState.empty() || - moduleDisableState == oam::UnassignedName || - moduleDisableState == oam::AUTODISABLEDSTATE ) - moduleDisableState = oam::ENABLEDSTATE; - } + for( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) + { + if (newModuleName == (*listPT).DeviceName) + { + if ( nicID == 1 ) + { + moduleDisableState = (*listPT).DisableState; - HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); + if ( moduleDisableState.empty() || + moduleDisableState == oam::UnassignedName || + moduleDisableState == oam::AUTODISABLEDSTATE ) + moduleDisableState = oam::ENABLEDSTATE; + { + HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); - for ( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) - { - if ((*pt1).NicID == nicID) - { - moduleHostName = (*pt1).HostName; - moduleIPAddr = (*pt1).IPAddr; - break; - } - } - } - } - - if ( nicID == 1 ) - { + for( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) + { + if ((*pt1).NicID == nicID) + { + moduleHostName = (*pt1).HostName; + moduleIPAddr = (*pt1).IPAddr; + break; + } + } + } + } + } + } + } + + if ( nicID == 1 ) + { if ( moduleDisableState != oam::ENABLEDSTATE ) { string disabled = "y"; @@ -1880,6 +2167,7 @@ int main(int argc, char* argv[]) break; } + bool moduleHostNameFound = true; if (moduleHostName.empty()) @@ -1924,28 +2212,30 @@ int main(int argc, char* argv[]) //check if need to create instance or user enter ID string create = "y"; - while (true) - { - pcommand = callReadline("Create Instance for " + newModuleName + " [y,n] (y) > "); + if ( !amazon_quick_install ) + { + while(true) + { + pcommand = callReadline("Create Instance for " + newModuleName + " [y,n] (y) > "); - if (pcommand) - { - if (strlen(pcommand) > 0) create = pcommand; + if (pcommand) + { + if (strlen(pcommand) > 0) create = pcommand; - callFree(pcommand); - } + callFree(pcommand); + } - if ( create == "y" || create == "n" ) - break; - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + if ( create == "y" || create == "n" ) + break; + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - create = "y"; - - if ( noPrompting ) - exit(1); - } + create = "y"; + if ( noPrompting ) + exit(1); + } + } if ( create == "y" ) { @@ -2591,32 +2881,37 @@ int main(int argc, char* argv[]) string dbrootList; - for ( unsigned int id = 1 ; id < count + 1 ; ) - { - string moduledbrootid = "ModuleDBRootID" + oam.itoa(moduleID) + "-" + oam.itoa(id) + "-" + oam.itoa(i + 1); + if (multi_server_quick_install || amazon_quick_install) + { + dbrootList = oam.itoa(moduleID); + } + else + { + for ( unsigned int id = 1 ; id < count+1 ; ) + { + string moduledbrootid = "ModuleDBRootID" + oam.itoa(moduleID) + "-" + oam.itoa(id) + "-" + oam.itoa(i+1); + try { + string dbrootid = sysConfig->getConfig(ModuleSection, moduledbrootid); - try - { - string dbrootid = sysConfig->getConfig(ModuleSection, moduledbrootid); - - if ( dbrootid != oam::UnassignedName) + if ( dbrootid != oam::UnassignedName) { - sysConfig->setConfig(ModuleSection, moduledbrootid, oam::UnassignedName); + sysConfig->setConfig(ModuleSection, moduledbrootid, oam::UnassignedName); - dbrootList = dbrootList + dbrootid; - id ++; - - if ( id < count + 1 ) - dbrootList = dbrootList + ","; - } - } - catch (...) - { - cout << "ERROR: Problem setting DBRoot ID in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - } + dbrootList = dbrootList + dbrootid; + id ++; + if ( id < count+1 ) + dbrootList = dbrootList + ","; + } + } + catch(...) + { + cout << "ERROR: Problem setting DBRoot ID in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + } + } + vector dbroots; string tempdbrootList; @@ -3151,8 +3446,9 @@ int main(int argc, char* argv[]) //check if dbrm data resides in older directory path and inform user if it does dbrmDirCheck(); - if ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM && pmNumber == 1) - { + if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || + ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) + { //run the mysql / mysqld setup scripts cout << endl << "===== Running the MariaDB ColumnStore MariaDB Server setup scripts =====" << endl << endl; @@ -3160,7 +3456,57 @@ int main(int argc, char* argv[]) // call the mysql setup scripts mysqlSetup(); - sleep(5); + sleep(3); + } + + if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || + pmNumber > 1 ) + { + if ( password.empty() ) + { + cout << endl; + cout << "Next step is to enter the password to access the other Servers." << endl; + cout << "This is either your password or you can default to using a ssh key" << endl; + cout << "If using a password, the password needs to be the same on all Servers." << endl << endl; + + if ( noPrompting ) { + cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl; + password = "ssh"; + } + else + { + while(true) + { + char *pass1, *pass2; + + pass1=getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > "); + if ( strcmp(pass1, "") == 0 ) { + password = "ssh"; + break; + } + + string p1 = pass1; + if ( p1 == "exit") + exit(0); + + pass2=getpass("Confirm password > "); + string p2 = pass2; + if ( p1 == p2 ) { + password = p2; + break; + } + else + cout << "Password mismatch, please re-enter" << endl; + } + + //add single quote for special characters + if ( password != "ssh" ) + { + password = "'" + password + "'"; + } + + } + } } int thread_id = 0; @@ -3235,74 +3581,8 @@ int main(int argc, char* argv[]) if ( !pkgCheck(columnstorePackage) ) exit(1); - if ( password.empty() ) - { - cout << endl; - cout << "Next step is to enter the password to access the other Servers." << endl; - cout << "This is either your password or you can default to using a ssh key" << endl; - cout << "If using a password, the password needs to be the same on all Servers." << endl << endl; - } - - while (true) - { - char* pass1, *pass2; - - if ( noPrompting ) - { - cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl; - - if ( password.empty() ) - password = "ssh"; - - break; - } - - //check for command line option password - if ( !password.empty() ) - break; - - pass1 = getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > "); - - if ( strcmp(pass1, "") == 0 ) - { - password = "ssh"; - break; - } - - if ( pass1 == "exit") - exit(0); - - string p1 = pass1; - pass2 = getpass("Confirm password > "); - string p2 = pass2; - - if ( p1 == p2 ) - { - password = p2; - break; - } - else - cout << "Password mismatch, please re-enter" << endl; - } - - //add single quote for special characters - if ( password != "ssh" ) - { - password = "'" + password + "'"; - } - checkSystemMySQLPort(mysqlPort, sysConfig, USER, password, childmodulelist, IserverTypeInstall, pmwithum); - if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || - ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) - { - cout << endl << "===== Running the MariaDB ColumnStore MariaDB ColumnStore setup scripts =====" << endl << endl; - - // call the mysql setup scripts - mysqlSetup(); - sleep(5); - } - string AmazonInstall = "0"; if ( amazonInstall ) @@ -3507,19 +3787,7 @@ int main(int argc, char* argv[]) cout << " DONE" << endl; } } - else - { - if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || - ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) - { - cout << endl << "===== Running the MariaDB ColumnStore MariaDB ColumnStore setup scripts =====" << endl << endl; - - // call the mysql setup scripts - mysqlSetup(); - sleep(5); - } - } - + //configure data redundancy if (DataRedundancy) { @@ -3762,9 +4030,6 @@ int main(int argc, char* argv[]) } //set mysql replication, if wasn't setup before on system -// if ( ( mysqlRep && pmwithum ) || -// ( mysqlRep && (umNumber > 1) ) || -// ( mysqlRep && (pmNumber > 1) && (IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM) ) ) if ( mysqlRep ) { cout << endl << "Run MariaDB ColumnStore Replication Setup.. "; @@ -3787,7 +4052,10 @@ int main(int argc, char* argv[]) cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; - cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl; + if ( !rootUser ) + cout << ". /etc/profile.d/columnstoreEnv.sh" << endl; + + cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl; cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; @@ -3804,7 +4072,10 @@ int main(int argc, char* argv[]) cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; - cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl; + if ( !rootUser ) + cout << ". /etc/profile.d/columnstoreEnv.sh" << endl; + + cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl; cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; @@ -3830,16 +4101,25 @@ bool checkSaveConfigFile() //check if Columnstore.xml.rpmsave exist ifstream File (oldFileName.c_str()); - if (!File) - { - if ( noPrompting ) - { - cout << endl << "Old Config File not found '" + oldFileName + "', exiting" << endl; - exit(1); - } - - return true; - } + if (!File) { + if (single_server_quick_install || multi_server_quick_install || amazon_quick_install) + { + return true; + } + if ( noPrompting ) { + cout << endl << "Old Config File not found '" + oldFileName + "', exiting" << endl; + exit(1); + } + return true; + } + else + { + if (single_server_quick_install || multi_server_quick_install || amazon_quick_install) + { + cout << endl << "Quick Install is for fresh installs only, '" + oldFileName + "' exist, exiting" << endl; + exit(1); + } + } File.close(); @@ -5443,28 +5723,34 @@ void setSystemName() Oam oam; //setup System Name - try - { - systemName = sysConfig->getConfig(SystemSection, "SystemName"); - } - catch (...) - { - systemName = oam::UnassignedName; - } - + + if ( systemName.empty() ) + { + try { + systemName = sysConfig->getConfig(SystemSection, "SystemName"); + } + catch(...) + { + systemName = oam::UnassignedName; + } + } + if ( systemName.empty() ) - systemName = oam::UnassignedName; + systemName = "columnstore-1"; - prompt = "Enter System Name (" + systemName + ") > "; - pcommand = callReadline(prompt.c_str()); + if (!single_server_quick_install || !multi_server_quick_install) + { + prompt = "Enter System Name (" + systemName + ") > "; + pcommand = callReadline(prompt.c_str()); - if (pcommand) - { - if (strlen(pcommand) > 0) systemName = pcommand; - - callFree(pcommand); - } + if (pcommand) + { + if (strlen(pcommand) > 0) systemName = pcommand; + callFree(pcommand); + } + } + try { sysConfig->setConfig(SystemSection, "SystemName", systemName); diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh new file mode 100755 index 000000000..4a1df903c --- /dev/null +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# $Id: quick_installer_amazon.sh 3705 2018-07-07 19:47:20Z dhill $ +# +# Poddst- Quick Installer for Amazon MariaDB Columnstore + +pmCount="" +umCount="" +systemName="" + +for arg in "$@"; do + if [ `expr -- "$arg" : '--pm-count='` -eq 11 ]; then + pmCount="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--um-count='` -eq 11 ]; then + umCount="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--system-name='` -eq 14 ]; then + systemName="`echo $arg | awk -F= '{print $2}'`" + systemName="-sn "$systemName + elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then + nonDistrubutedInstall=" " + elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then + echo "Usage ./quick_installer_amazon.sh [OPTION]" + echo "" + echo "Quick Installer for an Amazon MariaDB ColumnStore Install" + echo "This requires to be run on a MariaDB ColumnStore AMI" + echo "" + echo "Performace Module (pm) number is required" + echo "User Module (um) number is option" + echo "When only pm counts provided, system is combined setup" + echo "When both pm/um counts provided, system is seperate setup" + echo + echo "--pm-count=x Number of pm instances to create" + echo "--um-count=x Number of um instances to create, optional" + echo "--system-name=nnnn System Name, optional" + echo "" + exit 1 + else + echo "./quick_installer_amazon.sh: unknown argument: $arg, enter --help for help" 1>&2 + exit 1 + fi +done + +if [[ $pmCount = "" ]]; then + echo "" + echo "Performace Module (pm) count is required, exiting" + exit 1 +else + if [[ $umCount = "" ]]; then + echo "" + echo "NOTE: Performing a Multi-Server Combined install with um/pm running on some server" + echo"" + else + echo "" + echo "NOTE: Performing a Multi-Server Seperate install with um and pm running on seperate servers" + echo"" + fi +fi + +if [[ $HOME = "/root" ]]; then + echo "${bold}Run post-install script${normal}" + echo "" + /usr/local/mariadb/columnstore/bin/post-install + echo "${bold}Run postConfigure script${normal}" + echo "" + if [[ $umCount = "" ]]; then + /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount $systemName + else + /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount -um-count $umCount $systemName + fi +else + echo "${bold}Run post-install script${normal}" + echo "" + $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + echo "${bold}Run postConfigure script${normal}" + echo "" + if [[ $umCount = "" ]]; then + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName + else + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName + fi +fi diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh new file mode 100755 index 000000000..dbb603220 --- /dev/null +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# $Id: quick_installer_multi_server.sh 3705 2018-07-07 19:47:20Z dhill $ +# +# Poddst- Quick Installer for Multi Server MariaDB Columnstore + +pmIpAddrs="" +umIpAddrs="" +nonDistrubutedInstall="-n" +systemName="" + +for arg in "$@"; do + if [ `expr -- "$arg" : '--pm-ip-addresses='` -eq 18 ]; then + pmIpAddrs="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--um-ip-addresses='` -eq 18 ]; then + umIpAddrs="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--system-name='` -eq 14 ]; then + systemName="`echo $arg | awk -F= '{print $2}'`" + systemName="-sn "$systemName + elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then + nonDistrubutedInstall=" " + elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then + echo "Usage ./quick_installer_multi_server.sh [OPTION]" + echo "" + echo "Quick Installer for a Multi Server MariaDB ColumnStore Install" + echo "" + echo "Defaults to non-distrubuted install, meaning MariaDB Columnstore" + echo "needs to be preinstalled on all nodes in the system" + echo "" + echo "Performace Module (pm) IP addresses are required" + echo "User Module (um) IP addresses are option" + echo "When only pm IP addresses provided, system is combined setup" + echo "When both pm/um IP addresses provided, system is seperate setup" + echo + echo "--pm-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" + echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx, optional" + echo "--dist-install Use Distributed Install, optional" + echo "--system-name=nnnn System Name, optional" + echo "" + exit 1 + else + echo "quick_installer_multi_server.sh: unknown argument: $arg, enter --help for help" 1>&2 + exit 1 + fi +done + +if [[ $pmIpAddrs = "" ]]; then + echo "" + echo "Performace Module (pm) IP addresses required, exiting" + exit 1 +else + if [[ $umIpAddrs = "" ]]; then + echo "" + echo "NOTE: Performing a Multi-Server Combined install with um/pm running on some server" + echo"" + else + echo "" + echo "NOTE: Performing a Multi-Server Seperate install with um and pm running on seperate servers" + echo"" + fi +fi + +if [[ $HOME = "/root" ]]; then + echo "${bold}Run post-install script${normal}" + echo "" + /usr/local/mariadb/columnstore/bin/post-install + echo "${bold}Run postConfigure script${normal}" + echo "" + if [[ $umIpAddrs = "" ]]; then + /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName + else + /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName + fi +else + echo "${bold}Run post-install script${normal}" + echo "" + $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + echo "${bold}Run postConfigure script${normal}" + echo "" + if [[ $umIpAddrs = "" ]]; then + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName + else + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName + fi +fi diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh new file mode 100755 index 000000000..432b395c4 --- /dev/null +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# $Id: quick_installer_single_server.sh 3705 2018-07-07 19:47:20Z dhill $ +# +# Poddst- Quick Installer for Single Server MariaDB Columnstore + +for arg in "$@"; do + if [ `expr -- "$arg" : '--help'` -eq 6 ]; then + echo "Usage ./quick_installer_multi_server.sh" + echo "" + echo "Quick Installer for a Single Server MariaDB ColumnStore Install" + echo "" + exit 1 + else + echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2 + fi +done + + +if [ $HOME == "/root" ]; then + echo "Run post-install script" + echo "" + /usr/local/mariadb/columnstore/bin/post-install + echo "Run postConfigure script" + echo "" + /usr/local/mariadb/columnstore/bin/postConfigure -qs +else + echo "Run post-install script" + echo "" + $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + echo "Run postConfigure script" + echo "" + . /etc/profile.d/columnstoreEnv.sh; $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs +fi diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index f31273b1c..2b4450d2c 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -65,8 +65,6 @@ inline uint64_t order_swap(uint64_t x) template inline string fixChar(int64_t intval); -idb_regex_t placeholderRegex; - template inline int compareBlock( const void* a, const void* b ) { @@ -1095,6 +1093,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in, uint16_t* ridArray = 0; uint8_t* in8 = reinterpret_cast(in); const uint8_t filterSize = sizeof(uint8_t) + sizeof(uint8_t) + W; + idb_regex_t placeholderRegex; placeholderRegex.used = false; diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 140926266..1b19ccd0f 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -321,6 +321,22 @@ int main(int argc, char* argv[]) // This is unset due to the way we start it program_invocation_short_name = const_cast("PrimProc"); + int gDebug = 0; + int c; + + while ((c = getopt(argc, argv, "d")) != EOF) + { + switch(c) + { + case 'd': + gDebug++; + break; + case '?': + default: + break; + } + } + Config* cf = Config::makeConfig(); setupSignalHandlers(); @@ -329,7 +345,9 @@ int main(int argc, char* argv[]) mlp = new primitiveprocessor::Logger(); - int err = setupResources(); + int err = 0; + if (!gDebug) + err = setupResources(); string errMsg; switch (err) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 05160da1e..0aff2dcd6 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1657,9 +1657,6 @@ void pingDeviceThread() break; //set query system state not ready - BRM::DBRM dbrm; - dbrm.setSystemQueryReady(false); - processManager.setQuerySystemState(false); processManager.setSystemState(oam::BUSY_INIT); @@ -1734,7 +1731,7 @@ void pingDeviceThread() //set query system state ready processManager.setQuerySystemState(true); - break; + goto break_case; } } catch (...) @@ -1756,25 +1753,24 @@ void pingDeviceThread() if ( retry == 5 ) { log.writeLog(__LINE__, "autoUnMovePmDbroot: Failed. Fail Module", LOG_TYPE_WARNING); - + log.writeLog(__LINE__, "System DBRM READ ONLY - Verify dbroot mounts.", LOG_TYPE_WARNING); //Issue an alarm aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, SET); //set module to disable state processManager.disableModule(moduleName, true); - //call dbrm control - oam.dbrmctl("reload"); - log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); + // Need to do something here to verify data mounts before resuming + // Best to assume if we reach this you need to put into readonly and verify all dbroots are mounted - // resume the dbrm - oam.dbrmctl("resume"); - log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); + //call dbrm control + oam.dbrmctl("readonly"); + log.writeLog(__LINE__, "'dbrmctl readonly' done", LOG_TYPE_DEBUG); //clear count moduleInfoList[moduleName] = 0; - processManager.setSystemState(oam::ACTIVE); + processManager.setSystemState(oam::DEGRADED); //set query system state ready processManager.setQuerySystemState(true); @@ -1972,9 +1968,6 @@ void pingDeviceThread() } } - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); @@ -2034,9 +2027,6 @@ void pingDeviceThread() else processManager.setSystemState(oam::ACTIVE); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); @@ -2102,9 +2092,6 @@ void pingDeviceThread() log.writeLog(__LINE__, "module is down: " + moduleName, LOG_TYPE_CRITICAL); //set query system state not ready - BRM::DBRM dbrm; - dbrm.setSystemQueryReady(false); - processManager.setQuerySystemState(false); processManager.setSystemState(oam::BUSY_INIT); @@ -2169,9 +2156,6 @@ void pingDeviceThread() oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); @@ -2393,9 +2377,6 @@ void pingDeviceThread() //set recycle process processManager.recycleProcess(moduleName); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); @@ -2412,9 +2393,6 @@ void pingDeviceThread() oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); } @@ -2429,9 +2407,6 @@ void pingDeviceThread() //set recycle process processManager.recycleProcess(moduleName); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); } @@ -2564,6 +2539,7 @@ void pingDeviceThread() } } //end of for loop } + break_case: // check and take action if LAN outage is flagged if (LANOUTAGESUPPORT && !LANOUTAGEACTIVE && LOCALNICDOWN) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index f9f5a8d47..595a86f88 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -405,7 +405,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) msg >> target; msg >> graceful; msg >> ackIndicator; - msg >> manualFlag; + msg >> manualFlag; switch (actionType) { @@ -902,29 +902,31 @@ void processMSG(messageqcpp::IOSocket* cfIos) } if (opState == oam::MAN_OFFLINE || opState == oam::MAN_DISABLED - || opState == oam::AUTO_DISABLED ) + || opState == oam::AUTO_DISABLED || opState == oam::AUTO_OFFLINE) { - oam.dbrmctl("halt"); - log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG); + processManager.setSystemState(oam::BUSY_INIT); + + //set query system state not ready + processManager.setQuerySystemState(false); status = processManager.disableModule(moduleName, true); log.writeLog(__LINE__, "Disable Module Completed on " + moduleName, LOG_TYPE_INFO); - //call dbrm control - oam.dbrmctl("reload"); - log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); - - // resume the dbrm - oam.dbrmctl("resume"); - log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); + processManager.recycleProcess(moduleName); //check for SIMPLEX Processes on mate might need to be started processManager.checkSimplexModule(moduleName); + + processManager.setSystemState(oam::ACTIVE); + + //set query system state ready + processManager.setQuerySystemState(true); + } else { - log.writeLog(__LINE__, "ERROR: module not stopped", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "ERROR: module not stopped, state = " + oam.itoa(opState), LOG_TYPE_ERROR); status = API_FAILURE; break; } @@ -987,7 +989,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) DeviceNetworkList::iterator listPT = devicenetworklist.begin(); - //stopModules being removed with the REMOVE option, which will stop process + // do stopmodule then enable for ( ; listPT != devicenetworklist.end() ; listPT++) { string moduleName = (*listPT).DeviceName; @@ -1013,6 +1015,9 @@ void processMSG(messageqcpp::IOSocket* cfIos) if (opState == oam::MAN_DISABLED) { + processManager.stopModule(moduleName, graceful, manualFlag); + log.writeLog(__LINE__, "stop Module Completed on " + moduleName, LOG_TYPE_INFO); + status = processManager.enableModule(moduleName, oam::MAN_OFFLINE); log.writeLog(__LINE__, "Enable Module Completed on " + moduleName, LOG_TYPE_INFO); } @@ -1357,6 +1362,9 @@ void processMSG(messageqcpp::IOSocket* cfIos) log.writeLog(__LINE__, "STOPSYSTEM: ACK back to sender"); } + //set query system state ready + processManager.setQuerySystemState(true); + startsystemthreadStop = false; break; @@ -3049,9 +3057,6 @@ void processMSG(messageqcpp::IOSocket* cfIos) log.writeLog(__LINE__, "MSG RECEIVED: Process Restarted on " + moduleName + "/" + processName); //set query system states not ready - BRM::DBRM dbrm; - dbrm.setSystemQueryReady(false); - processManager.setQuerySystemState(false); processManager.setSystemState(oam::BUSY_INIT); @@ -3150,14 +3155,15 @@ void processMSG(messageqcpp::IOSocket* cfIos) sleep(1); } + processManager.setQuerySystemState(true); - dbrm.setSystemQueryReady(true); } // if a DDLProc was restarted, reinit DMLProc if ( processName == "DDLProc") { processManager.reinitProcessType("DMLProc"); + processManager.setQuerySystemState(true); } //only run on auto process restart @@ -3211,9 +3217,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) } } - //enable query stats - dbrm.setSystemQueryReady(true); - + //set query system states ready processManager.setQuerySystemState(true); processManager.setSystemState(oam::ACTIVE); @@ -3639,6 +3643,8 @@ int ProcessManager::disableModule(string target, bool manualFlag) if (opState == oam::AUTO_DISABLED && newState == oam::MAN_DISABLED) { + //removemodule to get proess in MAN_OFFLINE + stopModule(target, REMOVE, true); try { @@ -3691,7 +3697,7 @@ int ProcessManager::disableModule(string target, bool manualFlag) setModuleState(target, newState); - //set Columnstore.xml enbale state + //set Columnstore.xml enable state setEnableState( target, SnewState); log.writeLog(__LINE__, "disableModule - setEnableState", LOG_TYPE_DEBUG); @@ -3777,18 +3783,18 @@ void ProcessManager::recycleProcess(string module, bool enableModule) restartProcessType("PrimProc"); sleep(1); - restartProcessType("ExeMgr"); - sleep(1); - restartProcessType("mysqld"); restartProcessType("WriteEngineServer"); sleep(1); - restartProcessType("DDLProc", module); + startProcessType("ExeMgr"); sleep(1); - restartProcessType("DMLProc", module); + startProcessType("DDLProc"); + sleep(1); + + startProcessType("DMLProc"); return; } @@ -4128,6 +4134,7 @@ void ProcessManager::setSystemState(uint16_t state) Oam oam; ALARMManager aManager; Configuration config; + ProcessManager processManager(config, log); log.writeLog(__LINE__, "Set System State = " + oamState[state], LOG_TYPE_DEBUG); @@ -4149,9 +4156,10 @@ void ProcessManager::setSystemState(uint16_t state) // Process Alarms string system = "System"; - - if ( state == oam::ACTIVE ) - { + if( state == oam::ACTIVE ) { + //set query system states ready + processManager.setQuerySystemState(true); + //clear alarms if set aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_AUTO, CLEAR); aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_MANUAL, CLEAR); @@ -4542,7 +4550,8 @@ int ProcessManager::stopProcessType( std::string processName, bool manualFlag ) if ( systemprocessstatus.processstatus[i].ProcessName == processName) { //skip if in a COLD_STANDBY state - if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY ) +// if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY ) + if ( systemprocessstatus.processstatus[i].ProcessOpState != oam::ACTIVE ) continue; // found one, request restart of it @@ -4682,12 +4691,17 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski if ( systemprocessstatus.processstatus[i].ProcessName == processName ) { //skip if in a BUSY_INIT state - if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::BUSY_INIT || - systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_INIT || - systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_INIT || - ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) ) - continue; +// if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::BUSY_INIT || +// systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_OFFLINE || +// systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_OFFLINE || +// systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_INIT || +// systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_INIT || +// ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) ) +// continue; + if ( systemprocessstatus.processstatus[i].ProcessOpState != oam::ACTIVE ) + continue; + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { string procModuleType = systemprocessstatus.processstatus[i].Module.substr(0, MAX_MODULE_TYPE_SIZE); @@ -6825,7 +6839,7 @@ int ProcessManager::sendMsgProcMon( std::string module, ByteStream msg, int requ if ( IPAddr == oam::UnassignedIpAddr ) { - log.writeLog(__LINE__, "sendMsgProcMon ping failure", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "sendMsgProcMon ping failure " + module + " " + IPAddr, LOG_TYPE_ERROR); return oam::API_SUCCESS; } @@ -6836,7 +6850,7 @@ int ProcessManager::sendMsgProcMon( std::string module, ByteStream msg, int requ if ( system(cmd.c_str()) != 0) { //ping failure - log.writeLog(__LINE__, "sendMsgProcMon ping failure", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "sendMsgProcMon ping failure " + module + " " + IPAddr, LOG_TYPE_ERROR); return oam::API_SUCCESS; } } @@ -7649,7 +7663,7 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist) } //set query system state not ready - processManager.setQuerySystemState(true); + processManager.setQuerySystemState(false); // Bug 4554: Wait until DMLProc is finished with rollback if (status == oam::API_SUCCESS) @@ -7726,6 +7740,9 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist) processManager.setSystemState(rtn); } + //set query system state ready + processManager.setQuerySystemState(true); + // exit thread log.writeLog(__LINE__, "startSystemThread Exit", LOG_TYPE_DEBUG); startsystemthreadStatus = status; @@ -8254,19 +8271,18 @@ void ProcessManager::checkSimplexModule(std::string moduleName) if ( state == oam::COLD_STANDBY ) { - //set Primary UM Module - if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc" ) - { + //process DDL/DMLProc + if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc") + { + setPMProcIPs((*pt).DeviceName); + + log.writeLog(__LINE__, "Set Primary UM Module = " + (*pt).DeviceName, LOG_TYPE_DEBUG); + oam.setSystemConfig("PrimaryUMModuleName", (*pt).DeviceName); //distribute config file distributeConfigFile("system"); sleep(2); - - //add MySQL Replication setup, if needed - log.writeLog(__LINE__, "Setup MySQL Replication for COLD_STANDBY DMLProc going ACTIVE", LOG_TYPE_DEBUG); - oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, (*pt).DeviceName); } int status = processManager.startProcess((*pt).DeviceName, @@ -8277,12 +8293,24 @@ void ProcessManager::checkSimplexModule(std::string moduleName) { log.writeLog(__LINE__, "checkSimplexModule: mate process started: " + (*pt).DeviceName + "/" + systemprocessconfig.processconfig[j].ProcessName, LOG_TYPE_DEBUG); - //check to see if DDL/DML IPs need to be updated - if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc" ) - setPMProcIPs((*pt).DeviceName); + status = processManager.startProcess((*pt).DeviceName, + "DMLProc", + FORCEFUL); + if ( status == API_SUCCESS ) { + log.writeLog(__LINE__, "checkSimplexModule: mate process started: " + (*pt).DeviceName + "/DMLProc", LOG_TYPE_DEBUG); + } + else + log.writeLog(__LINE__, "checkSimplexModule: mate process failed to start: " + (*pt).DeviceName + "/DMLProc", LOG_TYPE_DEBUG); } else log.writeLog(__LINE__, "checkSimplexModule: mate process failed to start: " + (*pt).DeviceName + "/" + systemprocessconfig.processconfig[j].ProcessName, LOG_TYPE_DEBUG); + + //setup new MariaDB Replication Master + if ( systemprocessconfig.processconfig[j].ProcessName == "DMLProc" ) { + log.writeLog(__LINE__, "Setup MySQL Replication for COLD_STANDBY DMLProc going ACTIVE", LOG_TYPE_DEBUG); + oam::DeviceNetworkList devicenetworklist; + processManager.setMySQLReplication(devicenetworklist, (*pt).DeviceName); + } } else { @@ -10344,7 +10372,7 @@ int ProcessManager::OAMParentModuleChange() if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) && ( moduleNameList.size() <= 0 && config.moduleType() == "pm") ) { - int status = 0; + status = 0; } else { diff --git a/procmon/main.cpp b/procmon/main.cpp index 81b0d1864..eb2e6dd1f 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -340,8 +340,9 @@ int main(int argc, char** argv) { log.writeLog(__LINE__, "Standby PM not responding, infinidb shutting down", LOG_TYPE_CRITICAL); //Set the alarm - aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); - sleep (1); + // aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); + // sleep (1); + string cmd = startup::StartUp::installDir() + "/bin/infinidb stop > /dev/null 2>&1"; system(cmd.c_str()); } @@ -369,8 +370,7 @@ int main(int argc, char** argv) sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", IPaddr); log.writeLog(__LINE__, "set ProcMgr IPaddr to Old Standby Module: " + IPaddr, LOG_TYPE_DEBUG); - - //update Calpont Config table + //update MariaDB ColumnStore Config table try { sysConfig->write(); @@ -554,8 +554,8 @@ int main(int argc, char** argv) { log.writeLog(__LINE__, "Check DB mounts failed, shutting down", LOG_TYPE_CRITICAL); //Set the alarm - aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); - sleep (1); + // aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); + // sleep (1); string cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /dev/null 2>&1"; system(cmd.c_str()); } @@ -1463,7 +1463,7 @@ static void chldHandleThread(MonitorConfig config) (*listPtr).processID != 0 ) || ( (*listPtr).state == oam::ACTIVE && (*listPtr).processID == 0 ) ) { - log.writeLog(__LINE__, "*****Calpont Process Restarting: " + (*listPtr).ProcessName + ", old PID = " + oam.itoa((*listPtr).processID), LOG_TYPE_CRITICAL); + log.writeLog(__LINE__, "*****MariaDB ColumnStore Process Restarting: " + (*listPtr).ProcessName + ", old PID = " + oam.itoa((*listPtr).processID), LOG_TYPE_CRITICAL); if ( (*listPtr).dieCounter >= processRestartCount || processRestartCount == 0) @@ -1523,7 +1523,7 @@ static void chldHandleThread(MonitorConfig config) {} // check if process failover is needed due to process outage - aMonitor.checkProcessFailover((*listPtr).ProcessName); + aMonitor.checkModuleFailover((*listPtr).ProcessName); //check the db health if (DBFunctionalMonitorFlag == "y" ) @@ -1605,7 +1605,7 @@ static void chldHandleThread(MonitorConfig config) (*listPtr).processID = 0; // check if process failover is needed due to process outage - aMonitor.checkProcessFailover((*listPtr).ProcessName); + aMonitor.checkModuleFailover((*listPtr).ProcessName); break; } else @@ -1681,7 +1681,7 @@ static void chldHandleThread(MonitorConfig config) } //Log this event - log.writeLog(__LINE__, "Calpont Process " + (*listPtr).ProcessName + restartStatus, LOG_TYPE_INFO); + log.writeLog(__LINE__, "MariaDB ColumnStore Process " + (*listPtr).ProcessName + restartStatus, LOG_TYPE_INFO); } } } @@ -2707,6 +2707,9 @@ void processStatusMSG(messageqcpp::IOSocket* cfIos) memcpy(fShmSystemStatus[0].StateChangeDate, oam.getCurrentTime().c_str(), DATESIZE); log.writeLog(__LINE__, "statusControl: REQUEST RECEIVED: Set System State = " + oamState[state], LOG_TYPE_DEBUG); } + + BRM::DBRM dbrm; + dbrm.setSystemQueryReady(true); } } break; diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index ce9aa0449..2eb038189 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1261,7 +1261,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO // error in launching a process if ( requestStatus == oam::API_FAILURE && (*listPtr).RunType == SIMPLEX) - checkProcessFailover((*listPtr).ProcessName); + checkModuleFailover((*listPtr).ProcessName); else break; } @@ -4963,20 +4963,19 @@ std::string ProcessMonitor::sendMsgProcMon1( std::string module, ByteStream msg, } /****************************************************************************************** -* @brief checkProcessFailover +* @brief checkModuleFailover * -* purpose: check if process failover is needed due to a process outage +* purpose: check if module failover is needed due to a process outage * ******************************************************************************************/ -void ProcessMonitor::checkProcessFailover( std::string processName) +void ProcessMonitor::checkModuleFailover( std::string processName) { Oam oam; //force failover on certain processes if ( processName == "DDLProc" || - processName == "DMLProc" ) - { - log.writeLog(__LINE__, "checkProcessFailover: process failover, process outage of " + processName, LOG_TYPE_CRITICAL); + processName == "DMLProc" ) { + log.writeLog(__LINE__, "checkModuleFailover: process failover, process outage of " + processName, LOG_TYPE_CRITICAL); try { @@ -4999,27 +4998,36 @@ void ProcessMonitor::checkProcessFailover( std::string processName) systemprocessstatus.processstatus[i].ProcessOpState == oam::FAILED ) { // found a AVAILABLE mate, start it - log.writeLog(__LINE__, "start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Change UM Master to module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Disable local UM module " + config.moduleName(), LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Stop local UM module " + config.moduleName(), LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Disable Local will Enable UM module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); + + oam::DeviceNetworkConfig devicenetworkconfig; + oam::DeviceNetworkList devicenetworklist; + + devicenetworkconfig.DeviceName = config.moduleName(); + devicenetworklist.push_back(devicenetworkconfig); try { - oam.setSystemConfig("PrimaryUMModuleName", systemprocessstatus.processstatus[i].Module); + oam.stopModule(devicenetworklist, oam::FORCEFUL, oam::ACK_YES); + log.writeLog(__LINE__, "success stopModule on module " + config.moduleName(), LOG_TYPE_DEBUG); - //distribute config file - oam.distributeConfigFile("system"); - sleep(1); - } - catch (...) {} - - try - { - oam.startProcess(systemprocessstatus.processstatus[i].Module, processName, FORCEFUL, ACK_YES); - log.writeLog(__LINE__, "success start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); - } - catch (exception& e) - { - log.writeLog(__LINE__, "failed start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_ERROR); + try + { + oam.disableModule(devicenetworklist); + log.writeLog(__LINE__, "success disableModule on module " + config.moduleName(), LOG_TYPE_DEBUG); + } + catch (exception& e) + { + log.writeLog(__LINE__, "failed disableModule on module " + config.moduleName(), LOG_TYPE_ERROR); + } } + catch (exception& e) + { + log.writeLog(__LINE__, "failed stopModule on module " + config.moduleName(), LOG_TYPE_ERROR); + } break; } @@ -5036,9 +5044,6 @@ void ProcessMonitor::checkProcessFailover( std::string processName) // log.writeLog(__LINE__, "EXCEPTION ERROR on getProcessStatus: Caught unknown exception!", LOG_TYPE_ERROR); } } - - return; - } /****************************************************************************************** @@ -6583,6 +6588,8 @@ int ProcessMonitor::glusterAssign(std::string dbrootID) if ( WEXITSTATUS(ret) != 0 ) { + log.writeLog(__LINE__, "glusterAssign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); + ifstream in("/tmp/glusterAssign.txt"); in.seekg(0, std::ios::end); int size = in.tellg(); @@ -6630,6 +6637,8 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) if ( WEXITSTATUS(ret) != 0 ) { + log.writeLog(__LINE__, "glusterUnassign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); + ifstream in("/tmp/glusterUnassign.txt"); in.seekg(0, std::ios::end); int size = in.tellg(); diff --git a/procmon/processmonitor.h b/procmon/processmonitor.h index 04c72bb29..5ec131d42 100644 --- a/procmon/processmonitor.h +++ b/procmon/processmonitor.h @@ -488,7 +488,7 @@ public: /** *@brief check if module failover is needed due to a process outage */ - void checkProcessFailover( std::string processName); + void checkModuleFailover(std::string processName); /** *@brief run upgrade script diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 9f8f3e9a3..f68d16c55 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -10,7 +10,7 @@ CHECK=true REPORTPASS=true LOGFILE="" -OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16") +OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16" "ubuntu18") NODE_IPADDRESS="" @@ -37,7 +37,7 @@ checkContinue() { } ### -# Print Fucntions +# Print Functions ### helpPrint () { @@ -57,7 +57,7 @@ helpPrint () { echo "" echo "Additional information on Tool is documented at:" echo "" - echo "https://mariadb.com/kb/en/mariadb/*****/" + echo "https://mariadb.com/kb/en/library/mariadb-columnstore-cluster-test-tool/" echo "" echo "Items that are checked:" echo " Node Ping test" @@ -65,6 +65,7 @@ helpPrint () { echo " ColumnStore Port test" echo " OS version" echo " Locale settings" + echo " Umask settings" echo " Firewall settings" echo " Date/time settings" echo " Dependent packages installed" @@ -326,16 +327,18 @@ checkSSH() rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then if [ $PASSWORD == "ssh" ] ; then - echo $ipadd " Node Passed SSH login test using ssh-keys" + echo $ipadd " Node Passed SSH login test using ssh-keys" else - echo $ipadd " Node Passed SSH login test using user password" + echo $ipadd " Node Passed SSH login test using user password" fi else if [ $PASSWORD == "ssh" ] ; then - echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys" + echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys" else - echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password" + echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password" fi + + echo "Error - Fix the SSH login issue and rerun test" exit 1 fi done @@ -489,12 +492,47 @@ checkLocale() fi } -checkSELINUX() +checkLocalUMASK() +{ + # UMASK check + # + echo "" + echo "** Run Local UMASK check" + echo "" + + pass=true + filename=UMASKtest + + rm -f $filename + touch $filename + permission=$(stat -c "%A" "$filename") + result=${permission:4:1} + if [ ${result} == "r" ] ; then + result=${permission:7:1} + if [ ${result} == "r" ] ; then + echo "UMASK local setting test passed" + else + echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022" + pass=false + fi + else + echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022" + pass=false + fi + + if ! $pass; then + checkContinue + fi + + rm -f $filename +} + +checkLocalSELINUX() { # SELINUX check # echo "" - echo "** Run SELINUX check" + echo "** Run Local SELINUX check" echo "" pass=true @@ -511,21 +549,86 @@ checkSELINUX() echo "Local Node SELINUX setting is Not Enabled" fi - for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1` - if [ "$?" -ne 0 ]; then - echo "$ipadd Node SELINUX setting is Not Enabled" - else - `cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` - if [ "$?" -eq 0 ]; then - echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results" - pass=false - else - echo "$ipadd Node SELINUX setting is Not Enabled" - fi - `rm -f config` - fi - done + if ! $pass; then + checkContinue + fi +} + +checkUMASK() +{ + # UMASK check + # + echo "" + echo "** Run UMASK check" + echo "" + + pass=true + + for ipadd in "${NODE_IPADDRESS[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > /tmp/remote_command_check 2>&1` + if [ "$?" -eq 0 ]; then + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> /tmp/remote_scp_get 2>&1` + if [ "$?" -eq 0 ]; then + permission=`cat test.log` + result=${permission:4:1} + if [ ${result} == "r" ] ; then + result=${permission:7:1} + if [ ${result} == "r" ] ; then + echo "$ipadd Node UMASK setting test passed" + else + echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022" + pass=false + fi + else + echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022" + pass=false + fi + else + echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check /tmp/remote_scp_get" + pass=false + fi + else + echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check /tmp/remote_command_check" + pass=false + fi + `rm -f test.log` + done + + if ! $pass; then + checkContinue + fi + + rm -f $filename +} + +checkSELINUX() +{ + # SELINUX check + # + echo "" + echo "** Run SELINUX check" + echo "" + + pass=true + for ipadd in "${NODE_IPADDRESS[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "$ipadd Node SELINUX setting is Not Enabled" + else + `cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results" + pass=false + else + echo "$ipadd Node SELINUX setting is Not Enabled" + fi + `rm -f config` + fi + done + + if ! $pass; then + checkContinue + fi } checkFirewalls() @@ -949,7 +1052,7 @@ checkPackages() declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" ) declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") - if [ "$OS" == "ubuntu16" ] ; then + if [ "$OS" == "ubuntu16" ] || [ "$OS" == "ubuntu18" ]; then if [ ! `which dpkg 2>/dev/null` ] ; then echo "${bold}Failed${normal}, Local Node ${bold}rpm${normal} package not installed" pass=false @@ -1307,12 +1410,15 @@ echo "" checkLocalOS checkLocalDir +checkLocalUMASK +checkLocalSELINUX if [ "$IPADDRESSES" != "" ]; then checkPing checkSSH checkRemoteDir checkOS checkLocale + checkUMASK checkSELINUX checkFirewalls checkPorts diff --git a/utils/clusterTester/os_detect.sh b/utils/clusterTester/os_detect.sh index 7930c0daf..be69e870e 100755 --- a/utils/clusterTester/os_detect.sh +++ b/utils/clusterTester/os_detect.sh @@ -29,7 +29,7 @@ detectOS () { echo Operating System name: $osPrettyName echo Operating System tag: $osTag case "$osTag" in - centos6|centos7|ubuntu16|debian8|suse12|debian9) + centos6|centos7|ubuntu16|debian8|suse12|debian9|ubuntu18) ;; *) echo OS not supported diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 4c416360c..03ca241f8 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -315,20 +315,13 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType, bool simpleCase) { - FunctionParm::size_type n = fp.size(); + uint64_t simple = simpleCase ? 1 : 0; + bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist - if (simpleCase) // simple case has an expression - n -= 1; // remove expression from count of expression_i + result_i - bool hasElse = ((n % 2) != 0); // if 1, then ELSE exist - - if (hasElse) - --n; // n now is an even number uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); - uint64_t whereCount = hasElse ? (fp.size() - 2 + simpleCase) / 2 : (fp.size() - 1) / 2 + simpleCase; - - idbassert((n % 2) == 0); + uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple; bool allStringO = true; bool allStringR = true; @@ -341,33 +334,24 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, for (uint64_t i = 0; i <= parmCount; i++) { - // operation or result type - operation = ((i > 0) && (i <= whereCount)); - - // the result type of ELSE, if exists. - if (i == n) - { - if (!hasElse) - break; - - if (simpleCase) + // for SimpleCase, we return the type of the case expression, + // which will always be in position 0. + if (i == 0 && simpleCase) + { + if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) { - // the case expression - if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) - { - PredicateOperator op; - op.setOpType(oct, fp[i]->data()->resultType()); - allStringO = false; - oct = op.operationType(); - } - - i += 1; + PredicateOperator op; + op.setOpType(oct, fp[i]->data()->resultType()); + allStringO = false; + oct = op.operationType(); } + i += 1; + } - operation = false; - } + // operation or result type + operation = ((i > 0+simple) && (i <= whereCount)); if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && @@ -378,10 +362,13 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, if (operation) { - op.setOpType(oct, fp[i]->data()->resultType()); - allStringO = false; - oct = op.operationType(); - } + if (!simpleCase) + { + op.setOpType(oct, fp[i]->data()->resultType()); + allStringO = false; + oct = op.operationType(); + } + } // If any parm is of string type, the result type should be string. (same as if) else if (rct.colDataType != CalpontSystemCatalog::CHAR && @@ -457,6 +444,13 @@ bool Func_simple_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); + if (lop && rop) + { + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); + } + return parm[i]->data()->getBoolVal(row, isNull); } diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 613640a11..edc94b067 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -149,8 +149,7 @@ TupleJoiner::TupleJoiner( for (uint32_t i = 0; i < smallKeyColumns.size(); i++) { discreteValues[i] = false; - - if (isUnsigned(smallRG.getColType(i))) + if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { cpValues[i].push_back(static_cast(numeric_limits::max())); cpValues[i].push_back(0); @@ -1033,8 +1032,7 @@ boost::shared_ptr TupleJoiner::copyForDiskJoin() for (uint32_t i = 0; i < smallKeyColumns.size(); i++) { ret->discreteValues[i] = false; - - if (isUnsigned(smallRG.getColType(i))) + if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { ret->cpValues[i].push_back(static_cast(numeric_limits::max())); ret->cpValues[i].push_back(0); diff --git a/utils/messageqcpp/messagequeue.cpp b/utils/messageqcpp/messagequeue.cpp index 2bff9481d..142d4c19c 100644 --- a/utils/messageqcpp/messagequeue.cpp +++ b/utils/messageqcpp/messagequeue.cpp @@ -157,24 +157,42 @@ void MessageQueueClient::setup(bool syncProto) { string otherEndIPStr; string otherEndPortStr; - uint16_t port; + struct addrinfo hints, *servinfo; + int rc = 0; otherEndIPStr = fConfig->getConfig(fOtherEnd, "IPAddr"); otherEndPortStr = fConfig->getConfig(fOtherEnd, "Port"); if (otherEndIPStr.length() == 0) otherEndIPStr = "127.0.0.1"; - if (otherEndPortStr.length() == 0 || (port = static_cast(strtol(otherEndPortStr.c_str(), 0, 0))) == 0) + if (otherEndPortStr.length() == 0 || static_cast(strtol(otherEndPortStr.c_str(), 0, 0)) == 0) { - string msg = "MessageQueueClient::MessageQueueClient: config error: Invalid/Missing Port attribute"; + string msg = "MessageQueueClient::setup(): config error: Invalid/Missing Port attribute"; throw runtime_error(msg); } - memset(&fServ_addr, 0, sizeof(fServ_addr)); - sockaddr_in* sinp = reinterpret_cast(&fServ_addr); - sinp->sin_family = AF_INET; - sinp->sin_port = htons(port); - sinp->sin_addr.s_addr = inet_addr(otherEndIPStr.c_str()); + memset(&hints, 0, sizeof hints); + // ATM We support IPv4 only. + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + + if( !(rc = getaddrinfo(otherEndIPStr.c_str(), otherEndPortStr.c_str(), &hints, &servinfo)) ) + { + memset(&fServ_addr, 0, sizeof(fServ_addr)); + sockaddr_in* sinp = reinterpret_cast(&fServ_addr); + *sinp = *reinterpret_cast(servinfo->ai_addr); + freeaddrinfo(servinfo); + } + else + { + string msg = "MessageQueueClient::setup(): "; + msg.append(gai_strerror(rc)); + logging::Message::Args args; + logging::LoggingID li(31); + args.add(msg); + fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li); + } #ifdef SKIP_IDB_COMPRESSION fClientSock.setSocketImpl(new InetStreamSocket()); @@ -200,15 +218,34 @@ MessageQueueClient::MessageQueueClient(const string& otherEnd, Config* config, b setup(syncProto); } -MessageQueueClient::MessageQueueClient(const string& ip, uint16_t port, bool syncProto) : +MessageQueueClient::MessageQueueClient(const string& dnOrIp, uint16_t port, bool syncProto) : fLogger(31), fIsAvailable(true) { - memset(&fServ_addr, 0, sizeof(fServ_addr)); - sockaddr_in* sinp = reinterpret_cast(&fServ_addr); - sinp->sin_family = AF_INET; - sinp->sin_port = htons(port); - sinp->sin_addr.s_addr = inet_addr(ip.c_str()); + struct addrinfo hints, *servinfo; + int rc = 0; + memset(&hints, 0, sizeof hints); + // ATM We support IPv4 only. + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + if( !(rc = getaddrinfo(dnOrIp.c_str(), NULL, &hints, &servinfo)) ) + { + memset(&fServ_addr, 0, sizeof(fServ_addr)); + sockaddr_in* sinp = reinterpret_cast(&fServ_addr); + *sinp = *reinterpret_cast(servinfo->ai_addr); + sinp->sin_port = htons(port); + freeaddrinfo(servinfo); + } + else + { + string msg = "MessageQueueClient::MessageQueueClient(): "; + msg.append(gai_strerror(rc)); + logging::Message::Args args; + logging::LoggingID li(31); + args.add(msg); + fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li); + } #ifdef SKIP_IDB_COMPRESSION fClientSock.setSocketImpl(new InetStreamSocket()); #else diff --git a/utils/messageqcpp/messagequeue.h b/utils/messageqcpp/messagequeue.h index 3c61d074c..ca4d1f389 100644 --- a/utils/messageqcpp/messagequeue.h +++ b/utils/messageqcpp/messagequeue.h @@ -33,6 +33,7 @@ #include #else #include +#include #endif #include "serversocket.h" @@ -183,7 +184,7 @@ public: * * construct a queue from this process to otherEnd on the given IP and Port. */ - EXPORT explicit MessageQueueClient(const std::string& ip, uint16_t port, bool syncProto = true); + EXPORT explicit MessageQueueClient(const std::string& dnOrIp, uint16_t port, bool syncProto=true); /** diff --git a/utils/messageqcpp/messagequeuepool.cpp b/utils/messageqcpp/messagequeuepool.cpp index 1777b38c5..f986f734f 100644 --- a/utils/messageqcpp/messagequeuepool.cpp +++ b/utils/messageqcpp/messagequeuepool.cpp @@ -37,12 +37,12 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts) return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000; } -MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& ip, uint64_t port) +MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &dnOrIp, uint64_t port) { boost::mutex::scoped_lock lock(queueMutex); std::ostringstream oss; - oss << ip << "_" << port; + oss << dnOrIp << "_" << port; std::string searchString = oss.str(); MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(searchString); @@ -59,7 +59,7 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& ip, u clock_gettime(CLOCK_MONOTONIC, &now); uint64_t nowSeconds = TimeSpecToSeconds(&now); - newClientObject->client = new MessageQueueClient(ip, port); + newClientObject->client = new MessageQueueClient(dnOrIp, port); newClientObject->inUse = true; newClientObject->lastUsed = nowSeconds; clientMap.insert(std::pair(searchString, newClientObject)); diff --git a/utils/messageqcpp/messagequeuepool.h b/utils/messageqcpp/messagequeuepool.h index 472794a5a..db49d8e5c 100644 --- a/utils/messageqcpp/messagequeuepool.h +++ b/utils/messageqcpp/messagequeuepool.h @@ -42,7 +42,7 @@ class MessageQueueClientPool { public: static MessageQueueClient* getInstance(const std::string& module); - static MessageQueueClient* getInstance(const std::string& ip, uint64_t port); + static MessageQueueClient *getInstance(const std::string &dnOrIp, uint64_t port); static void releaseInstance(MessageQueueClient* client); static void deleteInstance(MessageQueueClient* client); static MessageQueueClient* findInPool(const std::string& search); diff --git a/writeengine/server/we_server.cpp b/writeengine/server/we_server.cpp index 31e0d8792..826b2dc73 100644 --- a/writeengine/server/we_server.cpp +++ b/writeengine/server/we_server.cpp @@ -110,6 +110,21 @@ int main(int argc, char** argv) printf ("Locale is : %s\n", systemLang.c_str() ); + int gDebug = 0; + int c; + while ((c = getopt(argc, argv, "d")) != EOF) + { + switch (c) + { + case 'd': + gDebug++; + break; + case '?': + default: + break; + } + } + //set BUSY_INIT state { // Is there a reason to have a seperate Oam instance for this? @@ -210,7 +225,9 @@ int main(int argc, char** argv) } } - int err = setupResources(); + int err = 0; + if (!gDebug) + err = setupResources(); string errMsg; switch (err)