From 5fe9a9129088793ef8e4c433d21c77b4e2423c07 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 25 Jun 2018 11:17:50 +0100 Subject: [PATCH 01/41] MCOL-1496 Fix array bug in joiner Found using -D_GLIBCXX_ASSERTIONS --- utils/joiner/tuplejoiner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 671e63466..b8dc2937f 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -131,7 +131,7 @@ TupleJoiner::TupleJoiner( cpValues.reset(new vector[smallKeyColumns.size()]); for (uint32_t i = 0; i < smallKeyColumns.size(); i++) { discreteValues[i] = false; - if (isUnsigned(smallRG.getColType(i))) + if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { cpValues[i].push_back(static_cast(numeric_limits::max())); cpValues[i].push_back(0); @@ -859,7 +859,7 @@ boost::shared_ptr TupleJoiner::copyForDiskJoin() ret->cpValues.reset(new vector[smallKeyColumns.size()]); for (uint32_t i = 0; i < smallKeyColumns.size(); i++) { ret->discreteValues[i] = false; - if (isUnsigned(smallRG.getColType(i))) + if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { ret->cpValues[i].push_back(static_cast(numeric_limits::max())); ret->cpValues[i].push_back(0); From 9e3f92b6a3ed1cf213f2d0766968601ddc8df975 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 25 Jun 2018 15:10:28 +0100 Subject: [PATCH 02/41] MCOL-1330 Make debug flag let valgrind work Add a '-d' flag to WriteEngine, ExeMgr and PrimProc to let valgrind work with them. --- exemgr/main.cpp | 4 +++- primitives/primproc/primproc.cpp | 20 +++++++++++++++++++- writeengine/server/we_server.cpp | 19 ++++++++++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 8106afb73..92f949f57 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1354,7 +1354,9 @@ int main(int argc, char* argv[]) setenv("CALPONT_CSC_IDENT", "um", 1); #endif setupSignalHandlers(); - int err = setupResources(); + int err = 0; + if (!gDebug) + err = setupResources(); string errMsg; switch (err) { diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index b86e2f74d..ed7dcb67d 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -299,6 +299,22 @@ int main(int argc, char* argv[]) // This is unset due to the way we start it program_invocation_short_name = const_cast("PrimProc"); + int gDebug = 0; + int c; + + while ((c = getopt(argc, argv, "d")) != EOF) + { + switch(c) + { + case 'd': + gDebug++; + break; + case '?': + default: + break; + } + } + Config* cf = Config::makeConfig(); setupSignalHandlers(); @@ -307,7 +323,9 @@ int main(int argc, char* argv[]) mlp = new primitiveprocessor::Logger(); - int err = setupResources(); + int err = 0; + if (!gDebug) + err = setupResources(); string errMsg; switch (err) { diff --git a/writeengine/server/we_server.cpp b/writeengine/server/we_server.cpp index 9bdb6faa6..b68a81a0b 100644 --- a/writeengine/server/we_server.cpp +++ b/writeengine/server/we_server.cpp @@ -103,6 +103,21 @@ int main(int argc, char** argv) printf ("Locale is : %s\n", systemLang.c_str() ); + int gDebug = 0; + int c; + while ((c = getopt(argc, argv, "d")) != EOF) + { + switch (c) + { + case 'd': + gDebug++; + break; + case '?': + default: + break; + } + } + //set BUSY_INIT state { // Is there a reason to have a seperate Oam instance for this? @@ -196,7 +211,9 @@ int main(int argc, char** argv) } } - int err = setupResources(); + int err = 0; + if (!gDebug) + err = setupResources(); string errMsg; switch (err) { From af87f47a8f705befd5c219a5ea31042a4b7a318b Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Mon, 25 Jun 2018 10:25:33 -0700 Subject: [PATCH 03/41] Fix MCOL-1155 to correctly process the NOT subtree. --- .gitignore | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 51 ++++++++++++++++++++++++++--- procmgr/processmanager.cpp | 2 +- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index f8246dc9f..917ffb360 100644 --- a/.gitignore +++ b/.gitignore @@ -106,4 +106,4 @@ install_manifest_platform.txt install_manifest_storage-engine.txt _CPack_Packages columnstoreversion.h - +.idea/ diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index a30d71688..c47bd8d52 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -1358,7 +1358,49 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) idbassert(ifp->argument_count() == 1); ParseTree *ptp = 0; - if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM) + if (((Item_func*)(ifp->arguments()[0]))->functype() == Item_func::EQUAL_FUNC) + { + // negate it in place + // Note that an EQUAL_FUNC ( a <=> b) was converted to + // ( a = b OR ( a is null AND b is null) ) + // NOT of the above expression is: ( a != b AND (a is not null OR b is not null ) + + if (!gwip->ptWorkStack.empty()) + ptp = gwip->ptWorkStack.top(); + + if (ptp) + { + ParseTree* or_ptp = ptp; + ParseTree* and_ptp = or_ptp->right(); + ParseTree* equal_ptp = or_ptp->left(); + ParseTree* nullck_left_ptp = and_ptp->left(); + ParseTree* nullck_right_ptp = and_ptp->right(); + SimpleFilter *sf = dynamic_cast(nullck_left_ptp->data()); + try + { + sf->op()->reverseOp(); + sf = dynamic_cast(nullck_right_ptp->data()); + sf->op()->reverseOp(); + sf = dynamic_cast(equal_ptp->data()); + sf->op()->reverseOp(); + // Rehook the trees + ptp = and_ptp; + ptp->left(equal_ptp); + ptp->right(or_ptp); + or_ptp->left(nullck_left_ptp); + or_ptp->right(nullck_right_ptp); + gwip->ptWorkStack.pop(); + gwip->ptWorkStack.push(ptp); + } + catch (std::exception& ex ) + { + gwip->fatalParseError = true; + gwip->parseErrorText = ex.what(); + return false; + } + } + } + else if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM) { // negate it in place if (!gwip->ptWorkStack.empty()) @@ -1425,7 +1467,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) } else if (ifp->functype() == Item_func::EQUAL_FUNC) { - // a = b OR (a IS NULL AND b IS NULL) + // Convert "a <=> b" to (a = b OR (a IS NULL AND b IS NULL))" idbassert (gwip->rcWorkStack.size() >= 2); ReturnedColumn* rhs = gwip->rcWorkStack.top(); gwip->rcWorkStack.pop(); @@ -1437,7 +1479,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) // b IS NULL ConstantColumn *nlhs1 = new ConstantColumn("", ConstantColumn::NULLDATA); sop.reset(new PredicateOperator("isnull")); - sop->setOpType(lhs->resultType(), rhs->resultType()); + sop->setOpType(lhs->resultType(), rhs->resultType()); sfn1 = new SimpleFilter(sop, rhs, nlhs1); ParseTree* ptpl = new ParseTree(sfn1); // a IS NULL @@ -1452,7 +1494,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) ptpn->right(ptpr); // a = b sop.reset(new PredicateOperator("=")); - sop->setOpType(lhs->resultType(), lhs->resultType()); + sop->setOpType(lhs->resultType(), rhs->resultType()); sfo = new SimpleFilter(sop, lhs->clone(), rhs->clone()); // OR with the NULL comparison tree ParseTree* ptp = new ParseTree(new LogicOperator("or")); @@ -7017,3 +7059,4 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) return 0; } } +// vim:ts=4 sw=4: diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 8b01179d2..a6ca886af 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -9469,7 +9469,7 @@ int ProcessManager::OAMParentModuleChange() if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) && ( moduleNameList.size() <= 0 && config.moduleType() == "pm") ) { - int status = 0; + status = 0; } else { From 5ea9cad2d49041cc2f62f59e9e850e8a68e1c15f Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Tue, 26 Jun 2018 13:13:59 -0700 Subject: [PATCH 04/41] MCOL-1155 Remove try-catch block by an explicit check for a null pointer. --- dbcon/mysql/ha_calpont_execplan.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index c47bd8d52..6113e7ca4 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -1375,15 +1375,16 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) ParseTree* equal_ptp = or_ptp->left(); ParseTree* nullck_left_ptp = and_ptp->left(); ParseTree* nullck_right_ptp = and_ptp->right(); - SimpleFilter *sf = dynamic_cast(nullck_left_ptp->data()); - try - { - sf->op()->reverseOp(); - sf = dynamic_cast(nullck_right_ptp->data()); - sf->op()->reverseOp(); - sf = dynamic_cast(equal_ptp->data()); - sf->op()->reverseOp(); - // Rehook the trees + SimpleFilter *sf_left_nullck = dynamic_cast(nullck_left_ptp->data()); + SimpleFilter *sf_right_nullck = dynamic_cast(nullck_right_ptp->data()); + SimpleFilter *sf_equal = dynamic_cast(equal_ptp->data()); + + if (sf_left_nullck && sf_right_nullck && sf_equal) { + // Negate the null checks + sf_left_nullck->op()->reverseOp(); + sf_right_nullck->op()->reverseOp(); + sf_equal->op()->reverseOp(); + // Rehook the nodes ptp = and_ptp; ptp->left(equal_ptp); ptp->right(or_ptp); @@ -1392,10 +1393,9 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) gwip->ptWorkStack.pop(); gwip->ptWorkStack.push(ptp); } - catch (std::exception& ex ) - { + else { gwip->fatalParseError = true; - gwip->parseErrorText = ex.what(); + gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ASSERTION_FAILURE); return false; } } From 476a5e551fe56384e58bf3c1982f5f002ea591c0 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 26 Jun 2018 19:11:57 -0500 Subject: [PATCH 05/41] remove rc.local chmod to 666 --- VERSION | 4 ++-- oam/install_scripts/module_installer.sh | 4 +--- oamapps/postConfigure/installer.cpp | 6 ++++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 8a397bd3a..52b5353ee 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=1 -COLUMNSTORE_VERSION_PATCH=6 -COLUMNSTORE_VERSION_RELEASE=1 +COLUMNSTORE_VERSION_PATCH=5 +COLUMNSTORE_VERSION_RELEASE=2 diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 6ca39b70f..8910cca7f 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -186,7 +186,6 @@ if [ $module = "um" ]; then echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE echo "done" >> $RCFILE else - sudo chmod 666 $RCFILE sudo echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE sudo echo "done" >> $RCFILE @@ -201,7 +200,6 @@ else echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE echo "done" >> $RCFILE else - sudo chmod 666 $RCFILE sudo echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE sudo echo "done" >> $RCFILE @@ -214,7 +212,7 @@ fi if [ $user != "root" ]; then echo "uncomment runuser in rc.local" - sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 + sudo sed -i -e 's/#sudo runuser/sudo runuser/g' $RCFILE >/dev/null 2>&1 fi echo "!!!Module Installation Successfully Completed!!!" diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index e85fb0989..1cc84ae0f 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -165,6 +165,12 @@ int main(int argc, char *argv[]) if (p && *p) USER = p; + // setup to start on reboot, for non-root amazon installs + if ( !rootUser ) + { + system("sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); + } + //copy Columnstore.xml.rpmsave if upgrade option is selected if ( installType == "upgrade" ) { cmd = "/bin/cp -f " + installDir + "/etc/Columnstore.xml " + installDir + "/etc/Columnstore.xml.new 2>&1"; From ffb76bb814eeb7bcd911a123a7e388a0e397b3d7 Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 27 Jun 2018 14:44:36 -0500 Subject: [PATCH 06/41] MCOL-1467 - chnages to get back to 1.1.6 --- VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 52b5353ee..8a397bd3a 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=1 -COLUMNSTORE_VERSION_PATCH=5 -COLUMNSTORE_VERSION_RELEASE=2 +COLUMNSTORE_VERSION_PATCH=6 +COLUMNSTORE_VERSION_RELEASE=1 From f786cf9c9e420a8f7b9f44ef7e480287b421e8db Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 28 Jun 2018 10:42:57 -0500 Subject: [PATCH 07/41] MCOL-1145 --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index 90421faa8..7e48157db 100644 --- a/README +++ b/README @@ -10,3 +10,4 @@ series are included in this release. Additional features will be pushed in future releases. A few things to notice: - The building of the ColumnStore engine needs a special build environment. We're working on making it available for everyone to build. + From 58dcdb341a2a3579b18e77d8a26e03e02a999858 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 5 Jul 2018 11:03:07 -0500 Subject: [PATCH 08/41] MCOL-1525 - fix nonroot corefile generating issue --- oam/install_scripts/columnstore | 6 +----- oam/install_scripts/post-install | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index 038ad0c43..15c00cea5 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -94,13 +94,9 @@ start() { CoreFileFlag=`$InstallDir/bin/getConfig -c $InstallDir/etc/Columnstore.xml Installation CoreFileFlag` if [ $CoreFileFlag = "y" ]; then - SUDO= - if [ "$user" != "root" ]; then - SUDO="$SUDO" - fi #columnstore core files - $SUDO ulimit -c unlimited > /dev/null 2>&1 + ulimit -c unlimited > /dev/null 2>&1 $SUDO sysctl -q -w kernel.core_uses_pid=1 > /dev/null 2>&1 $SUDO sysctl -q -w kernel.core_pattern=/var/log/mariadb/columnstore/corefiles/core.%e.%p > /dev/null 2>&1 fi diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index f7aeeb2ca..4eaa28dda 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -164,7 +164,7 @@ mkdir -p /tmp/columnstore_tmp_files >/dev/null 2>&1 #setup core file directory and link mkdir /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1 -chmod 755 /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1 +chmod 777 /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1 #create mount directories mkdir /mnt/tmp > /dev/null 2>&1 From d73d1bb8c1d8d731dd8111082a47b3d3a6cf75b5 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 5 Jul 2018 13:57:27 -0500 Subject: [PATCH 09/41] fix issue with systemname save --- oamapps/postConfigure/postConfigure.cpp | 442 +++++++++++++----------- 1 file changed, 245 insertions(+), 197 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 4225195de..cd17b880c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + MA 02110-1301, USA. */ /****************************************************************************************** * $Id: postConfigure.cpp 64 2006-10-12 22:21:51Z dhill $ @@ -112,7 +112,7 @@ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall); bool createDbrootDirs(string DBRootStorageType); bool pkgCheck(std::string columnstorePackage); bool storageSetup(bool amazonInstall); -void setSystemName(); +void setSystemName(bool single_server_quick_install = false); bool singleServerDBrootSetup(); bool copyFstab(string moduleName); bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath); @@ -219,6 +219,8 @@ int main(int argc, char *argv[]) noPrompting = false; string password; string cmd; + bool single_server_quick_install = false; + // struct sysinfo myinfo; // hidden options @@ -280,33 +282,36 @@ int main(int argc, char *argv[]) cout << "will perform a Package Installation of all of the Modules within the" << endl; cout << "System that is being configured." << endl; cout << endl; - cout << "IMPORTANT: This tool should only be run on a Performance Module Server," << endl; - cout << " preferably Module #1" << endl; + cout << "IMPORTANT: This tool is required to run on a Performance Module #1 (pm1) Server." << endl; cout << endl; cout << "Instructions:" << endl << endl; cout << " Press 'enter' to accept a value in (), if available or" << endl; cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-s][-port][-i][-n]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-sq][-port][-i][-n][-sn]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; cout << " If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl; cout << " -p Unix Password, used with no-prompting option" << endl; - cout << " -s Single Threaded Remote Install" << endl; + cout << " -sq Single Server Quick Install" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; cout << " -i Non-root Install directory, Only use for non-root installs" << endl; cout << " -n Non-distributed install, meaning it will not install the remote nodes" << endl; + cout << " -sn System Name" << endl; exit (0); } - else if( string("-s") == argv[i] ) - thread_remote_installer = false; - else if( string("-f") == argv[i] ) - nodeps = "--nodeps"; - else if( string("-o") == argv[i] ) - startOfflinePrompt = true; - else if( string("-c") == argv[i] ) { + else if( string("-sq") == argv[i] ) + { + single_server_quick_install = true; + noPrompting = true; + } + else if( string("-f") == argv[i] ) + nodeps = "--nodeps"; + else if( string("-o") == argv[i] ) + startOfflinePrompt = true; + else if( string("-c") == argv[i] ) { i++; if (i >= argc ) { cout << " ERROR: Config File not provided" << endl; @@ -355,6 +360,14 @@ int main(int argc, char *argv[]) exit (1); } installDir = argv[i]; + } + else if( string("-sn") == argv[i] ) { + i++; + if (i >= argc ) { + cout << " ERROR: System-name not provided" << endl; + exit (1); + } + systemName = argv[i]; } else { @@ -379,41 +392,43 @@ int main(int argc, char *argv[]) cout << "Installation of all of the Servers within the System that is being configured." << endl; cout << endl; - cout << "IMPORTANT: This tool should only be run on the Parent OAM Module" << endl; - cout << " which is a Performance Module, preferred Module #1" << endl; + cout << "IMPORTANT: This tool requires to run on the Performance Module #1" << endl; cout << endl; - if (!noPrompting) { - cout << "Prompting instructions:" << endl << endl; - cout << " Press 'enter' to accept a value in (), if available or" << endl; - cout << " Enter one of the options within [], if available, or" << endl; - cout << " Enter a new value" << endl << endl; - } - else + if (!single_server_quick_install) { - //get current time and date - time_t now; - now = time(NULL); - struct tm tm; - localtime_r(&now, &tm); - char timestamp[200]; - strftime (timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm); - string currentDate = timestamp; + if (!noPrompting) { + cout << "Prompting instructions:" << endl << endl; + cout << " Press 'enter' to accept a value in (), if available or" << endl; + cout << " Enter one of the options within [], if available, or" << endl; + cout << " Enter a new value" << endl << endl; + } + else + { + //get current time and date + time_t now; + now = time(NULL); + struct tm tm; + localtime_r(&now, &tm); + char timestamp[200]; + strftime (timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm); + string currentDate = timestamp; - string postConfigureLog = "/var/log/columnstore-postconfigure-" + currentDate; + string postConfigureLog = "/var/log/columnstore-postconfigure-" + currentDate; - cout << "With the no-Prompting Option being specified, you will be required to have the following:" << endl; - cout << endl; - cout << " 1. Root user ssh keys setup between all nodes in the system or" << endl; - cout << " use the password command line option." << endl; - cout << " 2. A Configure File to use to retrieve configure data, default to Columnstore.xml.rpmsave" << endl; - cout << " or use the '-c' option to point to a configuration file." << endl; - cout << endl; -// cout << " Output if being redirected to " << postConfigureLog << endl; + cout << "With the no-Prompting Option being specified, you will be required to have the following:" << endl; + cout << endl; + cout << " 1. Root user ssh keys setup between all nodes in the system or" << endl; + cout << " use the password command line option." << endl; + cout << " 2. A Configure File to use to retrieve configure data, default to Columnstore.xml.rpmsave" << endl; + cout << " or use the '-c' option to point to a configuration file." << endl; + cout << endl; + // cout << " Output if being redirected to " << postConfigureLog << endl; -// redirectStandardOutputToFile(postConfigureLog, false ); + // redirectStandardOutputToFile(postConfigureLog, false ); + } } - + //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { cout << "MariaDB ColumnStore is running, can't run postConfigure while MariaDB ColumnStore is running. Exiting.." << endl; @@ -421,45 +436,51 @@ int main(int argc, char *argv[]) } //check Config saved files - if ( !checkSaveConfigFile()) + if (!single_server_quick_install) { - cout << "ERROR: Configuration File not setup" << endl; - exit(1); + if ( !checkSaveConfigFile()) + { + cout << "ERROR: Configuration File not setup" << endl; + exit(1); + } } - + //determine package type string EEPackageType; - if (!rootUser) - EEPackageType = "binary"; - else - { - int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "rpm"; - else { - rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "deb"; - else - EEPackageType = "binary"; - } - } + if (single_server_quick_install) + { + if (!rootUser) + EEPackageType = "binary"; + else + { + int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "rpm"; + else { + rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "deb"; + else + EEPackageType = "binary"; + } + } - try { - sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); - } - catch(...) - { - cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + try { + sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); + } + catch(...) + { + cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } - + //check for local ip address as pm1 ModuleConfig moduleconfig; @@ -551,7 +572,6 @@ int main(int argc, char *argv[]) catch(...) {} - // run my.cnf upgrade script if ( reuseConfig == "y" ) { @@ -615,127 +635,148 @@ int main(int argc, char *argv[]) } cout << endl; + + if (single_server_quick_install) + { + cout << "===== Quick Single-Server Install Configuration =====" << endl << endl; - cout << "===== Setup System Server Type Configuration =====" << endl << endl; - - cout << "There are 2 options when configuring the System Server Type: single and multi" << endl << endl; - cout << " 'single' - Single-Server install is used when there will only be 1 server configured" << endl; - cout << " on the system. It can also be used for production systems, if the plan is" << endl; - cout << " to stay single-server." << endl << endl; - cout << " 'multi' - Multi-Server install is used when you want to configure multiple servers now or" << endl; - cout << " in the future. With Multi-Server install, you can still configure just 1 server" << endl; - cout << " now and add on addition servers/modules in the future." << endl << endl; - - string temp; - try { - temp = sysConfig->getConfig(InstallSection, "SingleServerInstall"); - } - catch(...) - {} - - if ( temp == "y" ) + cout << "Single-Server install is used when there will only be 1 server configured" << endl; + cout << "on the system. It can also be used for production systems, if the plan is" << endl; + cout << "to stay single-server." << endl; + singleServerInstall = "1"; + } else - singleServerInstall = "2"; + { + cout << "===== Setup System Server Type Configuration =====" << endl << endl; + + cout << "There are 2 options when configuring the System Server Type: single and multi" << endl << endl; + cout << " 'single' - Single-Server install is used when there will only be 1 server configured" << endl; + cout << " on the system. It can also be used for production systems, if the plan is" << endl; + cout << " to stay single-server." << endl << endl; + cout << " 'multi' - Multi-Server install is used when you want to configure multiple servers now or" << endl; + cout << " in the future. With Multi-Server install, you can still configure just 1 server" << endl; + cout << " now and add on addition servers/modules in the future." << endl << endl; - while(true) { - prompt = "Select the type of System Server install [1=single, 2=multi] (" + singleServerInstall + ") > "; - pcommand = callReadline(prompt.c_str()); string temp; - if (pcommand) { - if (strlen(pcommand) > 0) - temp = pcommand; - else - temp = singleServerInstall; - callFree(pcommand); - if (temp == "1") { - singleServerInstall = temp; - cout << endl << "Performing the Single Server Install." << endl; + try { + temp = sysConfig->getConfig(InstallSection, "SingleServerInstall"); + } + catch(...) + {} - if ( reuseConfig == "n" ) { - //setup to use the single server Columnstore.xml file + if ( temp == "y" ) + singleServerInstall = "1"; + else + singleServerInstall = "2"; - // we know that our Config instance just timestamped itself in the getConfig - // call above. if postConfigure is running non-interactively we may get here - // within the same second which means the changes that are about to happen - // when Columnstore.xml gets overwritten will be ignored because of the Config - // instance won't know to reload - sleep(2); + while(true) { + string temp = singleServerInstall; + prompt = "Select the type of System Server install [1=single, 2=multi] (" + singleServerInstall + ") > "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) + temp = pcommand; + else + temp = singleServerInstall; + + callFree(pcommand); - cmd = "rm -f " + installDir + "/etc/Columnstore.xml.installSave > /dev/null 2>&1"; - system(cmd.c_str()); - cmd = "mv -f " + installDir + "/etc/Columnstore.xml " + installDir + "/etc/Columnstore.xml.installSave > /dev/null 2>&1"; - system(cmd.c_str()); - cmd = "/bin/cp -f " + installDir + "/etc/Columnstore.xml.singleserver " + installDir + "/etc/Columnstore.xml > /dev/null 2>&1"; - system(cmd.c_str()); - } - - setSystemName(); - cout << endl; - - system(cmd.c_str()); - - // setup storage - if (!storageSetup(false)) - { - cout << "ERROR: Problem setting up storage" << endl; - exit(1); - } - - if (hdfs || !rootUser) - if( !updateBash() ) - cout << "updateBash error" << endl; - - // setup storage - if (!singleServerDBrootSetup()) - { - cout << "ERROR: Problem setting up DBRoot IDs" << endl; - exit(1); - } - - //set system DBRoot count and check 'files per parition' with number of dbroots - try { - sysConfig->setConfig(SystemSection, "DBRootCount", oam.itoa(DBRootCount)); - } - catch(...) - { - cout << "ERROR: Problem setting DBRoot Count in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - //check if dbrm data resides in older directory path and inform user if it does - dbrmDirCheck(); - - if (startOfflinePrompt) - offLineAppCheck(); - - checkMysqlPort(mysqlPort, sysConfig); - - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; - - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; - system(cmd.c_str()); - exit(0); - } - else - { - if (temp == "2") { + if (temp == "1") { singleServerInstall = temp; + + cout << endl << "Performing the Single Server Install." << endl; + break; } - } - cout << "Invalid Entry, please re-enter" << endl; - if ( noPrompting ) - exit(1); + else + { + if (temp == "2") { + singleServerInstall = temp; + break; + } + } - continue; + cout << "Invalid Entry, please re-enter (1 or 2)" << endl; + if ( noPrompting ) + exit(1); + } } - break; + } + + if (singleServerInstall == "1") + { + if ( reuseConfig == "n" ) { + //setup to use the single server Columnstore.xml file + + // we know that our Config instance just timestamped itself in the getConfig + // call above. if postConfigure is running non-interactively we may get here + // within the same second which means the changes that are about to happen + // when Columnstore.xml gets overwritten will be ignored because of the Config + // instance won't know to reload + sleep(1); + + cmd = "rm -f " + installDir + "/etc/Columnstore.xml.installSave > /dev/null 2>&1"; + system(cmd.c_str()); + cmd = "mv -f " + installDir + "/etc/Columnstore.xml " + installDir + "/etc/Columnstore.xml.installSave > /dev/null 2>&1"; + system(cmd.c_str()); + cmd = "/bin/cp -f " + installDir + "/etc/Columnstore.xml.singleserver " + installDir + "/etc/Columnstore.xml > /dev/null 2>&1"; + system(cmd.c_str()); + } + + setSystemName(single_server_quick_install); + + if (!single_server_quick_install) + { + cout << endl; + + // setup storage + if (!storageSetup(false)) + { + cout << "ERROR: Problem setting up storage" << endl; + exit(1); + } + + // setup storage + if (!singleServerDBrootSetup()) + { + cout << "ERROR: Problem setting up DBRoot IDs" << endl; + exit(1); + } + + //set system DBRoot count and check 'files per parition' with number of dbroots + try { + sysConfig->setConfig(SystemSection, "DBRootCount", oam.itoa(DBRootCount)); + } + catch(...) + { + cout << "ERROR: Problem setting DBRoot Count in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + //check if dbrm data resides in older directory path and inform user if it does + dbrmDirCheck(); + + if (startOfflinePrompt) + offLineAppCheck(); + } + + checkMysqlPort(mysqlPort, sysConfig); + + if (hdfs || !rootUser) + if( !updateBash() ) + cout << "updateBash error" << endl; + + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; + + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; + system(cmd.c_str()); + exit(0); } try { @@ -4796,29 +4837,36 @@ bool storageSetup(bool amazonInstall) } -void setSystemName() +void setSystemName(bool single_server_quick_install) { Oam oam; //setup System Name - try { - systemName = sysConfig->getConfig(SystemSection, "SystemName"); - } - catch(...) - { - systemName = oam::UnassignedName; - } - + if ( systemName.empty() ) - systemName = oam::UnassignedName; - - prompt = "Enter System Name (" + systemName + ") > "; - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) systemName = pcommand; - callFree(pcommand); + try { + systemName = sysConfig->getConfig(SystemSection, "SystemName"); + } + catch(...) + { + systemName = oam::UnassignedName; + } } + + if ( systemName.empty() ) + systemName = "columnstore-1"; + if (!single_server_quick_install) + { + prompt = "Enter System Name (" + systemName + ") > "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) + { + if (strlen(pcommand) > 0) systemName = pcommand; + callFree(pcommand); + } + } + try { sysConfig->setConfig(SystemSection, "SystemName", systemName); oam.changeMyCnf( "server_audit_syslog_info", systemName ); From 37a159064b517266901c9c80f2010d89cf0d5909 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 5 Jul 2018 15:21:33 -0500 Subject: [PATCH 10/41] added quick install script --- oamapps/postConfigure/postConfigure.cpp | 8 +++---- .../quick_installer_single_server.sh | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 oamapps/postConfigure/quick_installer_single_server.sh diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index cd17b880c..e7670ce99 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -289,20 +289,20 @@ int main(int argc, char *argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-sq][-port][-i][-n][-sn]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-port][-i][-n][-sn]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; cout << " If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl; cout << " -p Unix Password, used with no-prompting option" << endl; - cout << " -sq Single Server Quick Install" << endl; + cout << " -qs Single Server Quick Install" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; cout << " -i Non-root Install directory, Only use for non-root installs" << endl; cout << " -n Non-distributed install, meaning it will not install the remote nodes" << endl; cout << " -sn System Name" << endl; exit (0); } - else if( string("-sq") == argv[i] ) + else if( string("-qs") == argv[i] ) { single_server_quick_install = true; noPrompting = true; @@ -372,7 +372,7 @@ int main(int argc, char *argv[]) else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-s][-port][-i]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-port][-i][-n][-sn]" << endl; exit (1); } } diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh new file mode 100644 index 000000000..dfbc2d06c --- /dev/null +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# $Id: quick_installer_single_server.sh 3705 2018-07-07 19:47:20Z dhill $ +# +# Poddst- Quick Installer for Single Server MariaDB Columnstore + + +if [ $HOME == "root" ]; then + echo "Run post-install script" + echo "" + /usr/local/mariadb/columnstore/bin/post-install + echo "Run postConfigure script" + echo "" + /usr/local/mariadb/columnstore/bin/postConfigure -qs +else + echo "Run post-install script" + echo "" + $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + echo "Run postConfigure script" + echo "" + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -sq +fi From 2c533c000219308bd98fcd6471a58670eaf9558f Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 5 Jul 2018 16:40:36 -0500 Subject: [PATCH 11/41] add quick install script --- oamapps/postConfigure/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/oamapps/postConfigure/CMakeLists.txt b/oamapps/postConfigure/CMakeLists.txt index ebe7d7f3f..60f761e25 100644 --- a/oamapps/postConfigure/CMakeLists.txt +++ b/oamapps/postConfigure/CMakeLists.txt @@ -56,3 +56,8 @@ target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_ install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT platform) + +########### next target ############### + +install(PROGRAMS quick_installer_single_server.sh + DESTINATION ${ENGINE_BINDIR} COMPONENT platform) From f0ed78e92b6e9ac0ffa0a9fcccb68bd7a26a2d09 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 6 Jul 2018 08:43:58 -0500 Subject: [PATCH 12/41] fix root install path --- oamapps/postConfigure/quick_installer_single_server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh index dfbc2d06c..3cf1dac50 100644 --- a/oamapps/postConfigure/quick_installer_single_server.sh +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -5,7 +5,7 @@ # Poddst- Quick Installer for Single Server MariaDB Columnstore -if [ $HOME == "root" ]; then +if [ $HOME == "/root" ]; then echo "Run post-install script" echo "" /usr/local/mariadb/columnstore/bin/post-install From 4f659108f096e2b9077f61e0ac04fd3c3e73dd68 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 6 Jul 2018 16:39:58 -0500 Subject: [PATCH 13/41] MCOL-1146 - multi-node quick install --- oam/etc/Columnstore.xml | 2 +- oamapps/postConfigure/CMakeLists.txt | 2 +- oamapps/postConfigure/postConfigure.cpp | 258 ++++++++++++++---- .../quick_installer_multi_server.sh | 63 +++++ .../quick_installer_single_server.sh | 13 +- 5 files changed, 285 insertions(+), 53 deletions(-) create mode 100644 oamapps/postConfigure/quick_installer_multi_server.sh diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index ec985028f..d083cf60c 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -438,7 +438,7 @@ n n n - 2 + 1 n n internal diff --git a/oamapps/postConfigure/CMakeLists.txt b/oamapps/postConfigure/CMakeLists.txt index 60f761e25..cfe3d700e 100644 --- a/oamapps/postConfigure/CMakeLists.txt +++ b/oamapps/postConfigure/CMakeLists.txt @@ -59,5 +59,5 @@ install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### -install(PROGRAMS quick_installer_single_server.sh +install(PROGRAMS quick_installer_single_server.sh quick_installer_multi_server.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index e7670ce99..08fa549a8 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -99,6 +99,14 @@ typedef struct Performance_Module_struct typedef std::vector PerformanceModuleList; +typedef struct ModuleIP_struct +{ + std::string IPaddress; + std::string moduleName; +} ModuleIP; + +typedef std::vector ModuleIpList; + void offLineAppCheck(); bool setOSFiles(string parentOAMModuleName, int serverTypeInstall); bool checkSaveConfigFile(); @@ -112,7 +120,7 @@ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall); bool createDbrootDirs(string DBRootStorageType); bool pkgCheck(std::string columnstorePackage); bool storageSetup(bool amazonInstall); -void setSystemName(bool single_server_quick_install = false); +void setSystemName(); bool singleServerDBrootSetup(); bool copyFstab(string moduleName); bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath); @@ -121,12 +129,6 @@ void remoteInstallThread(void *); bool glusterSetup(string password); -typedef struct ModuleIP_struct -{ - std::string IPaddress; - std::string moduleName; -} ModuleIP; - std::string launchInstance(ModuleIP moduleip); string columnstorePackage; @@ -150,7 +152,6 @@ string PMVolumeType = "standard"; string PMVolumeIOPS = oam::UnassignedName; string UMVolumeIOPS = oam::UnassignedName; - int DBRootCount; string deviceName; @@ -185,6 +186,8 @@ string MySQLRep = "y"; string PMwithUM = "n"; bool amazonInstall = false; bool nonDistribute = false; +bool single_server_quick_install = false; +bool multi_server_quick_install = false; string DataFileEnvFile; @@ -219,8 +222,9 @@ int main(int argc, char *argv[]) noPrompting = false; string password; string cmd; - bool single_server_quick_install = false; - + string pmIpAddrs = ""; + string umIpAddrs = ""; + // struct sysinfo myinfo; // hidden options @@ -272,10 +276,12 @@ int main(int argc, char *argv[]) if (p && *p) HOME = p; } + for( int i = 1; i < argc; i++ ) { - if( string("-h") == argv[i] ) { + if( string("-h") == argv[i] ) + { cout << endl; cout << "This is the MariaDB ColumnStore System Configuration and Installation tool." << endl; cout << "It will Configure the MariaDB ColumnStore System based on Operator inputs and" << endl; @@ -289,48 +295,62 @@ int main(int argc, char *argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-port][-i][-n][-sn]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-port][-i][-n][-sn][-pm-ip-addrs][-um-ip-addrs]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; cout << " If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl; cout << " -p Unix Password, used with no-prompting option" << endl; - cout << " -qs Single Server Quick Install" << endl; + cout << " -qs Quick Install - Single Server" << endl; + cout << " -qm Quick Install - Multi Server" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; cout << " -i Non-root Install directory, Only use for non-root installs" << endl; cout << " -n Non-distributed install, meaning it will not install the remote nodes" << endl; cout << " -sn System Name" << endl; + cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; + cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; exit (0); } - else if( string("-qs") == argv[i] ) - { - single_server_quick_install = true; - noPrompting = true; - } - else if( string("-f") == argv[i] ) - nodeps = "--nodeps"; - else if( string("-o") == argv[i] ) - startOfflinePrompt = true; - else if( string("-c") == argv[i] ) { + else if( string("-qs") == argv[i] ) + { + single_server_quick_install = true; + noPrompting = true; + } + else if( string("-qm") == argv[i] ) + { + multi_server_quick_install = true; + noPrompting = true; + } + else if( string("-f") == argv[i] ) + nodeps = "--nodeps"; + else if( string("-o") == argv[i] ) + startOfflinePrompt = true; + else if( string("-c") == argv[i] ) + { i++; - if (i >= argc ) { + if (i >= argc ) + { cout << " ERROR: Config File not provided" << endl; exit (1); } oldFileName = argv[i]; - if ( oldFileName.find("Columnstore.xml") == string::npos ) { + if ( oldFileName.find("Columnstore.xml") == string::npos ) + { cout << " ERROR: Config File is not a Columnstore.xml file name" << endl; exit (1); } } - else if( string("-p") == argv[i] ) { + else if( string("-p") == argv[i] ) + { i++; - if (i >= argc ) { + if (i >= argc ) + { cout << " ERROR: Password not provided" << endl; exit (1); } password = argv[i]; - if ( password.find("-") != string::npos ) { + if ( password.find("-") != string::npos ) + { cout << " ERROR: Valid Password not provided" << endl; exit (1); } @@ -340,9 +360,11 @@ int main(int argc, char *argv[]) // for backward compatibility else if( string("-n") == argv[i] ) nonDistribute = true; - else if( string("-port") == argv[i] ) { + else if( string("-port") == argv[i] ) + { i++; - if (i >= argc ) { + if (i >= argc ) + { cout << " ERROR: MariaDB ColumnStore Port ID not supplied" << endl; exit (1); } @@ -353,29 +375,64 @@ int main(int argc, char *argv[]) exit (1); } } - else if( string("-i") == argv[i] ) { + else if( string("-i") == argv[i] ) + { i++; - if (i >= argc ) { + if (i >= argc ) + { cout << " ERROR: Path not provided" << endl; exit (1); } installDir = argv[i]; } - else if( string("-sn") == argv[i] ) { + else if( string("-sn") == argv[i] ) + { i++; - if (i >= argc ) { + if (i >= argc ) + { cout << " ERROR: System-name not provided" << endl; exit (1); } systemName = argv[i]; + } + else if( string("-pm-ip-addrs") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: PM-IP-ADRESSES not provided" << endl; + exit (1); + } + pmIpAddrs = argv[i]; + } + else if( string("-um-ip-addrs") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: UM-IP-ADRESSES not provided" << endl; + exit (1); + } + umIpAddrs = argv[i]; } else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-port][-i][-n][-sn]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-port][-i][-n][-sn][-pm-ip-addrs][-um-ip-addrs]" << endl; exit (1); } } + + //check if quick install multi-server has been given ip address + if (multi_server_quick_install) + { + if ( umIpAddrs.empty() && pmIpAddrs.empty() || + !umIpAddrs.empty() && pmIpAddrs.empty() ) + { + cout << " ERROR: Multi-Server option entered, but invalid UM/PM IP addresses were provided, exiting" << endl; + exit(1); + } + } if (installDir[0] != '/') { @@ -395,7 +452,7 @@ int main(int argc, char *argv[]) cout << "IMPORTANT: This tool requires to run on the Performance Module #1" << endl; cout << endl; - if (!single_server_quick_install) + if (!single_server_quick_install || !multi_server_quick_install) { if (!noPrompting) { cout << "Prompting instructions:" << endl << endl; @@ -436,7 +493,7 @@ int main(int argc, char *argv[]) } //check Config saved files - if (!single_server_quick_install) + if (!single_server_quick_install || !multi_server_quick_install) { if ( !checkSaveConfigFile()) { @@ -448,7 +505,7 @@ int main(int argc, char *argv[]) //determine package type string EEPackageType; - if (single_server_quick_install) + if (single_server_quick_install || multi_server_quick_install) { if (!rootUser) EEPackageType = "binary"; @@ -638,7 +695,7 @@ int main(int argc, char *argv[]) if (single_server_quick_install) { - cout << "===== Quick Single-Server Install Configuration =====" << endl << endl; + cout << "===== Quick Install Single-Server Configuration =====" << endl << endl; cout << "Single-Server install is used when there will only be 1 server configured" << endl; cout << "on the system. It can also be used for production systems, if the plan is" << endl; @@ -646,6 +703,12 @@ int main(int argc, char *argv[]) singleServerInstall = "1"; } + else if (multi_server_quick_install) + { + cout << "===== Quick Install Multi-Server Configuration =====" << endl << endl; + + singleServerInstall = "2"; + } else { cout << "===== Setup System Server Type Configuration =====" << endl << endl; @@ -724,7 +787,7 @@ int main(int argc, char *argv[]) system(cmd.c_str()); } - setSystemName(single_server_quick_install); + setSystemName(); if (!single_server_quick_install) { @@ -796,6 +859,76 @@ int main(int argc, char *argv[]) // // Multi-server install // + + ModuleIP InputModuleIP; + ModuleIpList InputModuleIPList; + + int MaxNicID = oam::MAX_NIC; + + if (multi_server_quick_install) + { + //set configuarion settings for default setup + try { + sysConfig->setConfig(InstallSection, "MySQLRep", "y"); + } + catch(...) + {} + + if (umIpAddrs == "" ) + { + // set Server Type Installation to combined + try { + sysConfig->setConfig(InstallSection, "ServerTypeInstall", "2"); + } + catch(...) + {} + } + else + { + int id = 1; + boost::char_separator sep(","); + boost::tokenizer< boost::char_separator > tokens(umIpAddrs, sep); + for ( boost::tokenizer< boost::char_separator >::iterator it = tokens.begin(); + it != tokens.end(); + ++it, ++id) + { + string module = "um" + oam.itoa(id); + + InputModuleIP.IPaddress = *it; + InputModuleIP.moduleName = module; + InputModuleIPList.push_back(InputModuleIP); + } + + umNumber = id; + } + + if (pmIpAddrs != "" ) + { + int id = 1; + boost::char_separator sep(","); + boost::tokenizer< boost::char_separator > tokens(pmIpAddrs, sep); + for ( boost::tokenizer< boost::char_separator >::iterator it = tokens.begin(); + it != tokens.end(); + ++it, ++id) + { + string module = "pm" + oam.itoa(id); + + InputModuleIP.IPaddress = *it; + InputModuleIP.moduleName = module; + InputModuleIPList.push_back(InputModuleIP); + } + + pmNumber = id; + } + + if ( !writeConfig(sysConfig) ) + { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + MaxNicID = 1; + } cout << endl; //cleanup/create local/etc directory @@ -851,7 +984,7 @@ int main(int argc, char *argv[]) } switch ( IserverTypeInstall ) { - case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server + case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - um/pm on a single server { cout << "Combined Server Installation will be performed." << endl; cout << "The Server will be configured as a Performance Module." << endl; @@ -986,8 +1119,8 @@ int main(int argc, char *argv[]) } if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; + mysqlRep = true; + MySQLRep = "y"; } else { @@ -1463,6 +1596,14 @@ int main(int argc, char *argv[]) catch(...) {} } + + if ( moduleType == "um") + if ( umNumber != 0 ) + moduleCount = umNumber; + + if ( moduleType == "pm") + if ( pmNumber != 0 ) + moduleCount = pmNumber; //verify and setup of modules count switch ( IserverTypeInstall ) { @@ -1545,7 +1686,7 @@ int main(int argc, char *argv[]) //clear any Equipped Module IP addresses that aren't in current ID range for ( int j = 0 ; j < listSize ; j++ ) { - for ( unsigned int k = 1 ; k < MAX_NIC+1 ; k++) + for ( unsigned int k = 1 ; k < MaxNicID+1 ; k++) { string ModuleIPAddr = "ModuleIPAddr" + oam.itoa(j+1) + "-" + oam.itoa(k) + "-" + oam.itoa(i+1); if ( !(sysConfig->getConfig(ModuleSection, ModuleIPAddr).empty()) ) { @@ -1594,7 +1735,8 @@ int main(int argc, char *argv[]) moduleDisableState = oam::ENABLEDSTATE; //setup HostName/IPAddress for each NIC - for( unsigned int nicID=1 ; nicID < MAX_NIC+1 ; nicID++ ) + + for( unsigned int nicID=1 ; nicID < MaxNicID +1 ; nicID++ ) { string moduleHostName = oam::UnassignedName; string moduleIPAddr = oam::UnassignedIpAddr; @@ -1615,14 +1757,30 @@ int main(int argc, char *argv[]) for( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) { if ((*pt1).NicID == nicID) { - moduleHostName = (*pt1).HostName; - moduleIPAddr = (*pt1).IPAddr; + bool found = false; + ModuleIpList::iterator pt2 = InputModuleIPList.begin(); + for( ; pt2 != InputModuleIPList.end() ; pt2++) + { + if ( (*pt2).moduleName == newModuleName ) + { + moduleHostName = (*pt2).IPaddress; + moduleIPAddr = (*pt2).IPaddress; + found = true; + } + } + + if ( !found ) + { + moduleHostName = (*pt1).HostName; + moduleIPAddr = (*pt1).IPAddr; + } break; } } } } - + + if ( nicID == 1 ) { if ( moduleDisableState != oam::ENABLEDSTATE ) { string disabled = "y"; @@ -4837,7 +4995,7 @@ bool storageSetup(bool amazonInstall) } -void setSystemName(bool single_server_quick_install) +void setSystemName() { Oam oam; //setup System Name @@ -4856,7 +5014,7 @@ void setSystemName(bool single_server_quick_install) if ( systemName.empty() ) systemName = "columnstore-1"; - if (!single_server_quick_install) + if (!single_server_quick_install || !multi_server_quick_install) { prompt = "Enter System Name (" + systemName + ") > "; pcommand = callReadline(prompt.c_str()); diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh new file mode 100644 index 000000000..25729ec7c --- /dev/null +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# $Id: quick_installer_multi_server.sh 3705 2018-07-07 19:47:20Z dhill $ +# +# Poddst- Quick Installer for Multi Server MariaDB Columnstore + +pmIpAddrs="" +umIpAddrs="" + +for arg in "$@"; do + if [ `expr -- "$arg" : '--pm-ip-addresses='` -eq 18 ]; then + pmIpAddrs="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--pm-ip-addresses='` -eq 18 ]; then + umIpAddrs="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then + echo "Usage ./quick_installer_multi_server.sh [OPTION]" + echo "" + echo "Quick Installer for a Multi Server MariaDB ColumnStore Install" + echo "" + echo "Performace Module (pm) IP addresses required" + echo "User Module (um) IP addresses option" + echo "When only pm IP addresses provided, system is combined setup" + echo "When both pm/um IP addresses provided, system is seperate setup" + echo + echo "--pm-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" + echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" + echo "" + else + echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2 + fi +done + +if [ $pmIpAddrs == "" ] ; then + echo "" + echo "Performace Module (pm) IP addresses required, exiting" + exit 1 +else + if [ $umIpAddrs == "" ] ; then + echo "" + echo "Performing a Multi-Server Combined install with um/pm running on some server" + echo"" + else + echo "" + echo "Performing a Multi-Server Seperate install with um and pm running on seperate servers" + echo"" + fi +fi + +if [ $HOME == "/root" ]; then + echo "Run post-install script" + echo "" + /usr/local/mariadb/columnstore/bin/post-install + echo "Run postConfigure script" + echo "" + /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs=$pmIpAddrs -um-ip-addrs=$umIpAddrs +else + echo "Run post-install script" + echo "" + $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + echo "Run postConfigure script" + echo "" + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs=$pmIpAddrs -um-ip-addrs=$umIpAddrs +fi diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh index 3cf1dac50..bfd9b81c4 100644 --- a/oamapps/postConfigure/quick_installer_single_server.sh +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -4,6 +4,17 @@ # # Poddst- Quick Installer for Single Server MariaDB Columnstore +for arg in "$@"; do + if [ `expr -- "$arg" : '--help'` -eq 6 ]; then + echo "Usage ./quick_installer_multi_server.sh" + echo "" + echo "Quick Installer for a Single Server MariaDB ColumnStore Install" + echo "" + else + echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2 + fi +done + if [ $HOME == "/root" ]; then echo "Run post-install script" @@ -18,5 +29,5 @@ else $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore echo "Run postConfigure script" echo "" - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -sq + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs fi From 20c2cc87addfbc38eafca535705cc8c5361ce73f Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 9 Jul 2018 09:42:54 -0500 Subject: [PATCH 14/41] changes for dbroot --- oamapps/postConfigure/postConfigure.cpp | 57 +++++++++++++++---------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 08fa549a8..20b9922cb 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -493,13 +493,10 @@ int main(int argc, char *argv[]) } //check Config saved files - if (!single_server_quick_install || !multi_server_quick_install) + if ( !checkSaveConfigFile()) { - if ( !checkSaveConfigFile()) - { - cout << "ERROR: Configuration File not setup" << endl; - exit(1); - } + cout << "ERROR: Configuration File not setup" << endl; + exit(1); } //determine package type @@ -2471,28 +2468,35 @@ int main(int argc, char *argv[]) string dbrootList; - for ( unsigned int id = 1 ; id < count+1 ; ) + if (multi_server_quick_install) { - string moduledbrootid = "ModuleDBRootID" + oam.itoa(moduleID) + "-" + oam.itoa(id) + "-" + oam.itoa(i+1); - try { - string dbrootid = sysConfig->getConfig(ModuleSection, moduledbrootid); + dbrootList = oam.itoa(moduleID); + } + else + { + for ( unsigned int id = 1 ; id < count+1 ; ) + { + string moduledbrootid = "ModuleDBRootID" + oam.itoa(moduleID) + "-" + oam.itoa(id) + "-" + oam.itoa(i+1); + try { + string dbrootid = sysConfig->getConfig(ModuleSection, moduledbrootid); - if ( dbrootid != oam::UnassignedName) { - sysConfig->setConfig(ModuleSection, moduledbrootid, oam::UnassignedName); + if ( dbrootid != oam::UnassignedName) { + sysConfig->setConfig(ModuleSection, moduledbrootid, oam::UnassignedName); - dbrootList = dbrootList + dbrootid; - id ++; - if ( id < count+1 ) - dbrootList = dbrootList + ","; + dbrootList = dbrootList + dbrootid; + id ++; + if ( id < count+1 ) + dbrootList = dbrootList + ","; + } + } + catch(...) + { + cout << "ERROR: Problem setting DBRoot ID in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); } } - catch(...) - { - cout << "ERROR: Problem setting DBRoot ID in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } } - + vector dbroots; string tempdbrootList; @@ -3581,6 +3585,15 @@ bool checkSaveConfigFile() } return true; } + else + { + if (single_server_quick_install || multi_server_quick_install) + { + cout << endl << "Quick Install is for fresh installs only, '" + oldFileName + "' exist, exiting" << endl; + exit(1); + } + } + File.close(); // If 'oldFileName' isn't configured, exit From 11d20940897835e49a3ba310a24dc5fc933ceaf3 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 10 Jul 2018 14:03:17 -0500 Subject: [PATCH 15/41] MCOL-1531 Don't compare alias in == operators for ReturnedColumn types --- dbcon/execplan/arithmeticcolumn.cpp | 8 ++++---- dbcon/execplan/constantcolumn.cpp | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index 5c12f7e5f..2c2eb5ace 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -345,10 +345,10 @@ bool ArithmeticColumn::operator==(const ArithmeticColumn& t) const } else if (fExpression != NULL || t.fExpression != NULL) return false; - if (fAlias != t.fAlias) - return false; - if (fTableAlias != t.fTableAlias) - return false; +// if (fAlias != t.fAlias) +// return false; +// if (fTableAlias != t.fTableAlias) +// return false; if (fData != t.fData) return false; return true; diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index fce3825b8..fadd2dfc9 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -316,8 +316,8 @@ bool ConstantColumn::operator==(const ConstantColumn& t) const return false; if (fType != t.fType) return false; - if (fAlias != t.fAlias) - return false; +// if (fAlias != t.fAlias) +// return false; if (fData != t.fData) return false; if (fReturnAll != t.fReturnAll) From 68c01a3d127c54a9b285d6b9ce95ac6ebe145228 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 10 Jul 2018 14:27:28 -0500 Subject: [PATCH 16/41] fixes for multi install --- oamapps/postConfigure/postConfigure.cpp | 176 +++++++++++------- .../quick_installer_multi_server.sh | 41 ++-- 2 files changed, 133 insertions(+), 84 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 20b9922cb..5e3fdd964 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -186,6 +186,7 @@ string MySQLRep = "y"; string PMwithUM = "n"; bool amazonInstall = false; bool nonDistribute = false; +bool nonDistributeFlag = false; bool single_server_quick_install = false; bool multi_server_quick_install = false; @@ -295,7 +296,7 @@ int main(int argc, char *argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-port][-i][-n][-sn][-pm-ip-addrs][-um-ip-addrs]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -305,7 +306,8 @@ int main(int argc, char *argv[]) cout << " -qm Quick Install - Multi Server" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; cout << " -i Non-root Install directory, Only use for non-root installs" << endl; - cout << " -n Non-distributed install, meaning it will not install the remote nodes" << endl; + cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; + cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; cout << " -sn System Name" << endl; cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; @@ -359,7 +361,15 @@ int main(int argc, char *argv[]) noPrompting = true; // for backward compatibility else if( string("-n") == argv[i] ) + { nonDistribute = true; + nonDistributeFlag = true; + } + else if( string("-d") == argv[i] ) + { + nonDistribute = false; + nonDistributeFlag = true; + } else if( string("-port") == argv[i] ) { i++; @@ -502,37 +512,34 @@ int main(int argc, char *argv[]) //determine package type string EEPackageType; - if (single_server_quick_install || multi_server_quick_install) + if (!rootUser) + EEPackageType = "binary"; + else { - if (!rootUser) - EEPackageType = "binary"; - else - { - int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "rpm"; - else { - rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "deb"; - else - EEPackageType = "binary"; - } + int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "rpm"; + else { + rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "deb"; + else + EEPackageType = "binary"; } + } - try { - sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); - } - catch(...) - { - cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + try { + sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); + } + catch(...) + { + cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); } //check for local ip address as pm1 @@ -668,12 +675,22 @@ int main(int argc, char *argv[]) } //check for non-Distributed Install - if ( nonDistribute ) + if ( nonDistributeFlag ) { - try { - oam.setSystemConfig("DistributedInstall", "n"); - } - catch(...) {} + if ( nonDistribute ) + { + try { + oam.setSystemConfig("DistributedInstall", "n"); + } + catch(...) {} + } + else + { + try { + oam.setSystemConfig("DistributedInstall", "y"); + } + catch(...) {} + } } else { @@ -896,7 +913,7 @@ int main(int argc, char *argv[]) InputModuleIPList.push_back(InputModuleIP); } - umNumber = id; + umNumber = id-1; } if (pmIpAddrs != "" ) @@ -915,7 +932,7 @@ int main(int argc, char *argv[]) InputModuleIPList.push_back(InputModuleIP); } - pmNumber = id; + pmNumber = id-1; } if ( !writeConfig(sysConfig) ) @@ -1733,52 +1750,64 @@ int main(int argc, char *argv[]) //setup HostName/IPAddress for each NIC - for( unsigned int nicID=1 ; nicID < MaxNicID +1 ; nicID++ ) + string moduleHostName = oam::UnassignedName; + string moduleIPAddr = oam::UnassignedIpAddr; + + bool found = false; + if (multi_server_quick_install) { - string moduleHostName = oam::UnassignedName; - string moduleIPAddr = oam::UnassignedIpAddr; - - DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); - for( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) + ModuleIpList::iterator pt2 = InputModuleIPList.begin(); + for( ; pt2 != InputModuleIPList.end() ; pt2++) { - if (newModuleName == (*listPT).DeviceName) { - if ( nicID == 1 ) { - moduleDisableState = (*listPT).DisableState; - if ( moduleDisableState.empty() || - moduleDisableState == oam::UnassignedName || - moduleDisableState == oam::AUTODISABLEDSTATE ) - moduleDisableState = oam::ENABLEDSTATE; - } - - HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); - for( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) + if ( (*pt2).moduleName == newModuleName ) + { + moduleHostName = (*pt2).IPaddress; + moduleIPAddr = (*pt2).IPaddress; + found = true; + break; + } + } + } + + unsigned int nicID=1; + for( ; nicID < MaxNicID +1 ; nicID++ ) + { + if ( !found ) + { + moduleHostName = oam::UnassignedName; + moduleIPAddr = oam::UnassignedIpAddr; + + DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); + for( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) + { + if (newModuleName == (*listPT).DeviceName) { - if ((*pt1).NicID == nicID) { - bool found = false; - ModuleIpList::iterator pt2 = InputModuleIPList.begin(); - for( ; pt2 != InputModuleIPList.end() ; pt2++) + if ( nicID == 1 ) + { + moduleDisableState = (*listPT).DisableState; + if ( moduleDisableState.empty() || + moduleDisableState == oam::UnassignedName || + moduleDisableState == oam::AUTODISABLEDSTATE ) + moduleDisableState = oam::ENABLEDSTATE; { - if ( (*pt2).moduleName == newModuleName ) + HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); + for( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) { - moduleHostName = (*pt2).IPaddress; - moduleIPAddr = (*pt2).IPaddress; - found = true; + if ((*pt1).NicID == nicID) + { + moduleHostName = (*pt1).HostName; + moduleIPAddr = (*pt1).IPAddr; + break; + } } } - - if ( !found ) - { - moduleHostName = (*pt1).HostName; - moduleIPAddr = (*pt1).IPAddr; - } - break; } } } } - - - if ( nicID == 1 ) { + + if ( nicID == 1 ) + { if ( moduleDisableState != oam::ENABLEDSTATE ) { string disabled = "y"; while (true) @@ -1860,6 +1889,7 @@ int main(int argc, char *argv[]) break; } + bool moduleHostNameFound = true; if (moduleHostName.empty()) { moduleHostNameFound = true; @@ -3579,6 +3609,10 @@ bool checkSaveConfigFile() //check if Columnstore.xml.rpmsave exist ifstream File (oldFileName.c_str()); if (!File) { + if (single_server_quick_install || multi_server_quick_install) + { + return true; + } if ( noPrompting ) { cout << endl << "Old Config File not found '" + oldFileName + "', exiting" << endl; exit(1); diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh index 25729ec7c..97aa670bc 100644 --- a/oamapps/postConfigure/quick_installer_multi_server.sh +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -6,16 +6,21 @@ pmIpAddrs="" umIpAddrs="" +nonDistrubutedInstall="-n" for arg in "$@"; do if [ `expr -- "$arg" : '--pm-ip-addresses='` -eq 18 ]; then pmIpAddrs="`echo $arg | awk -F= '{print $2}'`" - elif [ `expr -- "$arg" : '--pm-ip-addresses='` -eq 18 ]; then + elif [ `expr -- "$arg" : '--um-ip-addresses='` -eq 18 ]; then umIpAddrs="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then + nonDistrubutedInstall=" " elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then echo "Usage ./quick_installer_multi_server.sh [OPTION]" echo "" echo "Quick Installer for a Multi Server MariaDB ColumnStore Install" + echo "Defaults to non-distrubuted install, meaning MariaDB Columnstore" + echo "needs to be preinstalled on all nodes in the system" echo "" echo "Performace Module (pm) IP addresses required" echo "User Module (um) IP addresses option" @@ -24,40 +29,50 @@ for arg in "$@"; do echo echo "--pm-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" + echo "--dist-install Use Distributed Install Option" echo "" else - echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2 + echo "quick_installer_multi_server.sh: unknown argument: $arg, use --help for help" 1>&2 + exit 1 fi done -if [ $pmIpAddrs == "" ] ; then +if [[ $pmIpAddrs = "" ]]; then echo "" echo "Performace Module (pm) IP addresses required, exiting" exit 1 else - if [ $umIpAddrs == "" ] ; then + if [[ $umIpAddrs = "" ]]; then echo "" - echo "Performing a Multi-Server Combined install with um/pm running on some server" + echo "NOTE: Performing a Multi-Server Combined install with um/pm running on some server" echo"" else echo "" - echo "Performing a Multi-Server Seperate install with um and pm running on seperate servers" + echo "NOTE: Performing a Multi-Server Seperate install with um and pm running on seperate servers" echo"" fi fi -if [ $HOME == "/root" ]; then - echo "Run post-install script" +if [[ $HOME = "/root" ]]; then + echo "${bold}Run post-install script${normal}" echo "" /usr/local/mariadb/columnstore/bin/post-install - echo "Run postConfigure script" + echo "${bold}Run postConfigure script${normal}" echo "" - /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs=$pmIpAddrs -um-ip-addrs=$umIpAddrs + if [[ $umIpAddrs = "" ]]; then + /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall + else + /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall + fi else - echo "Run post-install script" + echo "${bold}Run post-install script${normal}" echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore - echo "Run postConfigure script" + echo "${bold}Run postConfigure script${normal}" echo "" - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs=$pmIpAddrs -um-ip-addrs=$umIpAddrs + if [[ $umIpAddrs = "" ]]; then + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall + else + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall + fi fi From 4f6949835d50a9f0b1e5e421832c61d50e522076 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 11 Jul 2018 15:32:27 +0100 Subject: [PATCH 17/41] MCOL-1037 Fix race condition in FIFO buffer The FIFO buffer could get data in next() whilst a data swap is happening due to a rare race condition. This patch adds mutexes around the parts that could race. The observed effect of this race was during a complex aggregate query the results would occasionally be incorrect. In addition this fixes a race condition in PrimProc's regex processor. --- dbcon/joblist/fifo.h | 10 ++++++++++ primitives/linux-port/column.cpp | 3 +-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/fifo.h b/dbcon/joblist/fifo.h index a6bd286da..6a0fca56d 100644 --- a/dbcon/joblist/fifo.h +++ b/dbcon/joblist/fifo.h @@ -345,17 +345,27 @@ void FIFO::signalPs() template inline bool FIFO::next(uint64_t id, element_t *out) { + base::mutex.lock(); fConsumptionStarted = true; if (cpos[id] >= fMaxElements) + { + base::mutex.unlock(); if (!waitForSwap(id)) return false; + base::mutex.lock(); + } *out = cBuffer[cpos[id]++]; #ifndef ONE_CS if (cpos[id] == fMaxElements) + { + base::mutex.unlock(); signalPs(); + return true; + } #endif + base::mutex.unlock(); return true; } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index cdb70ef67..bed121517 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -65,8 +65,6 @@ inline uint64_t order_swap(uint64_t x) template inline string fixChar(int64_t intval); -idb_regex_t placeholderRegex; - template inline int compareBlock( const void * a, const void * b ) { @@ -954,6 +952,7 @@ inline void p_Col_ridArray(NewColRequestHeader *in, uint16_t *ridArray=0; uint8_t *in8 = reinterpret_cast(in); const uint8_t filterSize = sizeof(uint8_t) + sizeof(uint8_t) + W; + idb_regex_t placeholderRegex; placeholderRegex.used = false; From e1d0916b6c4bd6f5d913f31fd1f6b58d8739c523 Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 11 Jul 2018 15:48:04 -0500 Subject: [PATCH 18/41] add in amazon quikc installer --- oamapps/postConfigure/CMakeLists.txt | 10 +- oamapps/postConfigure/postConfigure.cpp | 149 +++++++++++++++--- .../postConfigure/quick_installer_amazon.sh | 80 ++++++++++ .../quick_installer_multi_server.sh | 24 +-- 4 files changed, 224 insertions(+), 39 deletions(-) create mode 100644 oamapps/postConfigure/quick_installer_amazon.sh diff --git a/oamapps/postConfigure/CMakeLists.txt b/oamapps/postConfigure/CMakeLists.txt index cfe3d700e..4bdbadd1a 100644 --- a/oamapps/postConfigure/CMakeLists.txt +++ b/oamapps/postConfigure/CMakeLists.txt @@ -37,13 +37,13 @@ install(TARGETS getMySQLpw DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### -set(amazonInstaller_SRCS amazonInstaller.cpp helpers.cpp) +#set(amazonInstaller_SRCS amazonInstaller.cpp helpers.cpp) -add_executable(amazonInstaller ${amazonInstaller_SRCS}) +#add_executable(amazonInstaller ${amazonInstaller_SRCS}) -target_link_libraries(amazonInstaller ${ENGINE_LDFLAGS} readline ncurses ${SNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +#target_link_libraries(amazonInstaller ${ENGINE_LDFLAGS} readline ncurses ${SNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS amazonInstaller DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +#install(TARGETS amazonInstaller DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### @@ -59,5 +59,5 @@ install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### -install(PROGRAMS quick_installer_single_server.sh quick_installer_multi_server.sh +install(PROGRAMS quick_installer_single_server.sh quick_installer_multi_server.sh quick_installer_amazon.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 5e3fdd964..98227da9d 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -189,6 +189,7 @@ bool nonDistribute = false; bool nonDistributeFlag = false; bool single_server_quick_install = false; bool multi_server_quick_install = false; +bool amazon_quick_install = false; string DataFileEnvFile; @@ -296,7 +297,7 @@ int main(int argc, char *argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -323,6 +324,11 @@ int main(int argc, char *argv[]) multi_server_quick_install = true; noPrompting = true; } + else if( string("-qa") == argv[i] ) + { + amazon_quick_install = true; + noPrompting = true; + } else if( string("-f") == argv[i] ) nodeps = "--nodeps"; else if( string("-o") == argv[i] ) @@ -424,11 +430,31 @@ int main(int argc, char *argv[]) exit (1); } umIpAddrs = argv[i]; + } + else if( string("-pm-count") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: PM-COUNT not provided" << endl; + exit (1); + } + pmNumber = atoi(argv[i]); + } + else if( string("-um-count") == argv[i] ) + { + i++; + if (i >= argc ) + { + cout << " ERROR: UM-COUNT not provided" << endl; + exit (1); + } + umNumber = atoi(argv[i]); } else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-port][-i][-n][-sn][-pm-ip-addrs][-um-ip-addrs]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; exit (1); } } @@ -436,14 +462,25 @@ int main(int argc, char *argv[]) //check if quick install multi-server has been given ip address if (multi_server_quick_install) { - if ( umIpAddrs.empty() && pmIpAddrs.empty() || - !umIpAddrs.empty() && pmIpAddrs.empty() ) + if ( ( umIpAddrs.empty() && pmIpAddrs.empty() ) || + ( !umIpAddrs.empty() && pmIpAddrs.empty() )) { cout << " ERROR: Multi-Server option entered, but invalid UM/PM IP addresses were provided, exiting" << endl; exit(1); } } + //check if quick install multi-server has been given ip address + if (amazon_quick_install) + { + if ( ( umNumber == 0 && pmNumber == 0 ) || + ( umNumber != 0 && pmNumber == 0 ) ) + { + cout << " ERROR: Amazon option entered, but invalid UM/PM Counts were provided, exiting" << endl; + exit(1); + } + } + if (installDir[0] != '/') { cout << " ERROR: Install dir '" << installDir << "' is not absolute" << endl; @@ -462,7 +499,7 @@ int main(int argc, char *argv[]) cout << "IMPORTANT: This tool requires to run on the Performance Module #1" << endl; cout << endl; - if (!single_server_quick_install || !multi_server_quick_install) + if (!single_server_quick_install || !multi_server_quick_install || !amazon_quick_install) { if (!noPrompting) { cout << "Prompting instructions:" << endl << endl; @@ -723,6 +760,12 @@ int main(int argc, char *argv[]) singleServerInstall = "2"; } + else if (amazon_quick_install) + { + cout << "===== Quick Install Amazon Configuration =====" << endl << endl; + + singleServerInstall = "2"; + } else { cout << "===== Setup System Server Type Configuration =====" << endl << endl; @@ -943,6 +986,42 @@ int main(int argc, char *argv[]) MaxNicID = 1; } + else + { + if (amazon_quick_install) + { + //set configuarion settings for default setup + try { + sysConfig->setConfig(InstallSection, "MySQLRep", "y"); + } + catch(...) + {} + + try { + sysConfig->setConfig(InstallSection, "Cloud", "amazon-vpc"); + } + catch(...) + {} + + if (umNumber == 0 ) + { + // set Server Type Installation to combined + try { + sysConfig->setConfig(InstallSection, "ServerTypeInstall", "2"); + } + catch(...) + {} + } + + if ( !writeConfig(sysConfig) ) + { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + MaxNicID = 1; + } + } cout << endl; //cleanup/create local/etc directory @@ -1174,13 +1253,30 @@ int main(int argc, char *argv[]) in.seekg(0, std::ios::end); int size = in.tellg(); - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) - // not running on amazon with ec2-api-tools + if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) + { + // not running on amazon with ec2-api-tools + if (amazon_quick_install) + { + cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + exit(1); + } + amazonInstall = false; + } else { - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) + if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) + { + // not running on amazon with ec2-api-tools + if (amazon_quick_install) + { + cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + exit(1); + } + amazonInstall = false; + } else amazonInstall = true; } @@ -1216,7 +1312,7 @@ int main(int argc, char *argv[]) amazonInstall = false; try { - sysConfig->setConfig(InstallSection, "Cloud", "disable"); + sysConfig->setConfig(InstallSection, "Cloud", "disable"); } catch(...) {}; @@ -1674,6 +1770,7 @@ int main(int argc, char *argv[]) exit(1); continue; } + //update count try { string ModuleCountParm = "ModuleCount" + oam.itoa(i+1); @@ -1929,23 +2026,25 @@ int main(int argc, char *argv[]) //check if need to create instance or user enter ID string create = "y"; - while(true) + if ( !amazon_quick_install ) { - pcommand = callReadline("Create Instance for " + newModuleName + " [y,n] (y) > "); - if (pcommand) + while(true) { - if (strlen(pcommand) > 0) create = pcommand; - callFree(pcommand); + pcommand = callReadline("Create Instance for " + newModuleName + " [y,n] (y) > "); + if (pcommand) + { + if (strlen(pcommand) > 0) create = pcommand; + callFree(pcommand); + } + if ( create == "y" || create == "n" ) + break; + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + create = "y"; + if ( noPrompting ) + exit(1); } - if ( create == "y" || create == "n" ) - break; - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - create = "y"; - if ( noPrompting ) - exit(1); } - if ( create == "y" ) { ModuleIP moduleip; @@ -2498,7 +2597,7 @@ int main(int argc, char *argv[]) string dbrootList; - if (multi_server_quick_install) + if (multi_server_quick_install || amazon_quick_install) { dbrootList = oam.itoa(moduleID); } @@ -3609,7 +3708,7 @@ bool checkSaveConfigFile() //check if Columnstore.xml.rpmsave exist ifstream File (oldFileName.c_str()); if (!File) { - if (single_server_quick_install || multi_server_quick_install) + if (single_server_quick_install || multi_server_quick_install || amazon_quick_install) { return true; } @@ -3621,7 +3720,7 @@ bool checkSaveConfigFile() } else { - if (single_server_quick_install || multi_server_quick_install) + if (single_server_quick_install || multi_server_quick_install || amazon_quick_install) { cout << endl << "Quick Install is for fresh installs only, '" + oldFileName + "' exist, exiting" << endl; exit(1); diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh new file mode 100644 index 000000000..395a3bf30 --- /dev/null +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# $Id: quick_installer_amazon.sh 3705 2018-07-07 19:47:20Z dhill $ +# +# Poddst- Quick Installer for Amazon MariaDB Columnstore + +pmCount="" +umCount="" +systemName="" + +for arg in "$@"; do + if [ `expr -- "$arg" : '--pm-count='` -eq 11 ]; then + pmCount="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--um-count='` -eq 11 ]; then + umCount="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--systemName='` -eq 13 ]; then + systemName="`echo $arg | awk -F= '{print $2}'`" + systemName="-sn "$systemName + elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then + nonDistrubutedInstall=" " + elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then + echo "Usage ./quick_installer_amazon.sh [OPTION]" + echo "" + echo "Quick Installer for an Amazon MariaDB ColumnStore Install" + echo "This requires to be run on a MariaDB ColumnStore AMI" + echo "" + echo "Performace Module (pm) number is required" + echo "User Module (um) number is option" + echo "When only pm counts provided, system is combined setup" + echo "When both pm/um counts provided, system is seperate setup" + echo + echo "--pm-count=x Number of pm instances to create" + echo "--um-count=x Number of um instances to create, optional" + echo "--system-name=nnnn System Name, optional" + echo "" + else + echo "./quick_installer_amazon.sh: unknown argument: $arg, enter --help for help" 1>&2 + exit 1 + fi +done + +if [[ $pmCount = "" ]]; then + echo "" + echo "Performace Module (pm) count is required, exiting" + exit 1 +else + if [[ $umCount = "" ]]; then + echo "" + echo "NOTE: Performing a Multi-Server Combined install with um/pm running on some server" + echo"" + else + echo "" + echo "NOTE: Performing a Multi-Server Seperate install with um and pm running on seperate servers" + echo"" + fi +fi + +if [[ $HOME = "/root" ]]; then + echo "${bold}Run post-install script${normal}" + echo "" + /usr/local/mariadb/columnstore/bin/post-install + echo "${bold}Run postConfigure script${normal}" + echo "" + if [[ $umCount = "" ]]; then + /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount $systemName + else + /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount -um-count $umCount $systemName + fi +else + echo "${bold}Run post-install script${normal}" + echo "" + $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + echo "${bold}Run postConfigure script${normal}" + echo "" + if [[ $umCount = "" ]]; then + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName + else + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName + fi +fi diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh index 97aa670bc..1f4e29cd0 100644 --- a/oamapps/postConfigure/quick_installer_multi_server.sh +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -7,32 +7,38 @@ pmIpAddrs="" umIpAddrs="" nonDistrubutedInstall="-n" +systemName="" for arg in "$@"; do if [ `expr -- "$arg" : '--pm-ip-addresses='` -eq 18 ]; then pmIpAddrs="`echo $arg | awk -F= '{print $2}'`" elif [ `expr -- "$arg" : '--um-ip-addresses='` -eq 18 ]; then umIpAddrs="`echo $arg | awk -F= '{print $2}'`" + elif [ `expr -- "$arg" : '--systemName='` -eq 13 ]; then + systemName="`echo $arg | awk -F= '{print $2}'`" + systemName="-sn "$systemName elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then nonDistrubutedInstall=" " elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then echo "Usage ./quick_installer_multi_server.sh [OPTION]" echo "" echo "Quick Installer for a Multi Server MariaDB ColumnStore Install" + echo "" echo "Defaults to non-distrubuted install, meaning MariaDB Columnstore" echo "needs to be preinstalled on all nodes in the system" echo "" - echo "Performace Module (pm) IP addresses required" - echo "User Module (um) IP addresses option" + echo "Performace Module (pm) IP addresses are required" + echo "User Module (um) IP addresses are option" echo "When only pm IP addresses provided, system is combined setup" echo "When both pm/um IP addresses provided, system is seperate setup" echo echo "--pm-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" - echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" + echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx, optional" echo "--dist-install Use Distributed Install Option" + echo "--system-name=nnnn System Name, optional" echo "" else - echo "quick_installer_multi_server.sh: unknown argument: $arg, use --help for help" 1>&2 + echo "quick_installer_multi_server.sh: unknown argument: $arg, enter --help for help" 1>&2 exit 1 fi done @@ -58,11 +64,11 @@ if [[ $HOME = "/root" ]]; then echo "" /usr/local/mariadb/columnstore/bin/post-install echo "${bold}Run postConfigure script${normal}" - echo "" + echo "" if [[ $umIpAddrs = "" ]]; then - /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall + /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName else - /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall + /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName fi else echo "${bold}Run post-install script${normal}" @@ -71,8 +77,8 @@ else echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umIpAddrs = "" ]]; then - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName else - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName fi fi From 42448e90224306d464429fd1245f15989aba3df4 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 12 Jul 2018 09:25:57 -0500 Subject: [PATCH 19/41] add system name option --- oamapps/postConfigure/quick_installer_amazon.sh | 2 +- oamapps/postConfigure/quick_installer_multi_server.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh index 395a3bf30..f867aa5a9 100644 --- a/oamapps/postConfigure/quick_installer_amazon.sh +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -13,7 +13,7 @@ for arg in "$@"; do pmCount="`echo $arg | awk -F= '{print $2}'`" elif [ `expr -- "$arg" : '--um-count='` -eq 11 ]; then umCount="`echo $arg | awk -F= '{print $2}'`" - elif [ `expr -- "$arg" : '--systemName='` -eq 13 ]; then + elif [ `expr -- "$arg" : '--system-name='` -eq 14 ]; then systemName="`echo $arg | awk -F= '{print $2}'`" systemName="-sn "$systemName elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh index 1f4e29cd0..239fca1e9 100644 --- a/oamapps/postConfigure/quick_installer_multi_server.sh +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -14,7 +14,7 @@ for arg in "$@"; do pmIpAddrs="`echo $arg | awk -F= '{print $2}'`" elif [ `expr -- "$arg" : '--um-ip-addresses='` -eq 18 ]; then umIpAddrs="`echo $arg | awk -F= '{print $2}'`" - elif [ `expr -- "$arg" : '--systemName='` -eq 13 ]; then + elif [ `expr -- "$arg" : '--system-name='` -eq 14 ]; then systemName="`echo $arg | awk -F= '{print $2}'`" systemName="-sn "$systemName elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then @@ -34,7 +34,7 @@ for arg in "$@"; do echo echo "--pm-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx, optional" - echo "--dist-install Use Distributed Install Option" + echo "--dist-install Use Distributed Install, optional" echo "--system-name=nnnn System Name, optional" echo "" else From 16cf8b79cd03b132971cb71d3a76ddc6cc6e666e Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 12 Jul 2018 09:30:03 -0500 Subject: [PATCH 20/41] add system name option --- oamapps/postConfigure/quick_installer_amazon.sh | 1 + oamapps/postConfigure/quick_installer_multi_server.sh | 1 + oamapps/postConfigure/quick_installer_single_server.sh | 1 + 3 files changed, 3 insertions(+) mode change 100644 => 100755 oamapps/postConfigure/quick_installer_amazon.sh mode change 100644 => 100755 oamapps/postConfigure/quick_installer_multi_server.sh mode change 100644 => 100755 oamapps/postConfigure/quick_installer_single_server.sh diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh old mode 100644 new mode 100755 index f867aa5a9..c71dc8920 --- a/oamapps/postConfigure/quick_installer_amazon.sh +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -33,6 +33,7 @@ for arg in "$@"; do echo "--um-count=x Number of um instances to create, optional" echo "--system-name=nnnn System Name, optional" echo "" + exit 1 else echo "./quick_installer_amazon.sh: unknown argument: $arg, enter --help for help" 1>&2 exit 1 diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh old mode 100644 new mode 100755 index 239fca1e9..25d615d7e --- a/oamapps/postConfigure/quick_installer_multi_server.sh +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -37,6 +37,7 @@ for arg in "$@"; do echo "--dist-install Use Distributed Install, optional" echo "--system-name=nnnn System Name, optional" echo "" + exit 1 else echo "quick_installer_multi_server.sh: unknown argument: $arg, enter --help for help" 1>&2 exit 1 diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh old mode 100644 new mode 100755 index bfd9b81c4..2603bcec8 --- a/oamapps/postConfigure/quick_installer_single_server.sh +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -10,6 +10,7 @@ for arg in "$@"; do echo "" echo "Quick Installer for a Single Server MariaDB ColumnStore Install" echo "" + exit 1 else echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2 fi From 400ae5178730f6cd7aa65719345dca8d372c8bbf Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 12 Jul 2018 14:16:13 -0500 Subject: [PATCH 21/41] removed amazonInstaller and add 3 quick installers --- cpackEngineRPM.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index be5df3462..d2b240029 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -181,7 +181,6 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/bin/resourceReport.sh" "/usr/local/mariadb/columnstore/bin/hadoopReport.sh" "/usr/local/mariadb/columnstore/bin/alarmReport.sh" -"/usr/local/mariadb/columnstore/bin/amazonInstaller" "/usr/local/mariadb/columnstore/bin/remote_command_verify.sh" "/usr/local/mariadb/columnstore/bin/disable-rep-columnstore.sh" "/usr/local/mariadb/columnstore/bin/columnstore.service" @@ -217,6 +216,9 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/bin/os_detect.sh" "/usr/local/mariadb/columnstore/bin/columnstoreClusterTester.sh" "/usr/local/mariadb/columnstore/bin/mariadb-command-line.sh" +"/usr/local/mariadb/columnstore/bin/quick_installer_single_server.sh" +"/usr/local/mariadb/columnstore/bin/quick_installer_multi_server.sh" +"/usr/local/mariadb/columnstore/bin/quick_installer_amazon.sh" ${ignored}) SET(CPACK_RPM_libs_USER_FILELIST From 1995e927888597f26706badc89e4eee2a26de388 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 17 Jul 2018 20:17:51 +0300 Subject: [PATCH 22/41] MCOL-1527 CEJ DML statements now return correct number of affected rows. --- dbcon/mysql/ha_calpont_impl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 6cc7da0fb..c1f37c9ac 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1691,14 +1691,14 @@ uint32_t doUpdateDelete(THD *thd) } else { - thd->set_row_count_func(dmlRowCount); + thd->set_row_count_func(dmlRowCount+thd->get_row_count_func()); } push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, errorMsg.c_str()); } else { // if (dmlRowCount != 0) //Bug 5117. Handling self join. - thd->set_row_count_func(dmlRowCount); + thd->set_row_count_func(dmlRowCount+thd->get_row_count_func()); //cout << " error status " << ci->rc << " and rowcount = " << dmlRowCount << endl; From e05250915b4e578a88828ece32d7996450a03f6f Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 19 Jul 2018 09:29:06 +0100 Subject: [PATCH 23/41] MCOL-1545 Remove getPool call. It is completely unused and it breaks compiling in GCC 8.1 --- utils/common/simpleallocator.h | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/common/simpleallocator.h b/utils/common/simpleallocator.h index 36a8678c6..4d1dce71b 100644 --- a/utils/common/simpleallocator.h +++ b/utils/common/simpleallocator.h @@ -119,7 +119,6 @@ class SimpleAllocator void construct(pointer ptr, const T& val) { new ((void *)ptr) T(val); } void destroy(pointer ptr) { ptr->T::~T(); } - SimplePool* getPool() { return fPool; } void setPool(SimplePool* pool) { fPool = pool; } boost::shared_ptr fPool; From 979d00a679266d95f3ca449d03310a5036493a1b Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 23 Jul 2018 10:40:18 +0100 Subject: [PATCH 24/41] MCOL-1579 Remove chmod of /dev/shm We appear to chmod /dev/shm as a workaround to a CentOS 7 bug that was fixed last year which accidentally set it to 755. If a user has /dev/shm locked down we should get them to fix it rather than modifying it ourselves. The code before this fixed changed a root install to 755 for /dev/shm which instantly broke anything using mmap() with MAP_SHARED as an unprivileged user. --- oam/install_scripts/post-install | 1 - procmon/main.cpp | 7 ------- procmon/processmonitor.cpp | 7 ------- 3 files changed, 15 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 4eaa28dda..5a61a9074 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -247,7 +247,6 @@ else $SUDO chmod 777 /tmp $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1 $SUDO chown $user:$user $installdir/etc/Columnstore.xml - $SUDO chmod -R 777 /dev/shm $SUDO mkdir /var/lock/subsys > /dev/null 2>&1 $SUDO chmod 777 /var/lock/subsys > /dev/null 2>&1 $SUDO rm -f /var/lock/subsys/mysql-Columnstore diff --git a/procmon/main.cpp b/procmon/main.cpp index b4e23a6e1..096cffcee 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -131,13 +131,6 @@ int main(int argc, char **argv) if (p && *p) USER = p; - // change permissions on /dev/shm - if ( !rootUser) - { - string cmd = "sudo chmod 777 /dev/shm >/dev/null 2>&1"; - system(cmd.c_str()); - } - // get and set locale language string systemLang = "C"; diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 91f78e640..b0e0fc07f 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -988,13 +988,6 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO int requestStatus = oam::API_SUCCESS; log.writeLog(__LINE__, "MSG RECEIVED: Start All process request..."); - // change permissions on /dev/shm - string cmd = "chmod 755 /dev/shm >/dev/null 2>&1"; - if ( !rootUser) - cmd = "sudo chmod 777 /dev/shm >/dev/null 2>&1"; - - system(cmd.c_str()); - //start the mysqld daemon try { oam.actionMysqlCalpont(MYSQL_START); From db4279dac7558ed4e99ed3698d10363a798c108d Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 23 Jul 2018 17:32:38 -0500 Subject: [PATCH 25/41] MCOL-1535 caseOperationType adjusted for new simple case order --- utils/funcexp/func_case.cpp | 70 +++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 2b5ae58c2..d3f0aa0cf 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -259,17 +259,10 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType, bool simpleCase) { - FunctionParm::size_type n = fp.size(); - - if (simpleCase) // simple case has an expression - n -= 1; // remove expression from count of expression_i + result_i - bool hasElse = ((n % 2) != 0); // if 1, then ELSE exist - if (hasElse) - --n; // n now is an even number + uint64_t simple = simpleCase ? 1 : 0; + bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); - uint64_t whereCount = hasElse ? (fp.size() - 2 + simpleCase) / 2 : (fp.size() - 1) / 2 + simpleCase; - - idbassert((n % 2) == 0); + uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple; bool allStringO = true; bool allStringR = true; @@ -281,34 +274,25 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, bool operation = true; for (uint64_t i = 0; i <= parmCount; i++) { - // operation or result type - operation = ((i > 0) && (i <= whereCount)); - - // the result type of ELSE, if exists. - if (i == n) + // for SimpleCase, we return the type of the case expression, + // which will always be in position 0. + if (i == 0 && simpleCase) { - if (!hasElse) - break; - - if (simpleCase) - { - // the case expression - if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) - { - PredicateOperator op; - op.setOpType(oct, fp[i]->data()->resultType()); - allStringO = false; - oct = op.operationType(); - } - - i += 1; - } - - operation = false; + if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) + { + PredicateOperator op; + op.setOpType(oct, fp[i]->data()->resultType()); + allStringO = false; + oct = op.operationType(); + } + i += 1; } + // operation or result type + operation = ((i > 0+simple) && (i <= whereCount)); + if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) @@ -317,9 +301,12 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, PredicateOperator op; if (operation) { - op.setOpType(oct, fp[i]->data()->resultType()); - allStringO = false; - oct = op.operationType(); + if (!simpleCase) + { + op.setOpType(oct, fp[i]->data()->resultType()); + allStringO = false; + oct = op.operationType(); + } } // If any parm is of string type, the result type should be string. (same as if) @@ -395,6 +382,13 @@ bool Func_simple_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); + if (lop && rop) + { + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); + } + return parm[i]->data()->getBoolVal(row, isNull); } From ec9069853e3aa3e46ea54984cbb982fbc62d0528 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 24 Jul 2018 09:10:46 -0500 Subject: [PATCH 26/41] MCOL-1531 Remove commented fields from equality functions --- dbcon/execplan/aggregatecolumn.cpp | 2 -- dbcon/execplan/arithmeticcolumn.cpp | 4 ---- dbcon/execplan/constantcolumn.cpp | 2 -- dbcon/execplan/functioncolumn.cpp | 2 -- dbcon/execplan/returnedcolumn.cpp | 9 +-------- dbcon/execplan/simplecolumn.cpp | 6 ------ 6 files changed, 1 insertion(+), 24 deletions(-) diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index b6262f998..da50c983d 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -260,8 +260,6 @@ bool AggregateColumn::operator==(const AggregateColumn& t) const } else if (fFunctionParms.get() != NULL || t.fFunctionParms.get() != NULL) return false; - //if (fAlias != t.fAlias) - // return false; if (fTableAlias != t.fTableAlias) return false; if (fData != t.fData) diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index 2c2eb5ace..4306f3191 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -345,10 +345,6 @@ bool ArithmeticColumn::operator==(const ArithmeticColumn& t) const } else if (fExpression != NULL || t.fExpression != NULL) return false; -// if (fAlias != t.fAlias) -// return false; -// if (fTableAlias != t.fTableAlias) -// return false; if (fData != t.fData) return false; return true; diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index fadd2dfc9..04caafc9c 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -316,8 +316,6 @@ bool ConstantColumn::operator==(const ConstantColumn& t) const return false; if (fType != t.fType) return false; -// if (fAlias != t.fAlias) -// return false; if (fData != t.fData) return false; if (fReturnAll != t.fReturnAll) diff --git a/dbcon/execplan/functioncolumn.cpp b/dbcon/execplan/functioncolumn.cpp index 2316557ac..f7467f40f 100644 --- a/dbcon/execplan/functioncolumn.cpp +++ b/dbcon/execplan/functioncolumn.cpp @@ -309,8 +309,6 @@ bool FunctionColumn::operator==(const FunctionColumn& t) const ++it, ++it2) if (**it != **it2) return false; -// if (fAlias != t.fAlias) -// return false; if (fTableAlias != t.fTableAlias) return false; if (fData != t.fData) diff --git a/dbcon/execplan/returnedcolumn.cpp b/dbcon/execplan/returnedcolumn.cpp index f3b7f9a10..d78817f06 100644 --- a/dbcon/execplan/returnedcolumn.cpp +++ b/dbcon/execplan/returnedcolumn.cpp @@ -173,12 +173,11 @@ void ReturnedColumn::unserialize(messageqcpp::ByteStream& b) bool ReturnedColumn::operator==(const ReturnedColumn& t) const { + // Not all fields are considered for a positive equality. if (fData != t.fData) return false; if (fCardinality != t.fCardinality) return false; - //if (fAlias != t.fAlias) - // return false; if (fDistinct != t.fDistinct) return false; if (fJoinInfo != t.fJoinInfo) @@ -187,20 +186,14 @@ bool ReturnedColumn::operator==(const ReturnedColumn& t) const return false; if (fNullsFirst != t.fNullsFirst) return false; - //if (fOrderPos != t.fOrderPos) - // return false; if (fInputIndex != t.fInputIndex) return false; if (fOutputIndex != t.fOutputIndex) return false; - //if (fSequence != t.fSequence) - // return false; if (fResultType != t.fResultType) return false; if (fOperationType != t.fOperationType) return false; - //if (fExpressionId != t.fExpressionId) - // return false; return true; } diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 895334fc9..34eb4c919 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -333,7 +333,6 @@ void SimpleColumn::serialize(messageqcpp::ByteStream& b) const b << fViewName; b << (uint32_t) fOid; b << fData; - //b << fAlias; b << fTableAlias; b << (uint32_t) fSequence; b << static_cast(fIsInfiniDB); @@ -350,7 +349,6 @@ void SimpleColumn::unserialize(messageqcpp::ByteStream& b) b >> fViewName; b >> (uint32_t&) fOid; b >> fData; - //b >> fAlias; b >> fTableAlias; b >> (uint32_t&) fSequence; b >> reinterpret_cast< ByteStream::doublebyte&>(fIsInfiniDB); @@ -370,16 +368,12 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const return false; if (fColumnName != t.fColumnName) return false; -// if (fIndexName != t.fIndexName) -// return false; if (fViewName != t.fViewName) return false; if (fOid != t.fOid) return false; if (data() != t.data()) return false; -// if (fAlias != t.fAlias) -// return false; if (fTableAlias != t.fTableAlias) return false; if (fAsc != t.fAsc) From 7ec1ccac5e660b1ffb026fd66dfaf02aecd96cec Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 24 Jul 2018 10:16:26 -0500 Subject: [PATCH 27/41] MCOL-1472 Add switch to handle nested case --- dbcon/mysql/ha_calpont_execplan.cpp | 7 ++++++- dbcon/mysql/ha_calpont_impl_if.h | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 6113e7ca4..42d26108c 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -1268,7 +1268,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) ifp->functype() == Item_func::ISNOTNULL_FUNC) { ReturnedColumn* rhs = NULL; - if (!gwip->rcWorkStack.empty()) + if (!gwip->rcWorkStack.empty() && !gwip->inCaseStmt) { rhs = gwip->rcWorkStack.top(); gwip->rcWorkStack.pop(); @@ -3267,7 +3267,12 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS if (funcName == "case_searched" && (i < arg_offset)) { + // MCOL-1472 Nested CASE with an ISNULL predicate. We don't want the predicate + // to pull off of rcWorkStack, so we set this inCaseStmt flag to tell it + // not to. + gwi.inCaseStmt = true; sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); + gwi.inCaseStmt = false; if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data()) { gwi.ptWorkStack.pop(); diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 9a4fd8bd7..ab7b81034 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -147,6 +147,9 @@ struct gp_walk_info int32_t recursionHWM; std::stack rcBookMarkStack; + // Kludge for MCOL-1472 + bool inCaseStmt; + gp_walk_info() : sessionid(0), fatalParseError(false), condPush(false), @@ -162,7 +165,8 @@ struct gp_walk_info lastSub(0), derivedTbCnt(0), recursionLevel(-1), - recursionHWM(0) + recursionHWM(0), + inCaseStmt(false) {} ~gp_walk_info() {} From 0e856ce9b0e8925938d1651c87bd571287df37fb Mon Sep 17 00:00:00 2001 From: drrtuy Date: Tue, 24 Jul 2018 23:05:09 +0300 Subject: [PATCH 28/41] MCOL-1551 CS now supports hostnames in Columnstore.xml. --- utils/messageqcpp/messagequeue.cpp | 81 +++++++++++++++++++------- utils/messageqcpp/messagequeue.h | 3 +- utils/messageqcpp/messagequeuepool.cpp | 6 +- utils/messageqcpp/messagequeuepool.h | 2 +- 4 files changed, 65 insertions(+), 27 deletions(-) diff --git a/utils/messageqcpp/messagequeue.cpp b/utils/messageqcpp/messagequeue.cpp index 4800faf13..085426d9c 100644 --- a/utils/messageqcpp/messagequeue.cpp +++ b/utils/messageqcpp/messagequeue.cpp @@ -152,26 +152,44 @@ void MessageQueueClient::shutdown() void MessageQueueClient::setup(bool syncProto) { - string otherEndIPStr; - string otherEndPortStr; - uint16_t port; + string otherEndIPStr; + string otherEndPortStr; + struct addrinfo hints, *servinfo; + int rc = 0; - otherEndIPStr = fConfig->getConfig(fOtherEnd, "IPAddr"); - otherEndPortStr = fConfig->getConfig(fOtherEnd, "Port"); + otherEndIPStr = fConfig->getConfig(fOtherEnd, "IPAddr"); + otherEndPortStr = fConfig->getConfig(fOtherEnd, "Port"); - if (otherEndIPStr.length() == 0) otherEndIPStr = "127.0.0.1"; + if (otherEndIPStr.length() == 0) otherEndIPStr = "127.0.0.1"; - if (otherEndPortStr.length() == 0 || (port = static_cast(strtol(otherEndPortStr.c_str(), 0, 0))) == 0) - { - string msg = "MessageQueueClient::MessageQueueClient: config error: Invalid/Missing Port attribute"; - throw runtime_error(msg); - } + if (otherEndPortStr.length() == 0 || static_cast(strtol(otherEndPortStr.c_str(), 0, 0)) == 0) + { + string msg = "MessageQueueClient::setup(): config error: Invalid/Missing Port attribute"; + throw runtime_error(msg); + } - memset(&fServ_addr, 0, sizeof(fServ_addr)); - sockaddr_in* sinp = reinterpret_cast(&fServ_addr); - sinp->sin_family = AF_INET; - sinp->sin_port = htons(port); - sinp->sin_addr.s_addr = inet_addr(otherEndIPStr.c_str()); + memset(&hints, 0, sizeof hints); + // ATM We support IPv4 only. + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + + if( !(rc = getaddrinfo(otherEndIPStr.c_str(), otherEndPortStr.c_str(), &hints, &servinfo)) ) + { + memset(&fServ_addr, 0, sizeof(fServ_addr)); + sockaddr_in* sinp = reinterpret_cast(&fServ_addr); + *sinp = *reinterpret_cast(servinfo->ai_addr); + freeaddrinfo(servinfo); + } + else + { + string msg = "MessageQueueClient::setup(): "; + msg.append(gai_strerror(rc)); + logging::Message::Args args; + logging::LoggingID li(31); + args.add(msg); + fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li); + } #ifdef SKIP_IDB_COMPRESSION fClientSock.setSocketImpl(new InetStreamSocket()); @@ -197,15 +215,34 @@ MessageQueueClient::MessageQueueClient(const string& otherEnd, Config* config, b setup(syncProto); } -MessageQueueClient::MessageQueueClient(const string& ip, uint16_t port, bool syncProto) : +MessageQueueClient::MessageQueueClient(const string& dnOrIp, uint16_t port, bool syncProto) : fLogger(31), fIsAvailable(true) { - memset(&fServ_addr, 0, sizeof(fServ_addr)); - sockaddr_in* sinp = reinterpret_cast(&fServ_addr); - sinp->sin_family = AF_INET; - sinp->sin_port = htons(port); - sinp->sin_addr.s_addr = inet_addr(ip.c_str()); + struct addrinfo hints, *servinfo; + int rc = 0; + memset(&hints, 0, sizeof hints); + // ATM We support IPv4 only. + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + if( !(rc = getaddrinfo(dnOrIp.c_str(), NULL, &hints, &servinfo)) ) + { + memset(&fServ_addr, 0, sizeof(fServ_addr)); + sockaddr_in* sinp = reinterpret_cast(&fServ_addr); + *sinp = *reinterpret_cast(servinfo->ai_addr); + sinp->sin_port = htons(port); + freeaddrinfo(servinfo); + } + else + { + string msg = "MessageQueueClient::MessageQueueClient(): "; + msg.append(gai_strerror(rc)); + logging::Message::Args args; + logging::LoggingID li(31); + args.add(msg); + fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li); + } #ifdef SKIP_IDB_COMPRESSION fClientSock.setSocketImpl(new InetStreamSocket()); #else diff --git a/utils/messageqcpp/messagequeue.h b/utils/messageqcpp/messagequeue.h index 8de4df398..e33e5cd84 100644 --- a/utils/messageqcpp/messagequeue.h +++ b/utils/messageqcpp/messagequeue.h @@ -33,6 +33,7 @@ #include #else #include +#include #endif #include "serversocket.h" @@ -182,7 +183,7 @@ public: * * construct a queue from this process to otherEnd on the given IP and Port. */ - EXPORT explicit MessageQueueClient(const std::string& ip, uint16_t port, bool syncProto=true); + EXPORT explicit MessageQueueClient(const std::string& dnOrIp, uint16_t port, bool syncProto=true); /** diff --git a/utils/messageqcpp/messagequeuepool.cpp b/utils/messageqcpp/messagequeuepool.cpp index 5b8c9862c..27459991f 100644 --- a/utils/messageqcpp/messagequeuepool.cpp +++ b/utils/messageqcpp/messagequeuepool.cpp @@ -36,12 +36,12 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts) return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000; } -MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &ip, uint64_t port) +MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &dnOrIp, uint64_t port) { boost::mutex::scoped_lock lock(queueMutex); std::ostringstream oss; - oss << ip << "_" << port; + oss << dnOrIp << "_" << port; std::string searchString = oss.str(); MessageQueueClient *returnClient = MessageQueueClientPool::findInPool(searchString); @@ -58,7 +58,7 @@ MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &ip, u clock_gettime(CLOCK_MONOTONIC, &now); uint64_t nowSeconds = TimeSpecToSeconds(&now); - newClientObject->client = new MessageQueueClient(ip, port); + newClientObject->client = new MessageQueueClient(dnOrIp, port); newClientObject->inUse = true; newClientObject->lastUsed = nowSeconds; clientMap.insert(std::pair(searchString, newClientObject)); diff --git a/utils/messageqcpp/messagequeuepool.h b/utils/messageqcpp/messagequeuepool.h index fc5576203..227b13b2c 100644 --- a/utils/messageqcpp/messagequeuepool.h +++ b/utils/messageqcpp/messagequeuepool.h @@ -41,7 +41,7 @@ class MessageQueueClientPool { public: static MessageQueueClient *getInstance(const std::string &module); - static MessageQueueClient *getInstance(const std::string &ip, uint64_t port); + static MessageQueueClient *getInstance(const std::string &dnOrIp, uint64_t port); static void releaseInstance(MessageQueueClient * client); static void deleteInstance(MessageQueueClient * client); static MessageQueueClient *findInPool(const std::string &search); From 926314bf17cbe6848dcff76c63c6f60d4e3f3346 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 26 Jul 2018 14:30:14 -0500 Subject: [PATCH 29/41] MCOL-1523 - enhance to failover module when ddl/dmlproc crashes --- dbcon/execplan/clientrotator.cpp | 32 +++++++++++++++++++++++ procmgr/processmanager.cpp | 4 +-- procmon/main.cpp | 4 +-- procmon/processmonitor.cpp | 45 ++++++++++++++++++++------------ procmon/processmonitor.h | 2 +- 5 files changed, 65 insertions(+), 22 deletions(-) diff --git a/dbcon/execplan/clientrotator.cpp b/dbcon/execplan/clientrotator.cpp index 3fc45a7e5..f71299840 100644 --- a/dbcon/execplan/clientrotator.cpp +++ b/dbcon/execplan/clientrotator.cpp @@ -49,6 +49,15 @@ using namespace logging; #include "clientrotator.h" +//#include "idb_mysql.h" + +/** Debug macro */ +#ifdef INFINIDB_DEBUG +#define IDEBUG(x) {x;} +#else +#define IDEBUG(x) {} +#endif + #define LOG_TO_CERR namespace execplan @@ -60,13 +69,36 @@ const uint64_t LOCAL_EXEMGR_PORT = 8601; string ClientRotator::getModule() { string installDir = startup::StartUp::installDir(); + + //Log to debug.log + LoggingID logid( 24, 0, 0); + string fileName = installDir + "/local/module"; + string module; ifstream moduleFile (fileName.c_str()); if (moduleFile.is_open()) + { getline (moduleFile, module); + } + else + { + { + logging::Message::Args args1; + logging::Message msg(1); + std::ostringstream oss; + oss << "ClientRotator::getModule open status2 =" << strerror(errno); + args1.add(oss.str()); + args1.add(fileName); + msg.format( args1 ); + Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + } + } + moduleFile.close(); + return module; } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index a6ca886af..0b8f4af59 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -827,7 +827,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) } if (opState == oam::MAN_OFFLINE || opState == oam::MAN_DISABLED - || opState == oam::AUTO_DISABLED ) { + || opState == oam::AUTO_DISABLED || opState == oam::AUTO_OFFLINE) { oam.dbrmctl("halt"); log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG); @@ -848,7 +848,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) } else { - log.writeLog(__LINE__, "ERROR: module not stopped", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "ERROR: module not stopped, state = " + oam.itoa(opState), LOG_TYPE_ERROR); status = API_FAILURE; break; } diff --git a/procmon/main.cpp b/procmon/main.cpp index b4e23a6e1..d6edd4ac7 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -1395,7 +1395,7 @@ static void chldHandleThread(MonitorConfig config) {} // check if process failover is needed due to process outage - aMonitor.checkProcessFailover((*listPtr).ProcessName); + aMonitor.checkModuleFailover((*listPtr).ProcessName); //check the db health if (DBFunctionalMonitorFlag == "y" ) { @@ -1470,7 +1470,7 @@ static void chldHandleThread(MonitorConfig config) (*listPtr).processID = 0; // check if process failover is needed due to process outage - aMonitor.checkProcessFailover((*listPtr).ProcessName); + aMonitor.checkModuleFailover((*listPtr).ProcessName); break; } else diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 91f78e640..754d6ccf2 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1174,7 +1174,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO // error in launching a process if ( requestStatus == oam::API_FAILURE && (*listPtr).RunType == SIMPLEX) - checkProcessFailover((*listPtr).ProcessName); + checkModuleFailover((*listPtr).ProcessName); else break; } @@ -4625,19 +4625,19 @@ std::string ProcessMonitor::sendMsgProcMon1( std::string module, ByteStream msg, } /****************************************************************************************** -* @brief checkProcessFailover +* @brief checkModuleFailover * -* purpose: check if process failover is needed due to a process outage +* purpose: check if module failover is needed due to a process outage * ******************************************************************************************/ -void ProcessMonitor::checkProcessFailover( std::string processName) +void ProcessMonitor::checkModuleFailover( std::string processName) { Oam oam; //force failover on certain processes if ( processName == "DDLProc" || processName == "DMLProc" ) { - log.writeLog(__LINE__, "checkProcessFailover: process failover, process outage of " + processName, LOG_TYPE_CRITICAL); + log.writeLog(__LINE__, "checkModuleFailover: process failover, process outage of " + processName, LOG_TYPE_CRITICAL); try { @@ -4656,26 +4656,37 @@ void ProcessMonitor::checkProcessFailover( std::string processName) systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_OFFLINE || systemprocessstatus.processstatus[i].ProcessOpState == oam::FAILED ) { // found a AVAILABLE mate, start it - log.writeLog(__LINE__, "start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Change UM Master to module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Disable local UM module " + config.moduleName(), LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Stop local UM module " + config.moduleName(), LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Disable Local will Enable UM module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); - try { - oam.setSystemConfig("PrimaryUMModuleName", systemprocessstatus.processstatus[i].Module); - - //distribute config file - oam.distributeConfigFile("system"); - sleep(1); - } - catch(...) {} + oam::DeviceNetworkConfig devicenetworkconfig; + oam::DeviceNetworkList devicenetworklist; + + devicenetworkconfig.DeviceName = config.moduleName(); + devicenetworklist.push_back(devicenetworkconfig); try { - oam.startProcess(systemprocessstatus.processstatus[i].Module, processName, FORCEFUL, ACK_YES); - log.writeLog(__LINE__, "success start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); + oam.stopModule(devicenetworklist, oam::FORCEFUL, oam::ACK_YES); + log.writeLog(__LINE__, "success stopModule on module " + config.moduleName(), LOG_TYPE_DEBUG); + + try + { + oam.disableModule(devicenetworklist); + log.writeLog(__LINE__, "success disableModule on module " + config.moduleName(), LOG_TYPE_DEBUG); + } + catch (exception& e) + { + log.writeLog(__LINE__, "failed disableModule on module " + config.moduleName(), LOG_TYPE_ERROR); + } } catch (exception& e) { - log.writeLog(__LINE__, "failed start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_ERROR); + log.writeLog(__LINE__, "failed stopModule on module " + config.moduleName(), LOG_TYPE_ERROR); } + break; } } diff --git a/procmon/processmonitor.h b/procmon/processmonitor.h index 5a3145963..8a4c3c8be 100644 --- a/procmon/processmonitor.h +++ b/procmon/processmonitor.h @@ -487,7 +487,7 @@ public: /** *@brief check if module failover is needed due to a process outage */ - void checkProcessFailover( std::string processName); + void checkModuleFailover(std::string processName); /** *@brief run upgrade script From a3862a42f79622e7c31959aba1e7cf946283c379 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 31 Jul 2018 15:51:32 -0500 Subject: [PATCH 30/41] um failover changes --- procmgr/processmanager.cpp | 80 ++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 29 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 0b8f4af59..0a054f9c3 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -373,7 +373,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) msg >> target; msg >> graceful; msg >> ackIndicator; - msg >> manualFlag; + msg >> manualFlag; switch (actionType) { case STOPMODULE: @@ -835,16 +835,18 @@ void processMSG(messageqcpp::IOSocket* cfIos) status = processManager.disableModule(moduleName, true); log.writeLog(__LINE__, "Disable Module Completed on " + moduleName, LOG_TYPE_INFO); + processManager.recycleProcess(moduleName); + + //check for SIMPLEX Processes on mate might need to be started + processManager.checkSimplexModule(moduleName); + //call dbrm control - oam.dbrmctl("reload"); - log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); +// oam.dbrmctl("reload"); +// log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); // resume the dbrm oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); - - //check for SIMPLEX Processes on mate might need to be started - processManager.checkSimplexModule(moduleName); } else { @@ -3299,7 +3301,9 @@ int ProcessManager::disableModule(string target, bool manualFlag) // update state to MAN_DISABLED if (opState == oam::AUTO_DISABLED && newState == oam::MAN_DISABLED) { - + //removemodule to get proess in MAN_OFFLINE + stopModule(target, REMOVE, true); + try { oam.getSystemConfig(target, moduleconfig); @@ -3351,7 +3355,7 @@ int ProcessManager::disableModule(string target, bool manualFlag) setModuleState(target, newState); - //set Columnstore.xml enbale state + //set Columnstore.xml enable state setEnableState( target, SnewState); log.writeLog(__LINE__, "disableModule - setEnableState", LOG_TYPE_DEBUG); @@ -3435,18 +3439,18 @@ void ProcessManager::recycleProcess(string module, bool enableModule) restartProcessType("PrimProc"); sleep(1); - restartProcessType("ExeMgr"); - sleep(1); - restartProcessType("mysqld"); restartProcessType("WriteEngineServer"); sleep(1); - restartProcessType("DDLProc",module); + startProcessType("ExeMgr"); sleep(1); - restartProcessType("DMLProc",module); + startProcessType("DDLProc"); + sleep(1); + + startProcessType("DMLProc"); return; } @@ -4156,7 +4160,8 @@ int ProcessManager::stopProcessType( std::string processName, bool manualFlag ) { if ( systemprocessstatus.processstatus[i].ProcessName == processName) { //skip if in a COLD_STANDBY state - if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY ) +// if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY ) + if ( systemprocessstatus.processstatus[i].ProcessOpState != oam::ACTIVE ) continue; // found one, request restart of it @@ -4286,12 +4291,17 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski { if ( systemprocessstatus.processstatus[i].ProcessName == processName ) { //skip if in a BUSY_INIT state - if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::BUSY_INIT || - systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_INIT || - systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_INIT || - ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) ) - continue; +// if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::BUSY_INIT || +// systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_OFFLINE || +// systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_OFFLINE || +// systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_INIT || +// systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_INIT || +// ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) ) +// continue; + if ( systemprocessstatus.processstatus[i].ProcessOpState != oam::ACTIVE ) + continue; + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { string procModuleType = systemprocessstatus.processstatus[i].Module.substr(0,MAX_MODULE_TYPE_SIZE); @@ -7539,18 +7549,18 @@ void ProcessManager::checkSimplexModule(std::string moduleName) } if ( state == oam::COLD_STANDBY ) { - //set Primary UM Module - if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc" ) { + //process DDL/DMLProc + if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc") + { + setPMProcIPs((*pt).DeviceName); + + log.writeLog(__LINE__, "Set Primary UM Module = " + (*pt).DeviceName, LOG_TYPE_DEBUG); + oam.setSystemConfig("PrimaryUMModuleName", (*pt).DeviceName); //distribute config file distributeConfigFile("system"); sleep(2); - - //add MySQL Replication setup, if needed - log.writeLog(__LINE__, "Setup MySQL Replication for COLD_STANDBY DMLProc going ACTIVE", LOG_TYPE_DEBUG); - oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, (*pt).DeviceName); } int status = processManager.startProcess((*pt).DeviceName, @@ -7559,12 +7569,24 @@ void ProcessManager::checkSimplexModule(std::string moduleName) if ( status == API_SUCCESS ) { log.writeLog(__LINE__, "checkSimplexModule: mate process started: " + (*pt).DeviceName + "/" + systemprocessconfig.processconfig[j].ProcessName, LOG_TYPE_DEBUG); - //check to see if DDL/DML IPs need to be updated - if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc" ) - setPMProcIPs((*pt).DeviceName); + status = processManager.startProcess((*pt).DeviceName, + "DMLProc", + FORCEFUL); + if ( status == API_SUCCESS ) { + log.writeLog(__LINE__, "checkSimplexModule: mate process started: " + (*pt).DeviceName + "/DMLProc", LOG_TYPE_DEBUG); + } + else + log.writeLog(__LINE__, "checkSimplexModule: mate process failed to start: " + (*pt).DeviceName + "/DMLProc", LOG_TYPE_DEBUG); } else log.writeLog(__LINE__, "checkSimplexModule: mate process failed to start: " + (*pt).DeviceName + "/" + systemprocessconfig.processconfig[j].ProcessName, LOG_TYPE_DEBUG); + + //setup new MariaDB Replication Master + if ( systemprocessconfig.processconfig[j].ProcessName == "DMLProc" ) { + log.writeLog(__LINE__, "Setup MySQL Replication for COLD_STANDBY DMLProc going ACTIVE", LOG_TYPE_DEBUG); + oam::DeviceNetworkList devicenetworklist; + processManager.setMySQLReplication(devicenetworklist, (*pt).DeviceName); + } } else { // if found ACTIVE, skip to next process From d86fabff653e7fbff234bd97f725be1da668f344 Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 1 Aug 2018 10:17:13 -0500 Subject: [PATCH 31/41] MCOL-1145/1146 - fix nonroot install lib issue --- oamapps/postConfigure/installer.cpp | 10 +++- oamapps/postConfigure/postConfigure.cpp | 57 +++++++++++-------- .../postConfigure/quick_installer_amazon.sh | 4 +- .../quick_installer_multi_server.sh | 4 +- .../quick_installer_single_server.sh | 2 +- 5 files changed, 45 insertions(+), 32 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 1cc84ae0f..17c312c27 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -813,7 +813,10 @@ int main(int argc, char *argv[]) cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; - cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl; + if ( !rootUser ) + cout << ". /etc/profile.d/columnstoreEnv.sh" << endl; + + cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl; cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; @@ -829,7 +832,10 @@ int main(int argc, char *argv[]) cout << endl << "ERROR: MariaDB ColumnStore Process failed to start, check log files in /var/log/mariadb/columnstore" << endl; cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; - cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl; + if ( !rootUser ) + cout << ". /etc/profile.d/columnstoreEnv.sh" << endl; + + cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl; cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 98227da9d..e0df9761a 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1247,26 +1247,16 @@ int main(int argc, char *argv[]) //amazon install setup check bool amazonInstall = false; string cloud = oam::UnassignedName; - system("aws --version > /tmp/amazon.log 2>&1"); - - ifstream in("/tmp/amazon.log"); - - in.seekg(0, std::ios::end); - int size = in.tellg(); - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) + + if (!multi_server_quick_install) { - // not running on amazon with ec2-api-tools - if (amazon_quick_install) - { - cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; - exit(1); - } + system("aws --version > /tmp/amazon.log 2>&1"); - amazonInstall = false; - } - else - { - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) + ifstream in("/tmp/amazon.log"); + + in.seekg(0, std::ios::end); + int size = in.tellg(); + if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) { // not running on amazon with ec2-api-tools if (amazon_quick_install) @@ -1278,9 +1268,23 @@ int main(int argc, char *argv[]) amazonInstall = false; } else - amazonInstall = true; - } + { + if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) + { + // not running on amazon with ec2-api-tools + if (amazon_quick_install) + { + cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + exit(1); + } + amazonInstall = false; + } + else + amazonInstall = true; + } + } + try { cloud = sysConfig->getConfig(InstallSection, "Cloud"); } @@ -3641,9 +3645,6 @@ int main(int argc, char *argv[]) } //set mysql replication, if wasn't setup before on system -// if ( ( mysqlRep && pmwithum ) || -// ( mysqlRep && (umNumber > 1) ) || -// ( mysqlRep && (pmNumber > 1) && (IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM) ) ) if ( mysqlRep ) { cout << endl << "Run MariaDB ColumnStore Replication Setup.. "; @@ -3665,7 +3666,10 @@ int main(int argc, char *argv[]) cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; - cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl; + if ( !rootUser ) + cout << ". /etc/profile.d/columnstoreEnv.sh" << endl; + + cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl; cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; @@ -3682,7 +3686,10 @@ int main(int argc, char *argv[]) cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; - cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl; + if ( !rootUser ) + cout << ". /etc/profile.d/columnstoreEnv.sh" << endl; + + cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl; cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh index c71dc8920..4a1df903c 100755 --- a/oamapps/postConfigure/quick_installer_amazon.sh +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -74,8 +74,8 @@ else echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umCount = "" ]]; then - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName else - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName fi fi diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh index 25d615d7e..dbb603220 100755 --- a/oamapps/postConfigure/quick_installer_multi_server.sh +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -78,8 +78,8 @@ else echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umIpAddrs = "" ]]; then - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName else - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName fi fi diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh index 2603bcec8..432b395c4 100755 --- a/oamapps/postConfigure/quick_installer_single_server.sh +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -30,5 +30,5 @@ else $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore echo "Run postConfigure script" echo "" - $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs + . /etc/profile.d/columnstoreEnv.sh; $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs fi From 8043674432a0e0c5f4ca4af4acc2cac0ecee561b Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 1 Aug 2018 16:34:40 -0500 Subject: [PATCH 32/41] MCOL-1591 - add umask test to tester --- .../clusterTester/columnstoreClusterTester.sh | 156 +++++++++++++++--- utils/clusterTester/os_detect.sh | 2 +- 2 files changed, 132 insertions(+), 26 deletions(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 9f8f3e9a3..f68d16c55 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -10,7 +10,7 @@ CHECK=true REPORTPASS=true LOGFILE="" -OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16") +OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16" "ubuntu18") NODE_IPADDRESS="" @@ -37,7 +37,7 @@ checkContinue() { } ### -# Print Fucntions +# Print Functions ### helpPrint () { @@ -57,7 +57,7 @@ helpPrint () { echo "" echo "Additional information on Tool is documented at:" echo "" - echo "https://mariadb.com/kb/en/mariadb/*****/" + echo "https://mariadb.com/kb/en/library/mariadb-columnstore-cluster-test-tool/" echo "" echo "Items that are checked:" echo " Node Ping test" @@ -65,6 +65,7 @@ helpPrint () { echo " ColumnStore Port test" echo " OS version" echo " Locale settings" + echo " Umask settings" echo " Firewall settings" echo " Date/time settings" echo " Dependent packages installed" @@ -326,16 +327,18 @@ checkSSH() rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then if [ $PASSWORD == "ssh" ] ; then - echo $ipadd " Node Passed SSH login test using ssh-keys" + echo $ipadd " Node Passed SSH login test using ssh-keys" else - echo $ipadd " Node Passed SSH login test using user password" + echo $ipadd " Node Passed SSH login test using user password" fi else if [ $PASSWORD == "ssh" ] ; then - echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys" + echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys" else - echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password" + echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password" fi + + echo "Error - Fix the SSH login issue and rerun test" exit 1 fi done @@ -489,12 +492,47 @@ checkLocale() fi } -checkSELINUX() +checkLocalUMASK() +{ + # UMASK check + # + echo "" + echo "** Run Local UMASK check" + echo "" + + pass=true + filename=UMASKtest + + rm -f $filename + touch $filename + permission=$(stat -c "%A" "$filename") + result=${permission:4:1} + if [ ${result} == "r" ] ; then + result=${permission:7:1} + if [ ${result} == "r" ] ; then + echo "UMASK local setting test passed" + else + echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022" + pass=false + fi + else + echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022" + pass=false + fi + + if ! $pass; then + checkContinue + fi + + rm -f $filename +} + +checkLocalSELINUX() { # SELINUX check # echo "" - echo "** Run SELINUX check" + echo "** Run Local SELINUX check" echo "" pass=true @@ -511,21 +549,86 @@ checkSELINUX() echo "Local Node SELINUX setting is Not Enabled" fi - for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1` - if [ "$?" -ne 0 ]; then - echo "$ipadd Node SELINUX setting is Not Enabled" - else - `cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` - if [ "$?" -eq 0 ]; then - echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results" - pass=false - else - echo "$ipadd Node SELINUX setting is Not Enabled" - fi - `rm -f config` - fi - done + if ! $pass; then + checkContinue + fi +} + +checkUMASK() +{ + # UMASK check + # + echo "" + echo "** Run UMASK check" + echo "" + + pass=true + + for ipadd in "${NODE_IPADDRESS[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > /tmp/remote_command_check 2>&1` + if [ "$?" -eq 0 ]; then + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> /tmp/remote_scp_get 2>&1` + if [ "$?" -eq 0 ]; then + permission=`cat test.log` + result=${permission:4:1} + if [ ${result} == "r" ] ; then + result=${permission:7:1} + if [ ${result} == "r" ] ; then + echo "$ipadd Node UMASK setting test passed" + else + echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022" + pass=false + fi + else + echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022" + pass=false + fi + else + echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check /tmp/remote_scp_get" + pass=false + fi + else + echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check /tmp/remote_command_check" + pass=false + fi + `rm -f test.log` + done + + if ! $pass; then + checkContinue + fi + + rm -f $filename +} + +checkSELINUX() +{ + # SELINUX check + # + echo "" + echo "** Run SELINUX check" + echo "" + + pass=true + for ipadd in "${NODE_IPADDRESS[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "$ipadd Node SELINUX setting is Not Enabled" + else + `cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results" + pass=false + else + echo "$ipadd Node SELINUX setting is Not Enabled" + fi + `rm -f config` + fi + done + + if ! $pass; then + checkContinue + fi } checkFirewalls() @@ -949,7 +1052,7 @@ checkPackages() declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" ) declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") - if [ "$OS" == "ubuntu16" ] ; then + if [ "$OS" == "ubuntu16" ] || [ "$OS" == "ubuntu18" ]; then if [ ! `which dpkg 2>/dev/null` ] ; then echo "${bold}Failed${normal}, Local Node ${bold}rpm${normal} package not installed" pass=false @@ -1307,12 +1410,15 @@ echo "" checkLocalOS checkLocalDir +checkLocalUMASK +checkLocalSELINUX if [ "$IPADDRESSES" != "" ]; then checkPing checkSSH checkRemoteDir checkOS checkLocale + checkUMASK checkSELINUX checkFirewalls checkPorts diff --git a/utils/clusterTester/os_detect.sh b/utils/clusterTester/os_detect.sh index 7930c0daf..be69e870e 100755 --- a/utils/clusterTester/os_detect.sh +++ b/utils/clusterTester/os_detect.sh @@ -29,7 +29,7 @@ detectOS () { echo Operating System name: $osPrettyName echo Operating System tag: $osTag case "$osTag" in - centos6|centos7|ubuntu16|debian8|suse12|debian9) + centos6|centos7|ubuntu16|debian8|suse12|debian9|ubuntu18) ;; *) echo OS not supported From 5d245c8932fc8698ca26ab70d4f7b55efaa9567e Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 2 Aug 2018 10:59:38 -0500 Subject: [PATCH 33/41] MCOL-1498 - add prompt for password on non-distibute installs --- oamapps/postConfigure/postConfigure.cpp | 72 ++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 98227da9d..774a7a490 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3090,7 +3090,9 @@ int main(int argc, char *argv[]) //check if dbrm data resides in older directory path and inform user if it does dbrmDirCheck(); - if ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM && pmNumber == 1) { + if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || + ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) + { //run the mysql / mysqld setup scripts cout << endl << "===== Running the MariaDB ColumnStore MariaDB Server setup scripts =====" << endl << endl; @@ -3098,7 +3100,61 @@ int main(int argc, char *argv[]) // call the mysql setup scripts mysqlSetup(); - sleep(5); + sleep(3); + } + + if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || + pmNumber > 1 ) { + + if ( password.empty() ) + { + cout << endl; + cout << "Next step is to enter the password to access the other Servers." << endl; + cout << "This is either your password or you can default to using a ssh key" << endl; + cout << "If using a password, the password needs to be the same on all Servers." << endl << endl; + } + + while(true) + { + char *pass1, *pass2; + + if ( noPrompting ) { + cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl; + if ( password.empty() ) + password = "ssh"; + break; + } + + //check for command line option password + //if ( !password.empty() ) + // break; + + pass1=getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > "); + if ( strcmp(pass1, "") == 0 ) { + password = "ssh"; + break; + } + + string p1 = pass1; + if ( p1 == "exit") + exit(0); + + pass2=getpass("Confirm password > "); + string p2 = pass2; + if ( p1 == p2 ) { + password = p2; + break; + } + else + cout << "Password mismatch, please re-enter" << endl; + } + + //add single quote for special characters + if ( password != "ssh" ) + { + password = "'" + password + "'"; + } + } int thread_id = 0; @@ -3173,7 +3229,7 @@ int main(int argc, char *argv[]) if( !pkgCheck(columnstorePackage) ) exit(1); - if ( password.empty() ) +/* if ( password.empty() ) { cout << endl; cout << "Next step is to enter the password to access the other Servers." << endl; @@ -3221,10 +3277,10 @@ int main(int argc, char *argv[]) { password = "'" + password + "'"; } - +*/ checkSystemMySQLPort(mysqlPort, sysConfig, USER, password, childmodulelist, IserverTypeInstall, pmwithum); - if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || +/* if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) { cout << endl << "===== Running the MariaDB ColumnStore MariaDB ColumnStore setup scripts =====" << endl << endl; @@ -3233,7 +3289,7 @@ int main(int argc, char *argv[]) mysqlSetup(); sleep(5); } - +*/ string AmazonInstall = "0"; if ( amazonInstall ) AmazonInstall = "1"; @@ -3411,7 +3467,7 @@ int main(int argc, char *argv[]) cout << " DONE" << endl; } } - else +/* else { if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) @@ -3423,7 +3479,7 @@ int main(int argc, char *argv[]) sleep(5); } } - +*/ //configure data redundancy if (DataRedundancy) { From 0837f9a520eb4315f5b5d5e20d727d45bc35d9e4 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 2 Aug 2018 11:51:27 -0500 Subject: [PATCH 34/41] MCOL-1498 - add prompt for password on non-distibute installs --- oamapps/postConfigure/postConfigure.cpp | 141 ++++++------------------ 1 file changed, 33 insertions(+), 108 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 774a7a490..c31f0c5de 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3104,57 +3104,53 @@ int main(int argc, char *argv[]) } if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || - pmNumber > 1 ) { - + pmNumber > 1 ) + { if ( password.empty() ) { cout << endl; cout << "Next step is to enter the password to access the other Servers." << endl; cout << "This is either your password or you can default to using a ssh key" << endl; cout << "If using a password, the password needs to be the same on all Servers." << endl << endl; - } - - while(true) - { - char *pass1, *pass2; if ( noPrompting ) { cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl; - if ( password.empty() ) - password = "ssh"; - break; - } - - //check for command line option password - //if ( !password.empty() ) - // break; - - pass1=getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > "); - if ( strcmp(pass1, "") == 0 ) { password = "ssh"; - break; } + else + { + while(true) + { + char *pass1, *pass2; - string p1 = pass1; - if ( p1 == "exit") - exit(0); + pass1=getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > "); + if ( strcmp(pass1, "") == 0 ) { + password = "ssh"; + break; + } - pass2=getpass("Confirm password > "); - string p2 = pass2; - if ( p1 == p2 ) { - password = p2; - break; - } - else - cout << "Password mismatch, please re-enter" << endl; - } + string p1 = pass1; + if ( p1 == "exit") + exit(0); - //add single quote for special characters - if ( password != "ssh" ) - { - password = "'" + password + "'"; - } + pass2=getpass("Confirm password > "); + string p2 = pass2; + if ( p1 == p2 ) { + password = p2; + break; + } + else + cout << "Password mismatch, please re-enter" << endl; + } + //add single quote for special characters + if ( password != "ssh" ) + { + password = "'" + password + "'"; + } + + } + } } int thread_id = 0; @@ -3229,67 +3225,8 @@ int main(int argc, char *argv[]) if( !pkgCheck(columnstorePackage) ) exit(1); -/* if ( password.empty() ) - { - cout << endl; - cout << "Next step is to enter the password to access the other Servers." << endl; - cout << "This is either your password or you can default to using a ssh key" << endl; - cout << "If using a password, the password needs to be the same on all Servers." << endl << endl; - } - - while(true) - { - char *pass1, *pass2; - - if ( noPrompting ) { - cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl; - if ( password.empty() ) - password = "ssh"; - break; - } - - //check for command line option password - if ( !password.empty() ) - break; - - pass1=getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > "); - if ( strcmp(pass1, "") == 0 ) { - password = "ssh"; - break; - } - - if ( pass1 == "exit") - exit(0); - - string p1 = pass1; - pass2=getpass("Confirm password > "); - string p2 = pass2; - if ( p1 == p2 ) { - password = p2; - break; - } - else - cout << "Password mismatch, please re-enter" << endl; - } - - //add single quote for special characters - if ( password != "ssh" ) - { - password = "'" + password + "'"; - } -*/ checkSystemMySQLPort(mysqlPort, sysConfig, USER, password, childmodulelist, IserverTypeInstall, pmwithum); -/* if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || - ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) - { - cout << endl << "===== Running the MariaDB ColumnStore MariaDB ColumnStore setup scripts =====" << endl << endl; - - // call the mysql setup scripts - mysqlSetup(); - sleep(5); - } -*/ string AmazonInstall = "0"; if ( amazonInstall ) AmazonInstall = "1"; @@ -3467,19 +3404,7 @@ int main(int argc, char *argv[]) cout << " DONE" << endl; } } -/* else - { - if ( ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM ) || - ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) - { - cout << endl << "===== Running the MariaDB ColumnStore MariaDB ColumnStore setup scripts =====" << endl << endl; - - // call the mysql setup scripts - mysqlSetup(); - sleep(5); - } - } -*/ + //configure data redundancy if (DataRedundancy) { From 515cc31d4fe7924eba6d5141c52975c5c22e9245 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Mon, 6 Aug 2018 10:10:52 -0500 Subject: [PATCH 35/41] MCOL-1610: modify so if moving a dbroot fails with gluster it is reassigned to original owner. Add logging around failure for mounting gluster volumes. --- oam/oamcpp/liboamcpp.cpp | 101 +++++++++++++++++++++---------------- procmgr/main.cpp | 20 ++++---- procmgr/processmanager.cpp | 4 +- procmon/processmonitor.cpp | 7 +++ 4 files changed, 76 insertions(+), 56 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 9a405e978..7483ca239 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -5295,6 +5295,7 @@ namespace oam dbrootList dbroot1; dbroot1.push_back(*pt1); + bool returnDbRoot = false; //send msg to unmount dbroot if module is not offline int opState; @@ -5306,7 +5307,6 @@ namespace oam {} if (opState != oam::AUTO_OFFLINE || opState != oam::AUTO_DISABLED) { -// bool unmountPass = true; try { mountDBRoot(dbroot1, false); @@ -5316,13 +5316,8 @@ namespace oam writeLog("ERROR: dbroot failed to unmount", LOG_TYPE_ERROR ); cout << endl << "ERROR: umountDBRoot api failure" << endl; exceptionControl("manualMovePmDbroot", API_FAILURE); -// unmountPass = false; } -// if ( !unmountPass) { -// dbrootlist.erase(pt1); -// break; -// } } //check for amazon moving required @@ -5340,38 +5335,79 @@ namespace oam //if Gluster, do the assign command if ( DataRedundancyConfig == "y") { - try { + try + { string errmsg; int ret = glusterctl(oam::GLUSTER_ASSIGN, *pt1, toPM, errmsg); - if ( ret != 0 ) + if ( ret == 0 ) + { + todbrootConfigList.push_back(*pt2); + residedbrootConfigList.erase(pt2); + } + else { cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID + ", error: " + errmsg << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID + ", error: " + errmsg, LOG_TYPE_ERROR ); + returnDbRoot = true; } } catch (exception& e) { cout << endl << "**** glusterctl API exception: " << e.what() << endl; cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID, LOG_TYPE_ERROR ); + returnDbRoot = true; } catch (...) { cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID, LOG_TYPE_ERROR ); + returnDbRoot = true; } } - todbrootConfigList.push_back(*pt2); - - residedbrootConfigList.erase(pt2); - + if (returnDbRoot) + { + // something went wrong return it back to original owner + try + { + string errmsg; + writeLog("reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR ); + int ret = glusterctl(oam::GLUSTER_ASSIGN, *pt1, residePM, errmsg); + if ( ret != 0 ) + { + cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID + ", error: " + errmsg << endl; + writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID + ", error: " + errmsg, LOG_TYPE_ERROR ); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); + } + mountDBRoot(dbroot1); + //get updated Columnstore.xml distributed + distributeConfigFile("system"); + return; + } + catch (exception& e) + { + cout << endl << "**** glusterctl API exception: " << e.what() << endl; + cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID << endl; + writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR ); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); + } + catch (...) + { + cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; + cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID << endl; + writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR ); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); + } + } break; } } } + + //set the 2 pms dbroot config try { @@ -5381,7 +5417,7 @@ namespace oam { writeLog("ERROR: setPmDbrootConfig api failure for pm" + residePMID , LOG_TYPE_ERROR ); cout << endl << "ERROR: setPmDbrootConfig api failure for pm" + residePMID << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); } try @@ -5392,7 +5428,7 @@ namespace oam { writeLog("ERROR: setPmDbrootConfig api failure for pm" + toPMID , LOG_TYPE_ERROR ); cout << endl << "ERROR: setPmDbrootConfig api failure for pm" + toPMID << endl; - exceptionControl("manualMovePmDbroot", API_FAILURE); + exceptionControl("manualMovePmDbroot", API_INVALID_STATE); } //send msg to mount dbroot @@ -5980,7 +6016,7 @@ namespace oam } if (!found) { - writeLog("No dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_DEBUG ); + writeLog("No dbroots found in " + InstallDir + "/moveDbrootTransactionLog", LOG_TYPE_DEBUG ); cout << "No dbroots found in " << fileName << endl; } @@ -6518,32 +6554,7 @@ namespace oam for( ; pt3 != dbrootlist.end() ; pt3++) { todbrootConfigList.push_back(*pt3); - -/* if ( DataRedundancyConfig == "y") - { - try { - string errmsg; - int ret = glusterctl(oam::GLUSTER_ASSIGN, itoa(*pt3), toPM, errmsg); - if ( ret != 0 ) - { - cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID + ", error: " + errmsg << endl; - exceptionControl("assignPmDbrootConfig", API_FAILURE); - } - } - catch (exception& e) - { - cout << endl << "**** glusterctl API exception: " << e.what() << endl; - cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID << endl; - exceptionControl("assignPmDbrootConfig", API_FAILURE); - } - catch (...) - { - cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; - cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID << endl; - exceptionControl("assignPmDbrootConfig", API_FAILURE); - } - } -*/ } + } try { @@ -6961,12 +6972,14 @@ namespace oam { cout << endl << "**** glusterctl API exception: " << e.what() << endl; cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl; + writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); exceptionControl("removeDbroot", API_FAILURE); } catch (...) { cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl; + writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); exceptionControl("removeDbroot", API_FAILURE); } } diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 2747fda16..995c851c2 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1574,7 +1574,7 @@ void pingDeviceThread() { // no dbroots, fail module log.writeLog(__LINE__, "autoUnMovePmDbroot left no dbroots mounted, failing module restart: " + moduleName, LOG_TYPE_WARNING); - + //Issue an alarm aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, SET); @@ -1597,7 +1597,7 @@ void pingDeviceThread() //set query system state ready processManager.setQuerySystemState(true); - break; + goto break_case; } } catch(...) @@ -1619,25 +1619,24 @@ void pingDeviceThread() if ( retry == 5 ) { log.writeLog(__LINE__, "autoUnMovePmDbroot: Failed. Fail Module", LOG_TYPE_WARNING); - + log.writeLog(__LINE__, "System DBRM READ ONLY - Verify dbroot mounts.", LOG_TYPE_WARNING); //Issue an alarm aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, SET); //set module to disable state processManager.disableModule(moduleName, true); + // Need to do something here to verify data mounts before resuming + // Best to assume if we reach this you need to put into readonly and verify all dbroots are mounted + //call dbrm control - oam.dbrmctl("reload"); - log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); - - // resume the dbrm - oam.dbrmctl("resume"); - log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); + oam.dbrmctl("readonly"); + log.writeLog(__LINE__, "'dbrmctl readonly' done", LOG_TYPE_DEBUG); //clear count moduleInfoList[moduleName] = 0; - processManager.setSystemState(oam::ACTIVE); + processManager.setSystemState(oam::DEGRADED); //set query system state ready processManager.setQuerySystemState(true); @@ -2358,6 +2357,7 @@ void pingDeviceThread() } } //end of for loop } + break_case: // check and take action if LAN outage is flagged if (LANOUTAGESUPPORT && !LANOUTAGEACTIVE && LOCALNICDOWN) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 0a054f9c3..3cc094a0a 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -6244,7 +6244,7 @@ int ProcessManager::sendMsgProcMon( std::string module, ByteStream msg, int requ string IPAddr = sysConfig->getConfig(msgPort, "IPAddr"); if ( IPAddr == oam::UnassignedIpAddr ) { - log.writeLog(__LINE__, "sendMsgProcMon ping failure", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "sendMsgProcMon ping failure " + module + " " + IPAddr, LOG_TYPE_ERROR); return oam::API_SUCCESS; } @@ -6253,7 +6253,7 @@ int ProcessManager::sendMsgProcMon( std::string module, ByteStream msg, int requ string cmd = cmdLine + IPAddr + cmdOption; if ( system(cmd.c_str()) != 0) { //ping failure - log.writeLog(__LINE__, "sendMsgProcMon ping failure", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "sendMsgProcMon ping failure " + module + " " + IPAddr, LOG_TYPE_ERROR); return oam::API_SUCCESS; } } diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 754d6ccf2..8b7b13165 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -6107,10 +6107,13 @@ int ProcessMonitor::glusterAssign(std::string dbrootID) command = "sudo mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterAssign.txt 2>&1"; } + int ret = system(command.c_str()); if ( WEXITSTATUS(ret) != 0 ) { + log.writeLog(__LINE__, "glusterAssign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); + ifstream in("/tmp/glusterAssign.txt"); in.seekg(0, std::ios::end); int size = in.tellg(); @@ -6151,9 +6154,13 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) { command = "sudo umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterUnassign.txt 2>&1"; } + int ret = system(command.c_str()); + if ( WEXITSTATUS(ret) != 0 ) { + log.writeLog(__LINE__, "glusterUnassign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); + ifstream in("/tmp/glusterUnassign.txt"); in.seekg(0, std::ios::end); int size = in.tellg(); From 8a4294978f35f8728690641bdf0c5bbe195e34c8 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 7 Aug 2018 08:54:08 -0500 Subject: [PATCH 36/41] MCOL-1605 - changed error to debug, alarms trying to get issued before procmgr is up --- oamapps/alarmmanager/alarmmanager.cpp | 8 ++++---- procmon/main.cpp | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index b9ba4f702..b9e8c20df 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -422,7 +422,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st int pid = getpid(); int tid = gettid(); - // get reporting Pprocess Name + // get reporting Process Name string processName; if ( repProcessName.empty()) { // get current process name @@ -468,7 +468,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st args.add("sendAlarmReport error:"); args.add(e.what()); msg.format(args); - ml.logErrorMessage(msg); + ml.logDebugMessage(msg); } catch (std::exception& e) { @@ -479,7 +479,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st args.add("sendAlarmReport error:"); args.add(e.what()); msg.format(args); - ml.logErrorMessage(msg); + ml.logDebugMessage(msg); } catch (...) { @@ -490,7 +490,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st args.add("sendAlarmReport error:"); args.add("general failure"); msg.format(args); - ml.logErrorMessage(msg); + ml.logDebugMessage(msg); } return; diff --git a/procmon/main.cpp b/procmon/main.cpp index 424944f6b..1c8ce19a9 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -308,8 +308,9 @@ int main(int argc, char **argv) if ( count >= 120 ) { log.writeLog(__LINE__, "Standby PM not responding, infinidb shutting down", LOG_TYPE_CRITICAL); //Set the alarm - aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); - sleep (1); + // aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); + // sleep (1); + string cmd = startup::StartUp::installDir() + "/bin/infinidb stop > /dev/null 2>&1"; system(cmd.c_str()); } @@ -493,8 +494,8 @@ int main(int argc, char **argv) { log.writeLog(__LINE__, "Check DB mounts failed, shutting down", LOG_TYPE_CRITICAL); //Set the alarm - aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); - sleep (1); + // aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); + // sleep (1); string cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /dev/null 2>&1"; system(cmd.c_str()); } From 5927b1cf81d072300735266e7b9d16bfd6c57529 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 10 Aug 2018 16:01:12 -0500 Subject: [PATCH 37/41] MCOL-1523 - fix issue with query failing and enablemodule failing trying to get um1 back. --- oam/etc/ProcessConfig.xml | 4 ++-- procmgr/main.cpp | 24 ------------------------ procmgr/processmanager.cpp | 33 ++++++++++++++++----------------- 3 files changed, 18 insertions(+), 43 deletions(-) diff --git a/oam/etc/ProcessConfig.xml b/oam/etc/ProcessConfig.xml index 8a0c3618f..ca5d745f0 100644 --- a/oam/etc/ProcessConfig.xml +++ b/oam/etc/ProcessConfig.xml @@ -107,7 +107,7 @@ WriteEngineServer pm* DBRMWorkerNode - * + @ ExeMgr * SIMPLEX @@ -122,7 +122,7 @@ WriteEngineServer pm* DBRMWorkerNode - * + @ DDLProc @ SIMPLEX diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 995c851c2..55f675cec 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1523,9 +1523,6 @@ void pingDeviceThread() break; //set query system state not ready - BRM::DBRM dbrm; - dbrm.setSystemQueryReady(false); - processManager.setQuerySystemState(false); processManager.setSystemState(oam::BUSY_INIT); @@ -1806,9 +1803,6 @@ void pingDeviceThread() } } - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); @@ -1865,9 +1859,6 @@ void pingDeviceThread() else processManager.setSystemState(oam::ACTIVE); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); @@ -1929,9 +1920,6 @@ void pingDeviceThread() log.writeLog(__LINE__, "module is down: " + moduleName, LOG_TYPE_CRITICAL); //set query system state not ready - BRM::DBRM dbrm; - dbrm.setSystemQueryReady(false); - processManager.setQuerySystemState(false); processManager.setSystemState(oam::BUSY_INIT); @@ -1993,9 +1981,6 @@ void pingDeviceThread() oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); @@ -2201,9 +2186,6 @@ void pingDeviceThread() //set recycle process processManager.recycleProcess(moduleName); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); @@ -2220,9 +2202,6 @@ void pingDeviceThread() oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); } @@ -2236,9 +2215,6 @@ void pingDeviceThread() //set recycle process processManager.recycleProcess(moduleName); - //enable query stats - dbrm.setSystemQueryReady(true); - //set query system state ready processManager.setQuerySystemState(true); } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 3cc094a0a..5f15f3446 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -829,8 +829,10 @@ void processMSG(messageqcpp::IOSocket* cfIos) if (opState == oam::MAN_OFFLINE || opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED || opState == oam::AUTO_OFFLINE) { - oam.dbrmctl("halt"); - log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG); + processManager.setSystemState(oam::BUSY_INIT); + + //set query system state not ready + processManager.setQuerySystemState(false); status = processManager.disableModule(moduleName, true); log.writeLog(__LINE__, "Disable Module Completed on " + moduleName, LOG_TYPE_INFO); @@ -839,14 +841,12 @@ void processMSG(messageqcpp::IOSocket* cfIos) //check for SIMPLEX Processes on mate might need to be started processManager.checkSimplexModule(moduleName); + + processManager.setSystemState(oam::ACTIVE); + + //set query system state ready + processManager.setQuerySystemState(true); - //call dbrm control -// oam.dbrmctl("reload"); -// log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); - - // resume the dbrm - oam.dbrmctl("resume"); - log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); } else { @@ -910,7 +910,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) DeviceNetworkList::iterator listPT = devicenetworklist.begin(); - //stopModules being removed with the REMOVE option, which will stop process + // do stopmodule then enable for( ; listPT != devicenetworklist.end() ; listPT++) { string moduleName = (*listPT).DeviceName; @@ -933,6 +933,9 @@ void processMSG(messageqcpp::IOSocket* cfIos) } if (opState == oam::MAN_DISABLED) { + processManager.stopModule(moduleName, graceful, manualFlag); + log.writeLog(__LINE__, "stop Module Completed on " + moduleName, LOG_TYPE_INFO); + status = processManager.enableModule(moduleName, oam::MAN_OFFLINE); log.writeLog(__LINE__, "Enable Module Completed on " + moduleName, LOG_TYPE_INFO); } @@ -2758,9 +2761,6 @@ void processMSG(messageqcpp::IOSocket* cfIos) log.writeLog(__LINE__, "MSG RECEIVED: Process Restarted on " + moduleName + "/" + processName); //set query system states not ready - BRM::DBRM dbrm; - dbrm.setSystemQueryReady(false); - processManager.setQuerySystemState(false); processManager.setSystemState(oam::BUSY_INIT); @@ -2841,7 +2841,8 @@ void processMSG(messageqcpp::IOSocket* cfIos) break; sleep(1); } - dbrm.setSystemQueryReady(true); + processManager.setQuerySystemState(true); + } // if a DDLProc was restarted, reinit DMLProc @@ -2894,8 +2895,6 @@ void processMSG(messageqcpp::IOSocket* cfIos) } //enable query stats - dbrm.setSystemQueryReady(true); - processManager.setQuerySystemState(true); processManager.setSystemState(oam::ACTIVE); @@ -6489,7 +6488,7 @@ void ProcessManager::setQuerySystemState(bool set) log.writeLog(__LINE__, "setQuerySystemState = " + oam.itoa(set), LOG_TYPE_DEBUG); try { - dbrm.setSystemQueryReady(set); + dbrm.setSystemQueryReady(true); log.writeLog(__LINE__, "setQuerySystemState successful", LOG_TYPE_DEBUG); } catch(...) From b5a39ea78901dcd89078de5d8a674837c3b35dbc Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 10 Aug 2018 16:15:58 -0500 Subject: [PATCH 38/41] MCOL-1523 - fix issue with query failing and enablemodule failing trying to get um1 back. --- procmgr/processmanager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 5f15f3446..a29502ae8 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -6488,7 +6488,7 @@ void ProcessManager::setQuerySystemState(bool set) log.writeLog(__LINE__, "setQuerySystemState = " + oam.itoa(set), LOG_TYPE_DEBUG); try { - dbrm.setSystemQueryReady(true); + dbrm.setSystemQueryReady(set); log.writeLog(__LINE__, "setQuerySystemState successful", LOG_TYPE_DEBUG); } catch(...) From e903e47201337bd49129bb68c9161e3caa510063 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 10 Aug 2018 18:37:40 -0500 Subject: [PATCH 39/41] MCOL-1523 - fix issue with query failing and enablemodule failing trying to get um1 back. --- procmgr/processmanager.cpp | 15 +++++++++++++-- procmon/main.cpp | 9 ++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index a29502ae8..a8258f19f 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -1249,6 +1249,9 @@ void processMSG(messageqcpp::IOSocket* cfIos) log.writeLog(__LINE__, "STOPSYSTEM: ACK back to sender"); } + //set query system state ready + processManager.setQuerySystemState(true); + startsystemthreadStop = false; break; @@ -2848,6 +2851,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // if a DDLProc was restarted, reinit DMLProc if( processName == "DDLProc") { processManager.reinitProcessType("DMLProc"); + processManager.setQuerySystemState(true); } //only run on auto process restart @@ -2894,7 +2898,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) } } - //enable query stats + //set query system states ready processManager.setQuerySystemState(true); processManager.setSystemState(oam::ACTIVE); @@ -3773,6 +3777,7 @@ void ProcessManager::setSystemState(uint16_t state) Oam oam; ALARMManager aManager; Configuration config; + ProcessManager processManager(config, log); log.writeLog(__LINE__, "Set System State = " + oamState[state], LOG_TYPE_DEBUG); @@ -3793,6 +3798,9 @@ void ProcessManager::setSystemState(uint16_t state) // Process Alarms string system = "System"; if( state == oam::ACTIVE ) { + //set query system states ready + processManager.setQuerySystemState(true); + //clear alarms if set aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_AUTO, CLEAR); aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_MANUAL, CLEAR); @@ -6992,7 +7000,7 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist) } //set query system state not ready - processManager.setQuerySystemState(true); + processManager.setQuerySystemState(false); // Bug 4554: Wait until DMLProc is finished with rollback if (status == oam::API_SUCCESS) @@ -7061,6 +7069,9 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist) processManager.setSystemState(rtn); } + //set query system state ready + processManager.setQuerySystemState(true); + // exit thread log.writeLog(__LINE__, "startSystemThread Exit", LOG_TYPE_DEBUG); startsystemthreadStatus = status; diff --git a/procmon/main.cpp b/procmon/main.cpp index 1c8ce19a9..ad05a4f95 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -336,7 +336,7 @@ int main(int argc, char **argv) sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", IPaddr); log.writeLog(__LINE__, "set ProcMgr IPaddr to Old Standby Module: " + IPaddr, LOG_TYPE_DEBUG); - //update Calpont Config table + //update MariaDB ColumnStore Config table try { sysConfig->write(); sleep(1); @@ -1333,7 +1333,7 @@ static void chldHandleThread(MonitorConfig config) (*listPtr).processID != 0 ) || ( (*listPtr).state == oam::ACTIVE && (*listPtr).processID == 0 ) ) { - log.writeLog(__LINE__, "*****Calpont Process Restarting: " + (*listPtr).ProcessName + ", old PID = " + oam.itoa((*listPtr).processID), LOG_TYPE_CRITICAL); + log.writeLog(__LINE__, "*****MariaDB ColumnStore Process Restarting: " + (*listPtr).ProcessName + ", old PID = " + oam.itoa((*listPtr).processID), LOG_TYPE_CRITICAL); if ( (*listPtr).dieCounter >= processRestartCount || processRestartCount == 0) { @@ -1530,7 +1530,7 @@ static void chldHandleThread(MonitorConfig config) } //Log this event - log.writeLog(__LINE__, "Calpont Process " + (*listPtr).ProcessName + restartStatus, LOG_TYPE_INFO); + log.writeLog(__LINE__, "MariaDB ColumnStore Process " + (*listPtr).ProcessName + restartStatus, LOG_TYPE_INFO); } } } @@ -2455,6 +2455,9 @@ void processStatusMSG(messageqcpp::IOSocket* cfIos) memcpy(fShmSystemStatus[0].StateChangeDate, oam.getCurrentTime().c_str(), DATESIZE); log.writeLog(__LINE__, "statusControl: REQUEST RECEIVED: Set System State = " + oamState[state], LOG_TYPE_DEBUG); } + + BRM::DBRM dbrm; + dbrm.setSystemQueryReady(true); } } break; From 5df447b6ec612cab24f8d62892d2d5aca0c762d0 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Tue, 14 Aug 2018 11:59:09 -0700 Subject: [PATCH 40/41] Fix MCOL-1635 where an "insert into table select query" crashes the server. This happens for a MEDIUMBLOB column type. --- dbcon/mysql/ha_calpont_dml.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 9583d9f9e..c2dc36e47 100755 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -1615,8 +1615,11 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ } else if (ci.columnTypes[colpos].colWidth < 16777216) { - dataLength = *(uint32_t*) buf; - buf = buf + 3 ; + dataLength = *(uint16_t*) buf; + buf = buf + 2 ; + if (*(uint8_t*)buf) + dataLength += 256*256*(*(uint8_t*)buf) ; + buf++; } else { From a98aec07fad9339b74f641c1a4644c1d4738c7cb Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 17 Aug 2018 01:10:00 +0300 Subject: [PATCH 41/41] MCOL-1655 removed hardcoded %debug from ddl.y. --- dbcon/ddlpackage/CMakeLists.txt | 4 ++-- dbcon/ddlpackage/ddl.l | 3 +++ dbcon/ddlpackage/ddl.y | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index ae2f82fa9..82b7ba756 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -1,4 +1,3 @@ - include_directories( ${ENGINE_COMMON_INCLUDES} ) ADD_CUSTOM_COMMAND( @@ -9,9 +8,10 @@ ADD_CUSTOM_COMMAND( DEPENDS ddl.y ddl.l ) + # Parser puts extra info to stderr. INCLUDE(../../check_compiler_flag.cmake) -MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG" DEBUG) +MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG=1" DEBUG) ########### next target ############### diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 6eeaafb0b..7f9362cee 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -33,6 +33,9 @@ using namespace ddlpackage; typedef enum { NOOP, STRIP_QUOTES } copy_action_t; +#if YYDEBUG == 0 +int ddldebug = 0; +#endif int lineno = 1; void ddlerror(struct pass_to_bison* x, char const *s); diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 96867cfb8..2556b8340 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -63,7 +63,6 @@ char* copy_string(const char *str); %pure-parser %lex-param {void * scanner} %parse-param {struct ddlpackage::pass_to_bison * x} -%debug /* Bison uses this to generate a C union definition. This is used to store the application created values associated with syntactic