From 712c2ed90f3f21ff1ab704b011c04ebcbaf9f258 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 29 Jun 2017 14:41:17 -0500 Subject: [PATCH] MCOL-770 - more amazon related changes --- oam/install_scripts/binary_installer.sh | 10 ++--- oam/oamcpp/liboamcpp.cpp | 49 +++++++++++++++---------- procmgr/main.cpp | 8 ++-- procmgr/processmanager.cpp | 39 +++++++++++++++----- procmon/processmonitor.cpp | 25 +++++++++---- 5 files changed, 85 insertions(+), 46 deletions(-) diff --git a/oam/install_scripts/binary_installer.sh b/oam/install_scripts/binary_installer.sh index 03edd6bf9..f110ef9c4 100644 --- a/oam/install_scripts/binary_installer.sh +++ b/oam/install_scripts/binary_installer.sh @@ -7,8 +7,7 @@ # Argument 2 - Remote Server Host Name or IP address # Argument 3 - User Password of remote server # Argument 4 - Package name being installed -# Argument 5 - Module type? -# Argument 6 - Install Type, "initial", "upgrade", "uninstall", "nonDistribute" +# Argument 6 - Install Type, "initial", "upgrade", "uninstall" # Argument 7 - Server type? # Argument 8 - Debug flag 1 for on, 0 for off # Argument 9 - install dir (optional) @@ -52,7 +51,7 @@ if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } { send_user "Uninstall MariaDB Columnstore Package " send " \n" send date\n - send "ssh -v $USERNAME@$SERVER 'rm -f /etc/init.d/columnstore /etc/init.d/mysql-Columnstore $INSTALLDIR/releasenum >/dev/null 2>&1'\n" + send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/pre-uninstall --installdir=$INSTALLDIR >/dev/null 2>&1'\n" set timeout 20 expect { "Host key verification failed" { send_user "FAILED: Host key verification failed\n" ; exit 1} @@ -117,7 +116,7 @@ if { $INSTALLTYPE == "initial" } { send_user "Install MariaDB Columnstore Package on Module " send " \n" send date\n - send "ssh -v $USERNAME@$SERVER 'tar -C $PREFIX --exclude db -zxf $CALPONTPKG;cat $INSTALLDIR/releasenum'\n" + send "ssh -v $USERNAME@$SERVER 'tar -C $PREFIX --exclude db -zxf $CALPONTPKG'\n" set timeout 10 expect { "word: " { send "$PASSWORD\n" } @@ -126,7 +125,6 @@ if { $INSTALLTYPE == "initial" } { set timeout 120 expect { "Exit status 0" { send_user "DONE" } - "release=" { send_user "DONE" } "No such file" { send_user "ERROR: Binary Install Failed, binary/releasenum not found\n" ; exit 1 } "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } "Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1} @@ -151,7 +149,7 @@ set timeout 60 # check return expect { "No such file" { send_user "ERROR: post-install Not Found\n" ; exit 1 } - "MariaDB Columnstore syslog logging not working" { send_user "ERROR: MariaDB Columnstore System logging not setup\n" ; abort } + "MariaDB Columnstore syslog logging not working" { send_user "ERROR: MariaDB Columnstore System logging not setup\n" } "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } "Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1} "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index acd9227dc..c230cbdbb 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -8604,36 +8604,47 @@ namespace oam writeLog("updateFstab called: " + device + ":" + dbrootID, LOG_TYPE_DEBUG ); //check if entry already exist - int user; - user = getuid(); + int user; + user = getuid(); + + string entry; + if (user == 0) + entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0"; + else + entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto,user 0 0"; string cmd; if (user == 0) cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; else - cmd = "sudo grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; + cmd = "sudo grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; int status = system(cmd.c_str()); - if (WEXITSTATUS(status) == 0 ) - return ""; + if (WEXITSTATUS(status) != 0 ) + { + //update /etc/fstab with mount + + //update local fstab + if (user == 0) + cmd = "echo " + entry + " >> /etc/fstab"; + else + cmd = "sudo echo " + entry + " >> /etc/fstab"; + + system(cmd.c_str()); + } - //update /etc/fstab with mount - string entry; if (user == 0) - entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0"; + cmd = "grep /data" + dbrootID + " " + InstallDir + "/local/etc/pm1/fstab > /dev/null 2>&1"; else - entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto,user 0 0"; + cmd = "sudo grep /data" + dbrootID + " " + InstallDir + "/local/etc/pm1/fstab > /dev/null 2>&1"; - //update local fstab - if (user == 0) - cmd = "echo " + entry + " >> /etc/fstab"; - else - cmd = "sudo echo " + entry + " >> /etc/fstab"; - system(cmd.c_str()); - - //use from addmodule later - cmd = "touch " + InstallDir + "/local/etc/pm1/fstab;echo " + entry + " >> " + InstallDir + "/local/etc/pm1/fstab"; - system(cmd.c_str()); + status = system(cmd.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + //use from addmodule later + cmd = "touch " + InstallDir + "/local/etc/pm1/fstab;echo " + entry + " >> " + InstallDir + "/local/etc/pm1/fstab"; + system(cmd.c_str()); + } return entry; } diff --git a/procmgr/main.cpp b/procmgr/main.cpp index e3d0ebf71..63f3b246e 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -569,7 +569,7 @@ static void startMgrProcessThread() { int status = API_SUCCESS; int k = 0; - for( ; k < 1200 ; k++ ) + for( ; k < 180 ; k++ ) { if ( startsystemthreadStop ) { processManager.setSystemState(oam::MAN_OFFLINE); @@ -608,7 +608,7 @@ static void startMgrProcessThread() sleep(1); } - if ( k == 1200 || status == API_FAILURE) { + if ( k == 180 || status == API_FAILURE) { // system didn't successfull restart processManager.setSystemState(oam::FAILED); // exit thread @@ -705,7 +705,7 @@ static void startMgrProcessThread() //now wait until all procmons are up and validate rpms on each module int status = API_SUCCESS; int k = 0; - for( ; k < 1200 ; k++ ) + for( ; k < 180 ; k++ ) { if ( startsystemthreadStop ) { processManager.setSystemState(oam::MAN_OFFLINE); @@ -822,7 +822,7 @@ static void startMgrProcessThread() } } - if ( k == 1200 || status == API_FAILURE) { + if ( k == 180 || status == API_FAILURE) { // system didn't successfull restart processManager.setSystemState(oam::FAILED); // exit thread diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 641600115..e1a97e958 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -4527,6 +4527,13 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str homedir = p; } + //clear out the known_host file, sometimes causes a failure on amazon during addModule + if ( amazon ) + { + string cmd = "sudo unlink " + homedir + ".ssh/know_hosts > /dev/null 2>&1"; + system(cmd.c_str()); + } + if ( packageType == "rpm") calpontPackage = homedir + "/mariadb-columnstore*" + systemsoftware.Version + "-" + systemsoftware.Release + "*.rpm.tar.gz"; else @@ -5061,8 +5068,6 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str //default string binaryInstallDir = installDir; - string installType = "initial"; - //run installer on remote module if ( remoteModuleType == "um" || ( remoteModuleType == "pm" && config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) || @@ -5071,7 +5076,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str if ( packageType != "binary" ) { log.writeLog(__LINE__, "addModule - user_installer run for " + remoteModuleName, LOG_TYPE_DEBUG); - string cmd = installDir + "/bin/user_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + " --nodeps none 1 > /tmp/user_installer.log"; + string cmd = installDir + "/bin/user_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + " --nodeps none 1 > /tmp/" + remoteModuleName + "_user_installer.log"; log.writeLog(__LINE__, "addModule cmd: " + cmd, LOG_TYPE_DEBUG); @@ -5116,7 +5121,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string binservertype = oam.itoa(config.ServerInstallType()); if ( PMwithUM == "y" ) binservertype = "pmwithum"; - string cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + calpontPackage + " " + remoteModuleType + " " + installType + AmazonInstall + " 1 " + binaryInstallDir + " > /tmp/binary_installer.log"; + string cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + calpontPackage + " initial " + AmazonInstall + " 1 " + binaryInstallDir + " > /tmp/" + remoteModuleName + "_binary_installer.log"; log.writeLog(__LINE__, "addModule - " + cmd, LOG_TYPE_DEBUG); @@ -5160,7 +5165,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str if ( remoteModuleType == "pm" ) { if ( packageType != "binary" ) { log.writeLog(__LINE__, "addModule - performance_installer run for " + remoteModuleName, LOG_TYPE_DEBUG); - string cmd = installDir + "/bin/performance_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + + " --nodeps 1 > /tmp/performance_installer.log"; + string cmd = installDir + "/bin/performance_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + + " --nodeps 1 > /tmp/" + remoteModuleName + "_performance_installer.log"; log.writeLog(__LINE__, "addModule cmd: " + cmd, LOG_TYPE_DEBUG); system(cmd.c_str()); @@ -5207,7 +5212,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str if ( PMwithUM == "y" ) binservertype = "pmwithum"; - string cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + calpontPackage + " " + installType + AmazonInstall + " 1 " + binaryInstallDir + " > /tmp/binary_installer.log"; + string cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + calpontPackage + " initial " + AmazonInstall + " 1 " + binaryInstallDir + " > /tmp/" + remoteModuleName + "_binary_installer.log"; log.writeLog(__LINE__, "addModule - " + cmd, LOG_TYPE_DEBUG); @@ -5320,10 +5325,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } //if amazon, delay to give time for ProcMon to start - if (amazon) { - log.writeLog(__LINE__, "addModule - sleep 30 - give ProcMon time to start on new Instance", LOG_TYPE_DEBUG); - sleep(30); - } +// if (amazon) { +// log.writeLog(__LINE__, "addModule - sleep 30 - give ProcMon time to start on new Instance", LOG_TYPE_DEBUG); +// sleep(30); +// } } else { @@ -5636,6 +5641,20 @@ int ProcessManager::removeModule(oam::DeviceNetworkList devicenetworklist, bool return API_FAILURE; } + //clear out the known_host file, sometimes causes a failure on amazon during addModule + if ( amazon ) + { + string homedir = "/root"; + if (!rootUser) { + char* p= getenv("HOME"); + if (p && *p) + homedir = p; + } + + string cmd = "sudo unlink " + homedir + ".ssh/know_hosts > /dev/null 2>&1"; + system(cmd.c_str()); + } + pthread_mutex_unlock(&THREAD_LOCK); //check if any removed modules was Standby OAM or Active OAM diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index fcd782cb3..08a74d2ed 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1729,17 +1729,28 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string entry; msg >> entry; - //check if entry already exist + //check if entry already exist in /etc/fstab string cmd = "grep " + entry + " /etc/fstab /dev/null 2>&1"; int status = system(cmd.c_str()); - if (WEXITSTATUS(status) == 0 ) - break; + if (WEXITSTATUS(status) != 0 ) + { + cmd = "echo " + entry + " >> /etc/fstab"; + system(cmd.c_str()); - cmd = "echo " + entry + " >> /etc/fstab"; - system(cmd.c_str()); - - log.writeLog(__LINE__, "Add line entry to /etc/fstab : " + entry); + log.writeLog(__LINE__, "Add line entry to /etc/fstab : " + entry); + } + + //check if entry already exist in ../local/etc/pm1/fstab + cmd = "grep " + entry + " " + startup::StartUp::installDir() + "/local/etc/pm1/fstab /dev/null 2>&1"; + status = system(cmd.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cmd = "echo " + entry + " >> " + startup::StartUp::installDir() + "/local/etc/pm1/fstab"; + system(cmd.c_str()); + log.writeLog(__LINE__, "Add line entry to ../local/etc/pm1/fstab : " + entry); + } + //mkdir on entry directory string::size_type pos = entry.find(" ",0); string::size_type pos1 = entry.find(" ",pos+1);