MCOL-3608 Add module installer file copy

Adds back some stuff that was remove distributed that didn't work
properly for non-distributed. Also use /var/lib/columnstore/local
for module file as OAM intended.

oam/install_scripts/CMakeLists.txt

@@ -11,6 +11,7 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/master-rep-columnstore.sh.in" "${CMA
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/disable-rep-columnstore.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/disable-rep-columnstore.sh" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mariadb-command-line.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-command-line.sh" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/slave-rep-columnstore.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/slave-rep-columnstore.sh" @ONLY)
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs_module_installer.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs_module_installer.sh" @ONLY)
 
 install(PROGRAMS columnstore-post-install 
                 columnstore-pre-uninstall 
@@ -32,6 +33,7 @@ install(PROGRAMS columnstore-post-install
                 slave-rep-columnstore.sh 
                 disable-rep-columnstore.sh  
                 mariadb-command-line.sh 
+                mcs_module_installer.sh
                 DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
 
 install(FILES columnstore.service
@@ -44,5 +46,5 @@ install(FILES columnstore.service
               myCnf-exclude-args.text
               DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT platform)
 
-install(FILES  module DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT platform)
+          install(FILES  module DESTINATION ${ENGINE_DATADIR}/local COMPONENT platform)
 

oam/install_scripts/columnstoreAlias

@@ -9,4 +9,4 @@ alias terror='tail -f /var/log/mariadb/columnstore/err.log'
 alias twarning='tail -f /var/log/mariadb/columnstore/warning.log'
 alias tcrit='tail -f /var/log/mariadb/columnstore/crit.log'
 alias dbrm='cd /var/lib/columnstore/data1/systemFiles/dbrm'
-alias mcsmodule='cat /etc/columnstore/module'
+alias mcsmodule='cat /var/lib/columnstore/local/module'

oam/install_scripts/mcs_module_installer.sh.in

@@ -0,0 +1,138 @@
+#!/usr/bin/expect
+#
+# Install custom OS files on system
+# Argument 0 - Remote Module Name
+# Argument 1 - Remote Server Host Name or IP address
+# Argument 2 - Root Password of remote server
+# Argument 3 - Debug flag 1 for on, 0 for off
+# Argument 4 - Username on remote server (root default)
+set timeout 30
+set USERNAME root
+set MODULE [lindex $argv 0]
+set SERVER [lindex $argv 1]
+set PASSWORD [lindex $argv 2]
+set DEBUG [lindex $argv 3]
+set USERNAME "root"
+set UNM [lindex $argv 4]
+if { $UNM != "" } {
+	set USERNAME $UNM
+}
+
+set HOME "$env(HOME)"
+
+log_user $DEBUG
+spawn -noecho /bin/bash
+#
+
+#check and see if remote server has ssh keys setup, set PASSWORD if so
+send_user " "
+send "ssh -v $USERNAME@$SERVER 'time'\n"
+set timeout 60
+expect {
+	"authenticity" { send "yes\n" 
+    exp_continue
+	}
+	"word: " { send "$PASSWORD\n"
+    exp_continue
+	}
+	"passphrase" { send "$PASSWORD\n" 
+    exp_continue
+	}
+	"Exit status 0" { send_user "DONE"}	
+	"Exit status 1" { send_user "FAILED: Login Failure\n" ; exit 1 }
+	"Host key verification failed" { send_user "FAILED: Host key verification failed\n" ; exit 1 }
+	"service not known" { send_user "FAILED: Invalid Host\n" ; exit 1 }
+	"Permission denied, please try again"   { send_user "ERROR: Invalid password\n" ; exit 1 }
+	"Connection refused"   { send_user "ERROR: Connection refused\n" ; exit 1 }
+	"Connection closed"   { send_user "ERROR: Connection closed\n" ; exit 1 }
+	"No route to host"   { send_user "ERROR: No route to host\n" ; exit 1 }
+	timeout { send_user "ERROR: Timeout to host\n" ; exit 2 }
+}
+send_user "\n"
+
+send_user "Stop ColumnStore service                       "
+send "ssh -v $USERNAME@$SERVER 'columnstore stop'\n"
+set timeout 60
+# check return
+expect {
+	"word: " { send "$PASSWORD\n"
+    exp_continue
+	}
+	"passphrase" { send "$PASSWORD\n" 
+    exp_continue
+	}
+#	"No such file or directory" { send_user "DONE" }
+        "Exit status 127" { send_user "DONE" }
+	 "Exit status 0" { send_user "DONE" }
+	"Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1}
+	timeout { send_user "DONE" }
+}
+send_user "\n"
+
+#
+# copy over custom OS tmp files
+#
+send_user "Copy Custom OS files to Module                  "
+send_user " \n"
+send "scp -rv @ENGINE_DATADIR@/local/etc $USERNAME@$SERVER:@ENGINE_DATADIR@/local\n"
+set timeout 120
+expect {
+	"word: " { send "$PASSWORD\n"
+    exp_continue
+	}
+	"passphrase" { send "$PASSWORD\n" 
+    exp_continue
+	}
+	"Exit status 0" { send_user "DONE" }
+	"scp :"  	{ send_user "ERROR\n" ; 
+				send_user "\n*** Installation ERROR\n" ; 
+				exit 1 }
+	"Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1}
+	timeout { send_user "ERROR: Timeout\n" ; exit 2 }
+}
+send_user "\n"
+
+#
+# copy over MariaDB Columnstore Module file
+#
+send_user "Copy MariaDB Columnstore Module file to Module                 "
+send "scp -v @ENGINE_DATADIR@/local/etc/$MODULE/*  $USERNAME@$SERVER:@ENGINE_DATADIR@/local/.\n"
+set timeout 120
+expect {
+	"word: " { send "$PASSWORD\n"
+    exp_continue
+	}
+	"passphrase" { send "$PASSWORD\n" 
+    exp_continue
+	}
+	"scp :"  	{ send_user "ERROR\n" ; 
+				send_user "\n*** Installation ERROR\n" ; 
+				exit 1 }
+	"Exit status 0" { send_user "DONE" }
+	"Exit status 1" { send_user "ERROR: scp failed" ; exit 1 }
+	timeout { send_user "ERROR: Timeout to host\n" ; exit 2 }
+}
+send_user "\n"
+
+send_user "Start ColumnStore service                       "
+send_user " \n"
+send "ssh -v $USERNAME@$SERVER 'columnstore restart'\n"
+set timeout 60
+# check return
+expect {
+	"word: " { send "$PASSWORD\n"
+    exp_continue
+	}
+	"passphrase" { send "$PASSWORD\n" 
+    exp_continue
+	}
+    "Exit status 0" { send_user "DONE" }
+    "Exit status 127" { send_user "ERROR: columnstore Not Found\n" ; exit 1 }
+    timeout { send_user "ERROR: Timeout to host\n" ; exit 2 }
+}
+
+send_user "\n"
+
+send_user "\nInstallation Successfully Completed on '$MODULE'\n"
+exit 0
+

oam/oamcpp/liboamcpp.cpp

@@ -2838,7 +2838,7 @@ oamModuleInfo_t Oam::getModuleInfo()
     int localModuleID;
 
     // Get Module Name from module-file
-    string fileName = "/etc/columnstore/module";
+    string fileName = "/var/lib/columnstore/local/module";
 
     ifstream oldFile (fileName.c_str());
 

oam/oamcpp/oamcache.cpp

@@ -284,7 +284,7 @@ int OamCache::getLocalPMId()
 {
     mutex::scoped_lock lk(cacheLock);
 
-    // This comes from the file /etc/columnstore/module, not from the xml.
+    // This comes from the file /var/lib/columnstore/local/module, not from the xml.
     // Thus, it's not refreshed during checkReload().
     if (mLocalPMId > 0)
     {
@@ -293,7 +293,7 @@ int OamCache::getLocalPMId()
 
     string localModule;
     string moduleType;
-    string fileName = "/etc/columnstore/module";
+    string fileName = "/var/lib/columnstore/local/module";
     ifstream moduleFile (fileName.c_str());
     char line[400];
 
@@ -337,7 +337,7 @@ string OamCache::getModuleName()
     if (!moduleName.empty())
         return moduleName;
 
-    string fileName = "/etc/columnstore/module";
+    string fileName = "/var/lib/columnstore/local/module";
     ifstream moduleFile(fileName.c_str());
     getline(moduleFile, moduleName);
     moduleFile.close();

oam/post/columnstore_functions

@@ -10,7 +10,7 @@ fi
 module_type()
 {
 (
-	sed -r 's/[0-9]+$//' /etc/columnstore/module
+	sed -r 's/[0-9]+$//' /var/lib/columnstore/local/module
 ) 2>/dev/null
 }
 
@@ -30,13 +30,13 @@ oidbitmapfile()
 module_name()
 {
 (
-	cat /etc/columnstore/module
+	cat /var/lib/columnstore/local/module
 ) 2>/dev/null
 }
 
 module_id()
 {
 (
-	sed -r 's/[a-zA-Z]//g' /etc/columnstore/module
+	sed -r 's/[a-zA-Z]//g' /var/lib/columnstore/local/module
 ) 2>/dev/null
 }

oamapps/postConfigure/postConfigure.cpp

@@ -3570,19 +3570,45 @@ int main(int argc, char* argv[])
 			}
     }
 
-    int thread_id = 0;
-
-    pthread_t thr[childmodulelist.size()];
-
-    /* create a thread_data_t argument array */
-    thread_data_t thr_data[childmodulelist.size()];
-
     string install = "y";
 
     if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM ||
             pmNumber > 1 )
     {
 
+        ChildModuleList::iterator list1 = childmodulelist.begin();
+
+        for (; list1 != childmodulelist.end() ; list1++)
+        {
+            string remoteModuleName = (*list1).moduleName;
+            string remoteModuleIP = (*list1).moduleIP;
+            string remoteHostName = (*list1).hostName;
+            string remoteModuleType = remoteModuleName.substr(0, MAX_MODULE_TYPE_SIZE);
+
+            string debug_logfile;
+            string logfile;
+
+            if ( remote_installer_debug == "1" )
+            {
+                logfile = tmpDir + "/";
+                logfile += remoteModuleName + "_" + EEPackageType + "_install.log";
+                debug_logfile = " > " + logfile;
+            }
+
+            cout << endl << "----- Performing Install on '" + remoteModuleName + " / " + remoteHostName + "' -----" << endl << endl;
+            if ( remote_installer_debug == "1" )
+                cout << "Install log file is located here: " + logfile << endl << endl;
+
+            cmd = "mcs_module_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + remote_installer_debug + " " + debug_logfile;
+
+            int rtnCode = system(cmd.c_str());
+            if (WEXITSTATUS(rtnCode) != 0)
+            {
+                cout << endl << "Error returned from mcs_module_installer.sh" << endl;
+                exit(1);
+            }
+        }
+
         //configure data redundancy
         if (DataRedundancy)
         {

procmgr/processmanager.cpp

@@ -5462,11 +5462,95 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str
 
     listPT = devicenetworklist.begin();
 
+    //distribute config file
+    distributeConfigFile("system");
+    distributeConfigFile("system", "ProcessConfig.xml");
+
     for ( ; listPT != devicenetworklist.end() ; listPT++)
     {
-        string moduleName = (*listPT).DeviceName;
+        string remoteModuleName = (*listPT).DeviceName;
+        string remoteModuleType = remoteModuleName.substr(0, MAX_MODULE_TYPE_SIZE);
+        HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin();
+        string remoteModuleIP = (*pt1).IPAddr;
+        string remoteHostName = (*pt1).HostName;
 
-        processManager.configureModule(moduleName);
+        string dir = "/var/lib/columnstore/local/etc" + remoteModuleName;
+
+        cmd = "mkdir " + dir + " > /dev/null 2>&1";
+        system(cmd.c_str());
+
+        if ( remoteModuleType == "um" )
+        {
+            cmd = "cp /var/lib/columnstore/local/etc/um1/* " + dir + "/.";
+            system(cmd.c_str());
+        }
+        else if ( remoteModuleType == "pm" )
+        {
+            cmd = "cp /var/lib/columnstore/local/etc/pm1/* " + dir + "/.";
+            system(cmd.c_str());
+        }
+
+        log.writeLog(__LINE__, "addModule - created directory and custom OS files for " +  remoteModuleName, LOG_TYPE_DEBUG);
+
+        //create module file
+        if ( !createModuleFile(remoteModuleName) )
+        {
+            log.writeLog(__LINE__, "addModule - ERROR: createModuleFile failed", LOG_TYPE_ERROR);
+            pthread_mutex_unlock(&THREAD_LOCK);
+            return API_FAILURE;
+        }
+
+        log.writeLog(__LINE__, "addModule - create module file for " +  remoteModuleName, LOG_TYPE_DEBUG);
+
+        if ( remoteModuleType == "pm" )
+        {
+            //setup Standby OAM Parent, if needed
+            if ( config.OAMStandbyName() == oam::UnassignedName )
+                setStandbyModule(remoteModuleName, false);
+        }
+
+        string logFile = tmpLogDir + "/" + remoteModuleName + "_mcs_module_installer.log";
+        log.writeLog(__LINE__, "addModule - mcs_module_installer run for " +  remoteModuleName, LOG_TYPE_DEBUG);
+        cmd = "mcs_module_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " 1 >" + logFile;
+        log.writeLog(__LINE__, "addModule cmd: " + cmd, LOG_TYPE_DEBUG);
+        int rtnCode = system(cmd.c_str());
+
+        if (WEXITSTATUS(rtnCode) != 0)
+        {
+            log.writeLog(__LINE__, "addModule - ERROR: " + logFile + " failed, retry", LOG_TYPE_DEBUG);
+
+            DeviceNetworkList devicenetworklistR;
+            DeviceNetworkConfig devicenetworkconfigR;
+            HostConfig hostconfig;
+
+            devicenetworkconfigR.DeviceName = remoteModuleName;
+
+            hostconfig.IPAddr = oam::UnassignedName;
+
+            hostconfig.HostName = oam::UnassignedName;
+            hostconfig.NicID = 1;
+            devicenetworkconfigR.hostConfigList.push_back(hostconfig);
+
+            devicenetworklistR.push_back(devicenetworkconfigR);
+
+            processManager.removeModule(devicenetworklistR, false);
+
+            log.writeLog(__LINE__, "addModule - Remove Module Completed", LOG_TYPE_DEBUG);
+
+            pthread_mutex_unlock(&THREAD_LOCK);
+            cmd = "/bin/cp -f " + logFile + " " + logFile + "failed";
+            system(cmd.c_str());
+            processManager.setModuleState(remoteModuleName, oam::FAILED);
+            return API_FAILURE;
+        }
+        if (manualFlag)
+            //set new module to disable state if manual add
+            disableModule(remoteModuleName, true);
+
+        // add to monitor list
+        moduleInfoList.insert(moduleList::value_type(remoteModuleName, 0));
+
+        processManager.configureModule(remoteModuleName);
     }
 
     //delay to give time for ProcMon to start after the config is sent and procmon restarts