database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-3608 Add module installer file copy

Browse Source 
Adds back some stuff that was remove distributed that didn't work
properly for non-distributed. Also use /var/lib/columnstore/local
for module file as OAM intended.
This commit is contained in:
Andrew Hutchings
2019-11-14 13:02:26 +00:00
parent 0621762378
commit 25d22381c6
8 changed files with 268 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
oam/install_scripts/CMakeLists.txt
View File

@ -11,6 +11,7 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/master-rep-columnstore.sh.in" "${CMA
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/disable-rep-columnstore.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/disable-rep-columnstore.sh" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mariadb-command-line.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-command-line.sh" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/slave-rep-columnstore.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/slave-rep-columnstore.sh" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs_module_installer.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs_module_installer.sh" @ONLY)
install(PROGRAMS columnstore-post-install
columnstore-pre-uninstall
@ -32,6 +33,7 @@ install(PROGRAMS columnstore-post-install
slave-rep-columnstore.sh
disable-rep-columnstore.sh
mariadb-command-line.sh
mcs_module_installer.sh
DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
install(FILES columnstore.service
@ -44,5 +46,5 @@ install(FILES columnstore.service
myCnf-exclude-args.text
DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT platform)
install(FILES module DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT platform)
install(FILES module DESTINATION ${ENGINE_DATADIR}/local COMPONENT platform)

2
oam/install_scripts/columnstoreAlias
View File

@ -9,4 +9,4 @@ alias terror='tail -f /var/log/mariadb/columnstore/err.log'
alias twarning='tail -f /var/log/mariadb/columnstore/warning.log'
alias tcrit='tail -f /var/log/mariadb/columnstore/crit.log'
alias dbrm='cd /var/lib/columnstore/data1/systemFiles/dbrm'
alias mcsmodule='cat /etc/columnstore/module'
alias mcsmodule='cat /var/lib/columnstore/local/module'

138
oam/install_scripts/mcs_module_installer.sh.in Normal file
View File

@ -0,0 +1,138 @@
#!/usr/bin/expect
#
# Install custom OS files on system
# Argument 0 - Remote Module Name
# Argument 1 - Remote Server Host Name or IP address
# Argument 2 - Root Password of remote server
# Argument 3 - Debug flag 1 for on, 0 for off
# Argument 4 - Username on remote server (root default)
set timeout 30
set USERNAME root
set MODULE [lindex $argv 0]
set SERVER [lindex $argv 1]
set PASSWORD [lindex $argv 2]
set DEBUG [lindex $argv 3]
set USERNAME "root"
set UNM [lindex $argv 4]
if { $UNM != "" } {
set USERNAME $UNM
}
set HOME "$env(HOME)"
log_user $DEBUG
spawn -noecho /bin/bash
#
#check and see if remote server has ssh keys setup, set PASSWORD if so
send_user " "
send "ssh -v $USERNAME@$SERVER 'time'\n"
set timeout 60
expect {
"authenticity" { send "yes\n"
exp_continue
}
"word: " { send "$PASSWORD\n"
exp_continue
}
"passphrase" { send "$PASSWORD\n"
exp_continue
}
"Exit status 0" { send_user "DONE"}
"Exit status 1" { send_user "FAILED: Login Failure\n" ; exit 1 }
"Host key verification failed" { send_user "FAILED: Host key verification failed\n" ; exit 1 }
"service not known" { send_user "FAILED: Invalid Host\n" ; exit 1 }
"Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 }
"Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 }
"Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 }
"No route to host" { send_user "ERROR: No route to host\n" ; exit 1 }
timeout { send_user "ERROR: Timeout to host\n" ; exit 2 }
}
send_user "\n"
send_user "Stop ColumnStore service "
send "ssh -v $USERNAME@$SERVER 'columnstore stop'\n"
set timeout 60
# check return
expect {
"word: " { send "$PASSWORD\n"
exp_continue
}
"passphrase" { send "$PASSWORD\n"
exp_continue
}
# "No such file or directory" { send_user "DONE" }
"Exit status 127" { send_user "DONE" }
"Exit status 0" { send_user "DONE" }
"Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1}
timeout { send_user "DONE" }
}
send_user "\n"
#
# copy over custom OS tmp files
#
send_user "Copy Custom OS files to Module "
send_user " \n"
send "scp -rv @ENGINE_DATADIR@/local/etc $USERNAME@$SERVER:@ENGINE_DATADIR@/local\n"
set timeout 120
expect {
"word: " { send "$PASSWORD\n"
exp_continue
}
"passphrase" { send "$PASSWORD\n"
exp_continue
}
"Exit status 0" { send_user "DONE" }
"scp :" { send_user "ERROR\n" ;
send_user "\n*** Installation ERROR\n" ;
exit 1 }
"Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1}
timeout { send_user "ERROR: Timeout\n" ; exit 2 }
}
send_user "\n"
#
# copy over MariaDB Columnstore Module file
#
send_user "Copy MariaDB Columnstore Module file to Module "
send "scp -v @ENGINE_DATADIR@/local/etc/$MODULE/* $USERNAME@$SERVER:@ENGINE_DATADIR@/local/.\n"
set timeout 120
expect {
"word: " { send "$PASSWORD\n"
exp_continue
}
"passphrase" { send "$PASSWORD\n"
exp_continue
}
"scp :" { send_user "ERROR\n" ;
send_user "\n*** Installation ERROR\n" ;
exit 1 }
"Exit status 0" { send_user "DONE" }
"Exit status 1" { send_user "ERROR: scp failed" ; exit 1 }
timeout { send_user "ERROR: Timeout to host\n" ; exit 2 }
}
send_user "\n"
send_user "Start ColumnStore service "
send_user " \n"
send "ssh -v $USERNAME@$SERVER 'columnstore restart'\n"
set timeout 60
# check return
expect {
"word: " { send "$PASSWORD\n"
exp_continue
}
"passphrase" { send "$PASSWORD\n"
exp_continue
}
"Exit status 0" { send_user "DONE" }
"Exit status 127" { send_user "ERROR: columnstore Not Found\n" ; exit 1 }
timeout { send_user "ERROR: Timeout to host\n" ; exit 2 }
}
send_user "\n"
send_user "\nInstallation Successfully Completed on '$MODULE'\n"
exit 0

2
oam/oamcpp/liboamcpp.cpp
View File

@ -2838,7 +2838,7 @@ oamModuleInfo_t Oam::getModuleInfo()
int localModuleID;
// Get Module Name from module-file
string fileName = "/etc/columnstore/module";
string fileName = "/var/lib/columnstore/local/module";
ifstream oldFile (fileName.c_str());

6
oam/oamcpp/oamcache.cpp
View File

@ -284,7 +284,7 @@ int OamCache::getLocalPMId()
{
mutex::scoped_lock lk(cacheLock);
// This comes from the file /etc/columnstore/module, not from the xml.
// This comes from the file /var/lib/columnstore/local/module, not from the xml.
// Thus, it's not refreshed during checkReload().
if (mLocalPMId > 0)
{
@ -293,7 +293,7 @@ int OamCache::getLocalPMId()
string localModule;
string moduleType;
string fileName = "/etc/columnstore/module";
string fileName = "/var/lib/columnstore/local/module";
ifstream moduleFile (fileName.c_str());
char line[400];
@ -337,7 +337,7 @@ string OamCache::getModuleName()
if (!moduleName.empty())
return moduleName;
string fileName = "/etc/columnstore/module";
string fileName = "/var/lib/columnstore/local/module";
ifstream moduleFile(fileName.c_str());
getline(moduleFile, moduleName);
moduleFile.close();

6
oam/post/columnstore_functions
View File

@ -10,7 +10,7 @@ fi
module_type()
{
(
sed -r 's/[0-9]+$//' /etc/columnstore/module
sed -r 's/[0-9]+$//' /var/lib/columnstore/local/module
) 2>/dev/null
}
@ -30,13 +30,13 @@ oidbitmapfile()
module_name()
{
(
cat /etc/columnstore/module
cat /var/lib/columnstore/local/module
) 2>/dev/null
}
module_id()
{
(
sed -r 's/[a-zA-Z]//g' /etc/columnstore/module
sed -r 's/[a-zA-Z]//g' /var/lib/columnstore/local/module
) 2>/dev/null
}

40
oamapps/postConfigure/postConfigure.cpp
View File

@ -3570,19 +3570,45 @@ int main(int argc, char* argv[])
}
}
int thread_id = 0;
pthread_t thr[childmodulelist.size()];
/* create a thread_data_t argument array */
thread_data_t thr_data[childmodulelist.size()];
string install = "y";
if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM ||
pmNumber > 1 )
{
ChildModuleList::iterator list1 = childmodulelist.begin();
for (; list1 != childmodulelist.end() ; list1++)
{
string remoteModuleName = (*list1).moduleName;
string remoteModuleIP = (*list1).moduleIP;
string remoteHostName = (*list1).hostName;
string remoteModuleType = remoteModuleName.substr(0, MAX_MODULE_TYPE_SIZE);
string debug_logfile;
string logfile;
if ( remote_installer_debug == "1" )
{
logfile = tmpDir + "/";
logfile += remoteModuleName + "_" + EEPackageType + "_install.log";
debug_logfile = " > " + logfile;
}
cout << endl << "----- Performing Install on '" + remoteModuleName + " / " + remoteHostName + "' -----" << endl << endl;
if ( remote_installer_debug == "1" )
cout << "Install log file is located here: " + logfile << endl << endl;
cmd = "mcs_module_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + remote_installer_debug + " " + debug_logfile;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "Error returned from mcs_module_installer.sh" << endl;
exit(1);
}
}
//configure data redundancy
if (DataRedundancy)
{

88
procmgr/processmanager.cpp
View File

@ -5462,11 +5462,95 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str
listPT = devicenetworklist.begin();
//distribute config file
distributeConfigFile("system");
distributeConfigFile("system", "ProcessConfig.xml");
for ( ; listPT != devicenetworklist.end() ; listPT++)
{
string moduleName = (*listPT).DeviceName;
string remoteModuleName = (*listPT).DeviceName;
string remoteModuleType = remoteModuleName.substr(0, MAX_MODULE_TYPE_SIZE);
HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin();
string remoteModuleIP = (*pt1).IPAddr;
string remoteHostName = (*pt1).HostName;
processManager.configureModule(moduleName);
string dir = "/var/lib/columnstore/local/etc" + remoteModuleName;
cmd = "mkdir " + dir + " > /dev/null 2>&1";
system(cmd.c_str());
if ( remoteModuleType == "um" )
{
cmd = "cp /var/lib/columnstore/local/etc/um1/* " + dir + "/.";
system(cmd.c_str());
}
else if ( remoteModuleType == "pm" )
{
cmd = "cp /var/lib/columnstore/local/etc/pm1/* " + dir + "/.";
system(cmd.c_str());
}
log.writeLog(__LINE__, "addModule - created directory and custom OS files for " + remoteModuleName, LOG_TYPE_DEBUG);
//create module file
if ( !createModuleFile(remoteModuleName) )
{
log.writeLog(__LINE__, "addModule - ERROR: createModuleFile failed", LOG_TYPE_ERROR);
pthread_mutex_unlock(&THREAD_LOCK);
return API_FAILURE;
}
log.writeLog(__LINE__, "addModule - create module file for " + remoteModuleName, LOG_TYPE_DEBUG);
if ( remoteModuleType == "pm" )
{
//setup Standby OAM Parent, if needed
if ( config.OAMStandbyName() == oam::UnassignedName )
setStandbyModule(remoteModuleName, false);
}
string logFile = tmpLogDir + "/" + remoteModuleName + "_mcs_module_installer.log";
log.writeLog(__LINE__, "addModule - mcs_module_installer run for " + remoteModuleName, LOG_TYPE_DEBUG);
cmd = "mcs_module_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " 1 >" + logFile;
log.writeLog(__LINE__, "addModule cmd: " + cmd, LOG_TYPE_DEBUG);
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
log.writeLog(__LINE__, "addModule - ERROR: " + logFile + " failed, retry", LOG_TYPE_DEBUG);
DeviceNetworkList devicenetworklistR;
DeviceNetworkConfig devicenetworkconfigR;
HostConfig hostconfig;
devicenetworkconfigR.DeviceName = remoteModuleName;
hostconfig.IPAddr = oam::UnassignedName;
hostconfig.HostName = oam::UnassignedName;
hostconfig.NicID = 1;
devicenetworkconfigR.hostConfigList.push_back(hostconfig);
devicenetworklistR.push_back(devicenetworkconfigR);
processManager.removeModule(devicenetworklistR, false);
log.writeLog(__LINE__, "addModule - Remove Module Completed", LOG_TYPE_DEBUG);
pthread_mutex_unlock(&THREAD_LOCK);
cmd = "/bin/cp -f " + logFile + " " + logFile + "failed";
system(cmd.c_str());
processManager.setModuleState(remoteModuleName, oam::FAILED);
return API_FAILURE;
}
if (manualFlag)
//set new module to disable state if manual add
disableModule(remoteModuleName, true);
// add to monitor list
moduleInfoList.insert(moduleList::value_type(remoteModuleName, 0));
processManager.configureModule(remoteModuleName);
}
//delay to give time for ProcMon to start after the config is sent and procmon restarts