database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-12-24 14:20:59 +03:00
Code Activity

MCOL-770 - changes for amazon install

Browse Source
This commit is contained in:
david hill
2017-06-28 16:03:34 -05:00
parent 679ca67e4c
commit d9a1e1204d
9 changed files with 168 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
oam/install_scripts/binary_installer.sh
View File

@@ -62,7 +62,12 @@ if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } {
"word: " { send "$PASSWORD\n" }
"passphrase" { send "$PASSWORD\n" }
}
}
expect {
"Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1}
"MariaDB Columnstore uninstall completed" { send_user "DONE" }
"Exit status 0" { send_user "DONE" }
}
}
"word: " { send "$PASSWORD\n" }
"passphrase" { send "$PASSWORD\n" }
"Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 }
@@ -70,12 +75,6 @@ if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } {
"MariaDB Columnstore uninstall completed" { send_user "DONE" }
"Exit status 0" { send_user "DONE" }
}
set timeout 30
expect {
"Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1}
"MariaDB Columnstore uninstall completed" { send_user "DONE" }
"Exit status 0" { send_user "DONE" }
}
send_user "\n"
}
@@ -151,15 +150,14 @@ expect {
set timeout 60
# check return
expect {
"Exit status 0" { send_user "DONE" }
"No such file" { send_user "ERROR: post-install Not Found\n" ; exit 1 }
"MariaDB Columnstore syslog logging not working" { send_user "ERROR: MariaDB Columnstore System logging not setup\n" ; exit 1 }
"MariaDB Columnstore syslog logging not working" { send_user "ERROR: MariaDB Columnstore System logging not setup\n" ; abort }
"Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 }
"Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1}
"Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 }
"Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 }
"No route to host" { send_user "ERROR: No route to host\n" ; exit 1 }
"postConfigure" { send_user "DONE" }
"columnstore start" { send_user "DONE" }
}
send_user "\n"
@@ -194,7 +192,7 @@ if { $AMAZONINSTALL == "1" } {
send_user "Copy MariaDB Columnstore OS files to Module "
send " \n"
send date\n
send "scp -v -r $INSTALLDIR/local/etc/* $USERNAME@$SERVER:$INSTALLDIR/local\n"
send "scp -v -r $INSTALLDIR/local/etc $USERNAME@$SERVER:$INSTALLDIR/local\n"
set timeout 10
expect {
"word: " { send "$PASSWORD\n" }

12
oam/install_scripts/module_installer.sh
View File

@@ -52,9 +52,17 @@ else
export COLUMNSTORE_INSTALL_DIR=$installdir
fi
PMwithUM=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation PMwithUM`
ServerTypeInstall=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation ServerTypeInstall`
cloud=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation Cloud`
if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then
cp $COLUMNSTORE_INSTALL_DIR/local/etc/credentials $HOME/.aws/. > /dev/null 2>&1
if [ $user = "root" ]; then
sed -i -e s/#runuser/runuser/g /etc/rc.local
else
sudo sed -i -e s/#runuser/runuser/g /etc/rc.local
fi
if [ $module = "pm" ]; then
if test -f $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab ; then
@@ -66,6 +74,7 @@ if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then
cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab
else
sudo touch /etc/fstab
sudo chmod 666 /etc/fstab
sudo rm -f /etc/fstab.columnstoreSave
sudo cp /etc/fstab /etc/fstab.columnstoreSave
sudo cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab
@@ -129,12 +138,13 @@ if [ $MySQLRep = "y" ]; then
fi
if test -f $COLUMNSTORE_INSTALL_DIR/mysql/my.cnf ; then
mysqlPort=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation MySQLPort`
echo "Run Mysql Port update on my.cnf on Module"
$COLUMNSTORE_INSTALL_DIR/bin/mycnfUpgrade $mysqlPort > /tmp/mycnfUpgrade_port.log 2>&1
fi
# if um, run mysql install scripts
if [ $module = "um" ]; then
if [ $module = "um" ] || ( [ $module = "pm" ] && [ PMwithUM = "y" ] ) || [ $ServerTypeInstall = "2" ]; then
echo "Run post-mysqld-install"
$COLUMNSTORE_INSTALL_DIR/bin/post-mysqld-install --installdir=$COLUMNSTORE_INSTALL_DIR > /tmp/post-mysqld-install.log 2>&1
if [ $? -ne 0 ]; then

2
oam/install_scripts/performance_installer.sh
View File

@@ -214,7 +214,7 @@ if { $AMAZONINSTALL == "1" } {
send_user "Copy MariaDB Columnstore OS files to Module "
send " \n"
send date\n
send "scp -v -r $INSTALLDIR/local/etc/* $USERNAME@$SERVER:$INSTALLDIR/local\n"
send "scp -v -r $INSTALLDIR/local/etc $USERNAME@$SERVER:$INSTALLDIR/local\n"
set timeout 10
expect {
"word: " { send "$PASSWORD\n" }

70
oam/install_scripts/post-install
View File

@@ -277,18 +277,33 @@ if [ -z "$hadoop" ]; then
cat <<EOD
The next steps are:
If installing on a pm1 node:
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
$installdir/bin/postConfigure -i $installdir
If installing on a non-pm1 using the non-distrubuted option:
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
$installdir/bin/columnstore start
EOD
# else
#cat <<EOD
#The next step is:
else
cat <<EOD
The next step is:
#$installdir/bin/postConfigure
If installing on a pm1 node:
#EOD
$installdir/bin/postConfigure
If installing on a non-pm1 using the non-distrubuted option:
$installdir/bin/columnstore start
EOD
fi
exit 0
else
@@ -308,17 +323,33 @@ cat <<EOD
If you are intending to install MariaDB Columnstore over Hadoop, the next steps are:
If installing on a pm1 node:
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
. $installdir/bin/setenv-hdfs-20
$installdir/bin/postConfigure -i $installdir
If installing on a non-pm1 using the non-distrubuted option:
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
$installdir/bin/columnstore start
If you are intending to install MariaDB Columnstore without Hadoop, the next steps are:
If installing on a pm1 node:
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
$installdir/bin/postConfigure -i $installdir
If installing on a non-pm1 using the non-distrubuted option:
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
$installdir/bin/columnstore start
EOD
else
cat <<EOD
@@ -328,10 +359,21 @@ If you are intending to install MariaDB Columnstore over Hadoop, the next steps
. $installdir/bin/setenv-hdfs-20
$installdir/bin/postConfigure
If installing on a non-pm1 using the non-distrubuted option:
. $installdir/bin/setenv-hdfs-20
$installdir/bin/columnstore start
If you are intending to install MariaDB Columnstore without Hadoop, the next step is:
If installing on a pm1 node:
$installdir/bin/postConfigure
If installing on a non-pm1 using the non-distrubuted option:
$installdir/bin/columnstore start
EOD
fi
else
@@ -347,6 +389,8 @@ cat <<EOD
If you are intending to install MariaDB Columnstore over Hadoop, the next steps are:
If installing on a pm1 node:
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib:$libpath
. $installdir/bin/setenv-hdfs-12
@@ -354,8 +398,11 @@ $installdir/bin/postConfigure -i $installdir
If you are intending to install MariaDB Columnstore without Hadoop, the next steps are:
If installing on a pm1 node:
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib
. $installdir/bin/setenv-hdfs-12
$installdir/bin/postConfigure -i $installdir
EOD
@@ -364,13 +411,26 @@ cat <<EOD
If you are intending to install MariaDB Columnstore over Hadoop, the next steps are:
If installing on a pm1 node:
. $installdir/bin/setenv-hdfs-12
$installdir/bin/postConfigure
If installing on a non-pm1 using the non-distrubuted option:
. $installdir/bin/setenv-hdfs-12
$installdir/bin/columnstore start
If you are intending to install MariaDB Columnstore without Hadoop, the next step is:
If installing on a pm1 node:
$installdir/bin/postConfigure
If installing on a non-pm1 using the non-distrubuted option:
$installdir/bin/columnstore start
EOD
fi
else

2
oam/install_scripts/user_installer.sh
View File

@@ -214,7 +214,7 @@ if { $AMAZONINSTALL == "1" } {
send_user "Copy MariaDB Columnstore OS files to Module "
send " \n"
send date\n
send "scp -v -r $INSTALLDIR/local/etc/* $USERNAME@$SERVER:$INSTALLDIR/local\n"
send "scp -v -r $INSTALLDIR/local/etc $USERNAME@$SERVER:$INSTALLDIR/local\n"
set timeout 10
expect {
"word: " { send "$PASSWORD\n" }

17
oamapps/postConfigure/installer.cpp
View File

@@ -824,7 +824,6 @@ int main(int argc, char *argv[])
// /etc OS Files to be updated
string files[] = {
"rc.local",
" "
};
@@ -840,12 +839,6 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall)
//end of list
break;
//create or update date on file to make sure on exist
if ( files[i] == "rc.local") {
cmd = "touch " + installDir + "/local/etc/" + parentOAMModuleName + "/rc.local.calpont";
system(cmd.c_str());
}
string fileName = "/etc/" + files[i];
//make a backup copy before changing
@@ -1012,7 +1005,6 @@ bool updateProcessConfig(int serverTypeInstall)
bool makeRClocal(string moduleName, int IserverTypeInstall)
{
string cmd;
string fileName = installDir + "/local/etc/" + moduleName + "/rc.local.calpont";
string moduleType = moduleName.substr(0,MAX_MODULE_TYPE_SIZE);
@@ -1088,13 +1080,10 @@ bool makeRClocal(string moduleName, int IserverTypeInstall)
}
}
unlink (fileName.c_str());
if ( lines.begin() == lines.end()) {
cmd = "touch " + fileName + " >/dev/null 2>&1";
system(cmd.c_str());
if ( lines.begin() == lines.end())
return true;
}
string fileName = "etc/rc.local";
ofstream newFile (fileName.c_str());

59
oamapps/postConfigure/postConfigure.cpp
View File

@@ -974,7 +974,7 @@ int main(int argc, char *argv[])
}
catch(...)
{
cloud = oam::UnassignedName;
cloud = oam::UnassignedName;
}
if ( cloud == "disable" )
@@ -992,20 +992,20 @@ int main(int argc, char *argv[])
pcommand = callReadline(prompt.c_str());
if (pcommand) {
if (strlen(pcommand) > 0) enable = pcommand;
callFree(pcommand);
if (enable == "n") {
amazonInstall = false;
if (strlen(pcommand) > 0) enable = pcommand;
callFree(pcommand);
if (enable == "n") {
amazonInstall = false;
try {
sysConfig->setConfig(InstallSection, "Cloud", "disable");
}
catch(...)
{};
try {
sysConfig->setConfig(InstallSection, "Cloud", "disable");
}
catch(...)
{};
break;
}
break;
}
}
if ( enable != "y" )
@@ -1032,11 +1032,11 @@ int main(int argc, char *argv[])
if ( amazonInstall )
{
string cmd = installDir + "/bin/MCSgetCredentials.sh >/dev/null 2>&1";
int rtnCode = system(cmd.c_str());
if ( WEXITSTATUS(rtnCode) != 0 ) {
cout << endl << "Error: No IAM Profile with Security Certificates used or AWS CLI Certificate file configured" << endl;
cout << "Check Amazon Install Documenation for additional information, exiting..." << endl;
exit (1);
int rtnCode = system(cmd.c_str());
if ( WEXITSTATUS(rtnCode) != 0 ) {
cout << endl << "Error: No IAM Profile with Security Certificates used or AWS CLI Certificate file configured" << endl;
cout << "Check Amazon Install Documenation for additional information, exiting..." << endl;
exit (1);
}
}
@@ -1044,6 +1044,9 @@ int main(int argc, char *argv[])
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
// setup to start on reboot
system("sudo sed -i -e s/#runuser/runuser/g /etc/rc.local >/dev/null 2>&1");
}
if ( pmwithum )
@@ -3707,7 +3710,6 @@ bool checkSaveConfigFile()
// /etc OS Files to be updated
string files[] = {
"rc.local",
" "
};
@@ -3722,14 +3724,6 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall)
//end of list
break;
//create or update date on file to make sure on exist
if ( files[i] == "rc.local") {
string cmd = "touch " + installDir + "/local/etc/" + parentOAMModuleName + "/rc.local.calpont > /dev/null 2>&1";
if ( !rootUser )
cmd = "sudo touch " + installDir + "/local/etc/" + parentOAMModuleName + "/rc.local.calpont > /dev/null 2>&1";
system(cmd.c_str());
}
string fileName = "/etc/" + files[i];
//make a backup copy before changing
@@ -3934,8 +3928,6 @@ bool uncommentCalpontXml( string entry)
*/
bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall)
{
string fileName = installDir + "/local/etc/" + moduleName + "/rc.local.calpont";
vector <string> lines;
string mount1;
@@ -3987,13 +3979,10 @@ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall)
}
}
unlink (fileName.c_str());
if ( lines.begin() == lines.end()) {
string cmd = "touch " + fileName;
system(cmd.c_str());
if ( lines.begin() == lines.end())
return true;
}
string fileName = "/etc/rc.local";
ofstream newFile (fileName.c_str());

3
procmgr/processmanager.cpp
View File

@@ -6917,6 +6917,9 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist)
processManager.setSystemState(rtn);
}
//run command to build system table if they don't already exist
processManager.buildSystemTables("pm1");
// exit thread
log.writeLog(__LINE__, "startSystemThread Exit", LOG_TYPE_DEBUG);
startsystemthreadStatus = status;

113
procmon/main.cpp
View File

@@ -97,7 +97,6 @@ int main(int argc, char **argv)
MonitorLog log;
MonitorConfig config;
ProcessMonitor aMonitor(config, log);
Config* sysConfig = Config::makeConfig();
log.writeLog(__LINE__, " ");
log.writeLog(__LINE__, "**********Process Monitor Started**********");
@@ -142,6 +141,7 @@ int main(int argc, char **argv)
//check if this is a fresh install, meaning the Columnstore.xml file is not setup
//if so, wait for messages from Procmgr to start us up
Config* sysConfig = Config::makeConfig();
string exemgrIpadd = sysConfig->getConfig("ExeMgr1", "IPAddr");
if ( exemgrIpadd == "0.0.0.0" )
{
@@ -155,7 +155,7 @@ int main(int argc, char **argv)
count++;
if (count > 10 ) {
count = 0;
log.writeLog(__LINE__, "Waiting for ProcMgr to start us up", LOG_TYPE_DEBUG);
log.writeLog(__LINE__, "Waiting for ProcMgr to start up", LOG_TYPE_DEBUG);
}
sleep(1);
}
@@ -174,21 +174,9 @@ int main(int argc, char **argv)
}
string modType = config.moduleType();
if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) ||
( PMwithUM == "y") )
modType = "um";
string MySQLPort = "3306";
//MariaDB port
try {
oam.getSystemConfig( "MySQLPort", MySQLPort);
}
catch(...) {
MySQLPort = "3306";
}
//run the module install script
string cmd = startup::StartUp::installDir() + "/bin/module_installer.sh " + " --installdir=" + startup::StartUp::installDir() + " --module=" + modType + " --port=" + MySQLPort + " > /tmp/module_installer.log 2>&1";
string cmd = startup::StartUp::installDir() + "/bin/module_installer.sh " + " --installdir=" + startup::StartUp::installDir() + " --module=" + modType + " > /tmp/module_installer.log 2>&1";
log.writeLog(__LINE__, "run module_installer.sh", LOG_TYPE_DEBUG);
log.writeLog(__LINE__, cmd, LOG_TYPE_DEBUG);
@@ -200,6 +188,54 @@ int main(int argc, char **argv)
exit (0);
}
// if amazon cloud, check and update Instance IP Addresses and volumes
try {
oam.getSystemConfig( "Cloud", cloud);
log.writeLog(__LINE__, "Cloud setting = " + cloud, LOG_TYPE_DEBUG);
}
catch(...) {}
if ( cloud == "amazon-ec2" ) {
if(!aMonitor.amazonIPCheck()) {
log.writeLog(__LINE__, "ERROR: amazonIPCheck failed, exiting", LOG_TYPE_CRITICAL);
sleep(2);
string cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /dev/null 2>&1";
system(cmd.c_str());
exit(1);
}
}
//get gluster config
try {
oam.getSystemConfig( "GlusterConfig", GlusterConfig);
}
catch(...)
{
GlusterConfig = "n";
}
if ( GlusterConfig == "y" ) {
system("mount -a > /dev/null 2>&1");
}
//hdfs / hadoop config
string DBRootStorageType;
try {
oam.getSystemConfig( "DBRootStorageType", DBRootStorageType);
}
catch(...) {}
if ( DBRootStorageType == "hdfs" )
HDFS = true;
//PMwithUM config
try {
oam.getSystemConfig( "PMwithUM", PMwithUM);
}
catch(...) {
PMwithUM = "n";
}
//define entry if missing
if ( gOAMParentModuleFlag )
{
@@ -231,7 +267,6 @@ int main(int argc, char **argv)
int moduleStatus = oam::ACTIVE;
string DBRootStorageType;
//check if currently configured as Parent OAM Module on startup
if ( gOAMParentModuleFlag ) {
try {
@@ -412,52 +447,6 @@ int main(int argc, char **argv)
exit (1);
}
// if amazon cloud, check and update Instance IP Addresses and volumes
try {
oam.getSystemConfig( "Cloud", cloud);
}
catch(...) {}
if ( cloud == "amazon-ec2" ) {
if(!aMonitor.amazonIPCheck()) {
log.writeLog(__LINE__, "ERROR: amazonIPCheck failed, exiting", LOG_TYPE_CRITICAL);
sleep(2);
string cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /dev/null 2>&1";
system(cmd.c_str());
exit(1);
}
}
//get gluster config
try {
oam.getSystemConfig( "GlusterConfig", GlusterConfig);
}
catch(...)
{
GlusterConfig = "n";
}
if ( GlusterConfig == "y" ) {
system("mount -a > /dev/null 2>&1");
}
//hdfs / hadoop config
try {
oam.getSystemConfig( "DBRootStorageType", DBRootStorageType);
}
catch(...) {}
if ( DBRootStorageType == "hdfs" )
HDFS = true;
//PMwithUM config
try {
oam.getSystemConfig( "PMwithUM", PMwithUM);
}
catch(...) {
PMwithUM = "n";
}
//check if module is in a DISABLED state
bool DISABLED = false;
if ( moduleStatus == oam::MAN_DISABLED ||