From 0ea35564f01f8c1f61f2d761483145fc87557002 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 12 Apr 2018 10:49:09 -0500 Subject: [PATCH 001/127] MCOL-520 - remove sudo command --- dbcon/mysql/mysql-Columnstore | 2 - dmlproc/dmlproc.cpp | 5 +- oam/install_scripts/columnstore | 33 ++--- oam/install_scripts/module_installer.sh | 61 +++------ oam/install_scripts/post-install | 72 ++++------ oam/install_scripts/post-mysql-install | 2 +- oam/install_scripts/pre-uninstall | 39 +++--- oam/install_scripts/syslogSetup.sh | 55 ++++---- oam/oamcpp/liboamcpp.cpp | 52 ++------ oamapps/postConfigure/installer.cpp | 25 ---- oamapps/postConfigure/postConfigure.cpp | 125 ++---------------- oamapps/serverMonitor/msgProcessor.cpp | 3 - procmgr/main.cpp | 7 - procmgr/processmanager.cpp | 6 +- procmon/main.cpp | 27 +--- procmon/processmonitor.cpp | 84 ++---------- .../clusterTester/columnstoreClusterTester.sh | 18 +-- 17 files changed, 156 insertions(+), 460 deletions(-) diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore index 02a0de162..0aef31751 100755 --- a/dbcon/mysql/mysql-Columnstore +++ b/dbcon/mysql/mysql-Columnstore @@ -316,10 +316,8 @@ kill_by_pid() { if [ -n "$pid" ]; then ppid=$(ps -o ppid= -p $pid) - $SUDO kill -9 $ppid kill -9 $ppid sleep 1 - $SUDO kill -9 $pid kill -9 $pid echo $echo_n "Force shutting down (no/bad pid file)" log_success_msg diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index 64282d3ca..9c34dd7a5 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -599,10 +599,7 @@ int main(int argc, char* argv[]) try { string port = cf->getConfig(DMLProc, "Port"); - string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; - - if ( !rootUser) - cmd = "sudo fuser -k " + port + "/tcp >/dev/null 2>&1"; + string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; (void)::system(cmd.c_str()); } diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index 038ad0c43..4568c98ec 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -57,17 +57,13 @@ if [ "x$has_um" = x ]; then fi user=`whoami 2>/dev/null` -SUDO=" " -if [ $user != "root" ]; then - SUDO="sudo " -fi checkInstallSetup() { InitialInstallFlag=`$InstallDir/bin/getConfig -c $InstallDir/etc/Columnstore.xml Installation InitialInstallFlag` if [ $InitialInstallFlag != "y" ]; then echo "Please run the postConfigure install script, check the Installation Guide" echo "for additional details" - $SUDO rm -f /var/lock/subsys/columnstore + rm -f /var/lock/subsys/columnstore exit 1 fi } @@ -80,13 +76,13 @@ start() { exit 0 fi - ($SUDO mkdir -p /var/lock/subsys && $SUDO chmod 777 /var/lock/subsys && $SUDO touch /var/lock/subsys/columnstore) >/dev/null 2>&1 + (mkdir -p /var/lock/subsys && chmod 777 /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1 if [ -x $InstallDir/bin/columnstore.pre-start ]; then $InstallDir/bin/columnstore.pre-start if [ $? -ne 0 ]; then echo "Error running MariaDB Columnstore pre-start script, not starting MariaDB Columnstore" - $SUDO rm -f /var/lock/subsys/columnstore + rm -f /var/lock/subsys/columnstore exit 1 fi fi @@ -94,34 +90,29 @@ start() { CoreFileFlag=`$InstallDir/bin/getConfig -c $InstallDir/etc/Columnstore.xml Installation CoreFileFlag` if [ $CoreFileFlag = "y" ]; then - SUDO= - if [ "$user" != "root" ]; then - SUDO="$SUDO" - fi - #columnstore core files - $SUDO ulimit -c unlimited > /dev/null 2>&1 - $SUDO sysctl -q -w kernel.core_uses_pid=1 > /dev/null 2>&1 - $SUDO sysctl -q -w kernel.core_pattern=/var/log/mariadb/columnstore/corefiles/core.%e.%p > /dev/null 2>&1 + ulimit -c unlimited > /dev/null 2>&1 + sysctl -q -w kernel.core_uses_pid=1 > /dev/null 2>&1 + sysctl -q -w kernel.core_pattern=/var/log/mariadb/columnstore/corefiles/core.%e.%p > /dev/null 2>&1 fi RETVAL=0 echo "Starting MariaDB Columnstore Database Platform" - $SUDO rm -f /tmp/StopColumnstore + rm -f /tmp/StopColumnstore exec $InstallDir/bin/run.sh $InstallDir/bin/ProcMon > /dev/null 2>&1 & return $RETVAL } stop() { echo "Shutting down MariaDB Columnstore Database Platform" - $SUDO touch /tmp/StopColumnstore - $SUDO pkill ProcMon - $SUDO pkill ProcMgr + touch /tmp/StopColumnstore + pkill ProcMon + pkill ProcMgr sleep 1 $InstallDir/bin/clearShm RETVAL=$? - $SUDO rm -f /var/lock/subsys/columnstore - $SUDO fuser -k 8604/tcp > /dev/null 2>&1 + rm -f /var/lock/subsys/columnstore + fuser -k 8604/tcp > /dev/null 2>&1 test -f $InstallDir/mysql/mysql-Columnstore || return $RETVAL $InstallDir/mysql/mysql-Columnstore stop > /dev/null 2>&1 if [ -x $InstallDir/bin/columnstore.post-stop ]; then diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 6ca39b70f..457edb9ac 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -63,17 +63,10 @@ if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then if [ $module = "pm" ]; then if test -f $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab ; then echo "Setup fstab on Module" - if [ $user = "root" ]; then - touch /etc/fstab - rm -f /etc/fstab.columnstoreSave - cp /etc/fstab /etc/fstab.columnstoreSave - cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab - else - sudo touch /etc/fstab - sudo chmod 666 /etc/fstab - sudo rm -f /etc/fstab.columnstoreSave - sudo cp /etc/fstab /etc/fstab.columnstoreSave - sudo cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab + touch /etc/fstab + rm -f /etc/fstab.columnstoreSave + cp /etc/fstab /etc/fstab.columnstoreSave + cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab fi fi fi @@ -93,12 +86,8 @@ if [ $module = "um" ]; then echo "Setup UM Volume Mount" device=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation UMVolumeDeviceName$mid` mkdir -p $COLUMNSTORE_INSTALL_DIR/mysql/db > /dev/null 2>&1 - if [ $user = "root" ]; then - mount $device $COLUMNSTORE_INSTALL_DIR/mysql/db -t ext2 -o noatime,nodiratime,noauto - chown mysql:mysql -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1 - else - sudo mount $device $COLUMNSTORE_INSTALL_DIR/mysql/db -t ext2 -o noatime,nodiratime,noauto,user - sudo chown $user:$user -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1 + mount $device $COLUMNSTORE_INSTALL_DIR/mysql/db -t ext2 -o noatime,nodiratime,noauto + chown mysql:mysql -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1 fi fi fi @@ -181,40 +170,22 @@ touch $RCFILE echo "add deadline to rc.local" if [ $module = "um" ]; then - if [ $user = "root" ]; then - echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - echo "done" >> $RCFILE - else - sudo chmod 666 $RCFILE - sudo echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - sudo echo "done" >> $RCFILE - fi + echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE + echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE + echo "done" >> $RCFILE else - if [ $user = "root" ]; then - echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - echo "done" >> $RCFILE + echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE + echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE + echo "done" >> $RCFILE - echo "for scsi_dev in \`mount | awk '/columnstore\\/data/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - echo "done" >> $RCFILE - else - sudo chmod 666 $RCFILE - sudo echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - sudo echo "done" >> $RCFILE - - sudo echo "for scsi_dev in \`mount | awk '/columnstore\\/data/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - sudo echo "done" >> $RCFILE - fi + echo "for scsi_dev in \`mount | awk '/columnstore\\/data/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE + echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE + echo "done" >> $RCFILE fi if [ $user != "root" ]; then echo "uncomment runuser in rc.local" - sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 + sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 fi echo "!!!Module Installation Successfully Completed!!!" diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index f7aeeb2ca..757116788 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -8,10 +8,6 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install user=`whoami 2>/dev/null` -SUDO=" " -if [ $user != "root" ]; then - SUDO="sudo " -fi quiet=0 @@ -93,7 +89,7 @@ if [ -f libstdc++.so.6.0.14 ]; then fi # softlink for libperl.sp, used by cplogger -$SUDO ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&1 +ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&1 #setup profile files profileFileAlias=/etc/profile.d/columnstoreAlias.sh @@ -104,36 +100,36 @@ if [ $installdir != "/usr/local/mariadb/columnstore" ]; then fi if [ $user != "root" ]; then - sudo rm -f $profileFileEnv - sudo rm -f $profileFileAlias + rm -f $profileFileEnv + rm -f $profileFileAlias - sudo touch $profileFileEnv - sudo chmod 666 $profileFileEnv + touch $profileFileEnv + chmod 666 $profileFileEnv egrep -qs 'MariaDB Columnstore Non-Root' ${profileFileEnv} rc1=$? if [ $rc1 -ne 0 ]; then - sudo echo " " >> ${profileFileEnv} - sudo echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${profileFileEnv} - sudo echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${profileFileEnv} - sudo echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib" >> ${profileFileEnv} + echo " " >> ${profileFileEnv} + echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${profileFileEnv} + echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${profileFileEnv} + echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib" >> ${profileFileEnv} . ${profileFileEnv} fi - sudo /bin/cp -f $installdir/bin/columnstoreAlias $profileFileAlias - sudo chmod 666 $profileFileAlias + /bin/cp -f $installdir/bin/columnstoreAlias $profileFileAlias + chmod 666 $profileFileAlias else /bin/cp -f $installdir/bin/columnstoreAlias $profileFileAlias - chmod 644 $profileFileAlias + 644 $profileFileAlias fi cd / -test -d /var/log/mariadb || $SUDO mkdir /var/log/mariadb >/dev/null 2>&1 -test -d /var/log/mariadb/columnstore || $SUDO mkdir /var/log/mariadb/columnstore >/dev/null 2>&1 +test -d /var/log/mariadb || mkdir /var/log/mariadb >/dev/null 2>&1 +test -d /var/log/mariadb/columnstore || mkdir /var/log/mariadb/columnstore >/dev/null 2>&1 if [ $user != "root" ]; then - $SUDO chmod -R 777 /var/log/mariadb >/dev/null 2>&1 - $SUDO chown -R $user:$user /var/log/mariadb >/dev/null 2>&1 + chmod -R 777 /var/log/mariadb >/dev/null 2>&1 + chown -R $user:$user /var/log/mariadb >/dev/null 2>&1 fi test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 @@ -190,17 +186,9 @@ if [ $user = "root" ]; then touch $RCFILE chmod +x $RCFILE else - $SUDO touch $RCFILE - $SUDO chmod 777 $RCFILE - $SUDO printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILE > /dev/null 2>&1 - - if [ -n "$systemctl" ]; then - $SUDO systemctl start rc-local >/dev/null 2>&1 - $SUDO systemctl enable rc-local >/dev/null 2>&1 - fi + printf '%s\n' '#!/bin/bash' "#" | tee -a $RCFILE > /dev/null 2>&1 fi - #setup the columnstore service script rm -f /etc/init.d/columnstore >/dev/null 2>&1 rm -f /etc/init.d/mysql-Columnstore >/dev/null 2>&1 @@ -236,28 +224,24 @@ fi if [ $user = "root" ]; then $installdir/bin/syslogSetup.sh install > /tmp/syslog_install.log 2>&1 rm -f /etc/default/columnstore + + #check if MariaDB Columnstore system logging was setup + cat /tmp/syslog_install.log | grep 'No System Logging' >/dev/null 2>&1 + if [ $? -eq 0 ]; then + cat /tmp/syslog_install.log + fi else sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstore.def sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstoreLogRotate - $SUDO cp $installdir/bin/columnstore.def /etc/default/columnstore + cp $installdir/bin/columnstore.def /etc/default/columnstore sed -i -e s@prefix=/home/guest@prefix=$prefix@g $installdir/bin/* - $SUDO chmod 777 /tmp - $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1 - $SUDO chown $user:$user $installdir/etc/Columnstore.xml - $SUDO chmod -R 777 /dev/shm - $SUDO mkdir /var/lock/subsys > /dev/null 2>&1 - $SUDO chmod 777 /var/lock/subsys > /dev/null 2>&1 - $SUDO rm -f /var/lock/subsys/mysql-Columnstore - $SUDO chmod 666 /etc/fstab -fi - -#check if MariaDB Columnstore system logging was setup -cat /tmp/syslog_install.log | grep 'No System Logging' >/dev/null 2>&1 -if [ $? -eq 0 ]; then - cat /tmp/syslog_install.log + chown $user:$user $installdir/etc/Columnstore.xml + mkdir /var/lock/subsys > /dev/null 2>&1 + chmod 777 /var/lock/subsys > /dev/null 2>&1 + rm -f /var/lock/subsys/mysql-Columnstore fi #backup copy of Alarm Config File diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 24711b048..9163c3833 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -62,7 +62,7 @@ done USER=`whoami 2>/dev/null` if [ $USER != "root" ]; then - sudo ldconfig >/dev/null 2>&1 + ldconfig >/dev/null 2>&1 export COLUMNSTORE_INSTALL_DIR=$installdir export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib else diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 2f58b5421..56d2fbb3e 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -8,10 +8,6 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install user=`whoami 2>/dev/null` -SUDO=" " -if [ $user != "root" ]; then - SUDO="sudo " -fi quiet=0 @@ -39,9 +35,9 @@ $installdir/myql/columnstore-Mysql stop > /dev/null 2>&1 # uninstall OS scripts updated by postConfigure if test -f /etc/exports.columnstoreSave ; then - $SUDO mv -f /etc/exports.columnstoreSave /etc/exports > /dev/null 2>&1 + mv -f /etc/exports.columnstoreSave /etc/exports > /dev/null 2>&1 else - $SUDO rm -f /etc/exports > /dev/null 2>&1 + rm -f /etc/exports > /dev/null 2>&1 fi export COLUMNSTORE_INSTALL_DIR=$installdir @@ -49,30 +45,30 @@ export COLUMNSTORE_INSTALL_DIR=$installdir cloud=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation Cloud` if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then if test -f /etc/fstab ; then - $SUDO sed -i '/Columnstore\/data/d' /etc/fstab > /dev/null 2>&1 + sed -i '/Columnstore\/data/d' /etc/fstab > /dev/null 2>&1 fi fi #remove log file directories #rm -rf /var/log/mariadb/columnstore > /dev/null 2>&1 #rm -f $installdir/mysql/db/*.err > /dev/null 2>&1 -$SUDO rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1 -$SUDO rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1 + rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1 + rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1 rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1 -$SUDO rm -f /etc/default/columnstore + rm -f /etc/default/columnstore # remove Mariab Columnstore Log Rotate File abd Transaction Log File -$SUDO rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1 -$SUDO rm -f /etc/cron.d/transactionLog > /dev/null 2>&1 + rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1 + rm -f /etc/cron.d/transactionLog > /dev/null 2>&1 # delete Mariab Columnstore shared memory segments $installdir/bin/clearShm > /dev/null 2>&1 # delete prat files -$SUDO rm -f /etc/cron.d/ps > /dev/null 2>&1 -$SUDO rm -f /etc/pscollect > /dev/null 2>&1 -$SUDO /etc/init.d/crond reload > /dev/null 2>&1 -$SUDO systemctl reload crond.service > /dev/null 2>&1 + rm -f /etc/cron.d/ps > /dev/null 2>&1 + rm -f /etc/pscollect > /dev/null 2>&1 + /etc/init.d/crond reload > /dev/null 2>&1 + systemctl reload crond.service > /dev/null 2>&1 # delete tmp files rm -f $installdir/local/*.columnstore @@ -81,7 +77,7 @@ rm -rf /tmp/bucketreuse rm -f /tmp/columnstore.txt rm -f /tmp/dbbuilder.* rm -f /tmp/dbrmfiles -$SUDO rm -f /var/lock/subsys/columnstore + rm -f /var/lock/subsys/columnstore rm -f /tmp/pkgcheck rm -f /tmp/upgrade-status.log.* rm -f /tmp/mount.log @@ -103,9 +99,6 @@ fi if [ -x $installdir/bin/syslogSetup.sh ]; then if [ $user = "root" ]; then $installdir/bin/syslogSetup.sh uninstall >/dev/null 2>&1 - else - $SUDO $installdir/bin/syslogSetup.sh uninstall >/dev/null 2>&1 - fi fi #remove the start service command @@ -145,7 +138,7 @@ fi if [ $user = "root" ]; then rm -f $installdir/data*/OAMdbrootCheck > /dev/null 2>&1 else - $SUDO rm -f $installdir/data*/OAMdbrootCheck > /dev/null 2>&1 + rm -f $installdir/data*/OAMdbrootCheck > /dev/null 2>&1 fi #remove library paths @@ -153,14 +146,14 @@ if [ $user = "root" ]; then rm -f /etc/ld.so.conf.d/columnstore.conf ldconfig else - $SUDO rm -f /etc/defaults/columnstore + rm -f /etc/defaults/columnstore fi #cleanup profile file if [ $user = "root" ]; then rm -f /etc/profile.d/columnstore* else - $SUDO rm -f /etc/profile.d/columnstore* + rm -f /etc/profile.d/columnstore* fi #tell user to run post configure script diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 1f4235a30..00ade18f4 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -14,11 +14,6 @@ rsyslog7=0 user=`whoami 2>/dev/null` -SUDO=" " -if [ "$user" != "root" ]; then - SUDO="sudo " -fi - for arg in "$@"; do if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then prefix="`echo $arg | awk -F= '{print $2}'`" @@ -69,13 +64,13 @@ if [ "$daemon" = "nodaemon" ]; then if [ -f /etc/syslog.conf ]; then daemon="syslog" - $SUDO /etc/init.d/syslog start > /dev/null 2>&1 + /etc/init.d/syslog start > /dev/null 2>&1 elif [ -f /etc/rsyslog.conf ]; then daemon="rsyslog" - $SUDO /etc/init.d/rsyslog start > /dev/null 2>&1 + /etc/init.d/rsyslog start > /dev/null 2>&1 elif [ -f /etc/init.d/syslog-ng ]; then daemon="syslog-ng" - $SUDO /etc/init.d/syslog-ng start > /dev/null 2>&1 + /etc/init.d/syslog-ng start > /dev/null 2>&1 fi fi @@ -99,7 +94,7 @@ if [ "$daemon" = "syslog-ng" ]; then fi elif [ "$daemon" = "rsyslog" ]; then #check if rsyslog version 7 or greater - $SUDO rsyslogd -v > /tmp/rsyslog.ver + rsyslogd -v > /tmp/rsyslog.ver cnt=`grep "rsyslogd 7" /tmp/rsyslog.ver | wc -l` if [ $cnt -gt 0 ]; then rsyslog7=1 @@ -156,31 +151,31 @@ checkSyslog if [ ! -z "$syslog_conf" ] ; then $installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1 if [ "$syslog_conf" != /etc/rsyslog.d/columnstore.conf ]; then - $SUDO rm -f ${syslog_conf}.columnstoreSave - $SUDO cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 - $SUDO sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 + rm -f ${syslog_conf}.columnstoreSave + cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 + sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 fi egrep -qs 'MariaDB Columnstore Database Platform Logging' ${syslog_conf} if [ $? -ne 0 ]; then #set the syslog for ColumnStore logging # remove older version incase it was installed by previous build - $SUDO rm -rf /etc/rsyslog.d/columnstore.conf + rm -rf /etc/rsyslog.d/columnstore.conf if [ $rsyslog7 == 1 ]; then sed -i -e s/groupname/adm/g ${columnstoreSyslogFile7} sed -i -e s/username/syslog/g ${columnstoreSyslogFile7} - $SUDO rm -f /etc/rsyslog.d/49-columnstore.conf - $SUDO cp ${columnstoreSyslogFile7} ${syslog_conf} + rm -f /etc/rsyslog.d/49-columnstore.conf + cp ${columnstoreSyslogFile7} ${syslog_conf} else - $SUDO cp ${columnstoreSyslogFile} ${syslog_conf} + cp ${columnstoreSyslogFile} ${syslog_conf} fi fi # install Columnstore Log Rotate File - $SUDO cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1 - $SUDO chmod 644 /etc/logrotate.d/columnstore + cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1 + chmod 644 /etc/logrotate.d/columnstore restartSyslog fi @@ -196,19 +191,19 @@ if [ ! -z "$syslog_conf" ] ; then if [ $? -eq 0 ]; then if [ -f ${syslog_conf}.columnstoreSave ] ; then #uninstall the syslog for ColumnStore logging - $SUDO v -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup - $SUDO mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1 + v -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup + mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1 if [ ! -f ${syslog_conf} ] ; then - $SUDO cp ${syslog_conf}.ColumnStoreBackup ${syslog_conf} + cp ${syslog_conf}.ColumnStoreBackup ${syslog_conf} fi fi fi - $SUDO sed -i '/# MariaDB/,$d' ${syslog_conf} > /dev/null 2>&1 + sed -i '/# MariaDB/,$d' ${syslog_conf} > /dev/null 2>&1 else - $SUDO rm -f "$syslog_conf" + rm -f "$syslog_conf" fi else - $SUDO rm -f "$syslog_conf" + rm -f "$syslog_conf" fi restartSyslog @@ -248,21 +243,21 @@ restartSyslog() { if [ "$daemon" = "syslog-ng" ]; then if [ -f /etc/init.d/syslog-ng ]; then - $SUDO /etc/init.d/syslog-ng restart > /dev/null 2>&1 + /etc/init.d/syslog-ng restart > /dev/null 2>&1 else - $SUDO systemctl restart syslog-ng.service > /dev/null 2>&1 + systemctl restart syslog-ng.service > /dev/null 2>&1 fi elif [ "$daemon" = "rsyslog" ]; then if [ -f /etc/init.d/rsyslog ]; then - $SUDO /etc/init.d/rsyslog restart > /dev/null 2>&1 + /etc/init.d/rsyslog restart > /dev/null 2>&1 else - $SUDO systemctl restart rsyslog.service > /dev/null 2>&1 + systemctl restart rsyslog.service > /dev/null 2>&1 fi elif [ "$daemon" = "syslog" ]; then if [ -f /etc/init.d/syslog ]; then - $SUDO /etc/init.d/syslog restart > /dev/null 2>&1 + /etc/init.d/syslog restart > /dev/null 2>&1 else - $SUDO systemctl restart syslog.service > /dev/null 2>&1 + systemctl restart syslog.service > /dev/null 2>&1 fi fi } diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 92b2db3b7..dfe3822be 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -6423,10 +6423,7 @@ void Oam::addUMdisk(const int moduleID, std::string& volumeName, std::string& de int user; user = getuid(); - if (user == 0) - cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1"; - else - cmd = "sudo mkfs.ext2 -F " + device + " > /dev/null 2>&1"; + cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1"; system(cmd.c_str()); @@ -6673,10 +6670,7 @@ void Oam::addDbroot(const int dbrootNumber, DBRootConfigList& dbrootlist, string int user; user = getuid(); - if (user == 0) - cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1"; - else - cmd = "sudo mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1"; + cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1"; writeLog("addDbroot format cmd: " + cmd, LOG_TYPE_DEBUG ); @@ -8660,21 +8654,11 @@ void Oam::syslogAction( std::string action) } else { - int user; - user = getuid(); + cmd = "systemctl " + action + " " + systemlog + ".service > /dev/null 2>&1"; + system(cmd.c_str()); - if (user == 0) - cmd = "systemctl " + action + " " + systemlog + ".service > /dev/null 2>&1"; - else - cmd = "sudo systemctl " + action + " " + systemlog + ".service > /dev/null 2>&1"; - - system(cmd.c_str()); - - if (user == 0) - cmd = "/service " + systemlog + " " + action + " > /dev/null 2>&1"; - else - cmd = "sudo service" + systemlog + " " + action + " > /dev/null 2>&1"; - } + cmd = "service " + systemlog + " " + action + " > /dev/null 2>&1"; + } // take action on syslog service writeLog("syslogAction cmd: " + cmd, LOG_TYPE_DEBUG ); @@ -8761,11 +8745,6 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string string glustercmd = "gluster "; - if (user != 0) - { - glustercmd = "sudo " + glustercmd; - } - errmsg = ""; switch ( command ) @@ -9022,7 +9001,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string if (user != 0) { - command = "sudo gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-uid " + itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; + command = "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-uid " + itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9031,7 +9010,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string exceptionControl("GLUSTER_ADD", API_FAILURE); } - command = "sudo gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-gid " + itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; + command = "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-gid " + itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9449,10 +9428,7 @@ std::string Oam::updateFstab(std::string device, std::string dbrootID) string cmd; - if (user == 0) - cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; - else - cmd = "sudo grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; + cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1"; int status = system(cmd.c_str()); @@ -9461,18 +9437,12 @@ std::string Oam::updateFstab(std::string device, std::string dbrootID) //update /etc/fstab with mount //update local fstab - if (user == 0) - cmd = "echo " + entry + " >> /etc/fstab"; - else - cmd = "sudo echo " + entry + " >> /etc/fstab"; + cmd = "echo " + entry + " >> /etc/fstab"; system(cmd.c_str()); } - if (user == 0) - cmd = "grep /data" + dbrootID + " " + InstallDir + "/local/etc/pm1/fstab > /dev/null 2>&1"; - else - cmd = "sudo grep /data" + dbrootID + " " + InstallDir + "/local/etc/pm1/fstab > /dev/null 2>&1"; + cmd = "grep /data" + dbrootID + " " + InstallDir + "/local/etc/pm1/fstab > /dev/null 2>&1"; status = system(cmd.c_str()); diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 3bc02c6ae..1aa8106f0 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -400,10 +400,6 @@ int main(int argc, char* argv[]) exit(1); } - //store syslog config file in Calpont Config file - cmd = installDir + "/bin/syslogSetup.sh install > /dev/null 2>&1"; - system(cmd.c_str()); - //get PPackage Type try { @@ -951,8 +947,6 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall) string fileName = "/etc/" + files[i]; //make a backup copy before changing - if ( rootUser ) - { cmd = "rm -f " + fileName + ".columnstoreSave"; system(cmd.c_str()); @@ -968,25 +962,6 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall) cmd = "cp " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont " + installDir + "/local/. > /dev/null 2>&1"; system(cmd.c_str()); - } - else - { - cmd = "sudo rm -f " + fileName + ".columnstoreSave"; - system(cmd.c_str()); - - cmd = "sudo cp " + fileName + " " + fileName + ".columnstoreSave > /dev/null 2>&1"; - system(cmd.c_str()); - - cmd = "sudo cat " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont >> " + fileName; - - if (geteuid() == 0) system(cmd.c_str()); - - cmd = "sudo rm -f " + installDir + "/local/ " + files[i] + "*.calpont > /dev/null 2>&1"; - system(cmd.c_str()); - - cmd = "sudo cp " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont " + installDir + "/local/. > /dev/null 2>&1"; - system(cmd.c_str()); - } } return allfound; diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9cc3bf5b2..7fb1a0fce 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1190,8 +1190,8 @@ int main(int argc, char* argv[]) // setup to start on reboot, for non-root amazon installs if ( !rootUser ) { - system("sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); - system("sudo chmod 666 /etc/fstab >/dev/null 2>&1"); + system("sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); + system("chmod 666 /etc/fstab >/dev/null 2>&1"); } if ( !writeConfig(sysConfig) ) @@ -3612,17 +3612,7 @@ int main(int argc, char* argv[]) //check if local MariaDB ColumnStore system logging is working cout << endl << "===== Checking MariaDB ColumnStore System Logging Functionality =====" << endl << endl; - if ( rootUser) - cmd = installDir + "/bin/syslogSetup.sh install > /dev/null 2>&1"; - else - cmd = "sudo " + installDir + "/bin/syslogSetup.sh --installdir=" + installDir + " install > /dev/null 2>&1"; - - system(cmd.c_str()); - - if ( rootUser) - cmd = installDir + "/bin/syslogSetup.sh status > /dev/null 2>&1"; - else - cmd = "sudo " + installDir + "/bin/syslogSetup.sh --installdir=" + installDir + " status > /dev/null 2>&1"; + cmd = installDir + "/bin/syslogSetup.sh status > /dev/null 2>&1"; int ret = system(cmd.c_str()); @@ -4068,22 +4058,13 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall) //make a backup copy before changing string cmd = "rm -f " + fileName + ".columnstoreSave"; - if ( !rootUser ) - cmd = "sudo rm -f " + fileName + ".columnstoreSave"; - system(cmd.c_str()); cmd = "cp " + fileName + " " + fileName + ".columnstoreSave > /dev/null 2>&1"; - if ( !rootUser ) - cmd = "sudo cp " + fileName + " " + fileName + ".columnstoreSave > /dev/null 2>&1"; - system(cmd.c_str()); - if ( rootUser ) - cmd = "cat " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont >> " + fileName; - else - cmd = "sudo bash -c 'sudo cat " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont >> " + fileName + "'"; + cmd = "cat " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont >> " + fileName; int rtnCode = system(cmd.c_str()); @@ -4385,7 +4366,7 @@ bool createDbrootDirs(string DBRootStorageType) if ( !rootUser) { - cmd = "sudo chown -R " + USER + ":" + USER + " " + installDir + "/data1 > /dev/null"; + cmd = "chown -R " + USER + ":" + USER + " " + installDir + "/data1 > /dev/null"; system(cmd.c_str()); } @@ -4845,10 +4826,7 @@ bool storageSetup(bool amazonInstall) //check if gluster is installed int rtnCode = 1; - if (rootUser) - rtnCode = system("gluster --version > /tmp/gluster.log 2>&1"); - else - rtnCode = system("sudo gluster --version > /tmp/gluster.log 2>&1"); + rtnCode = system("gluster --version > /tmp/gluster.log 2>&1"); if (rtnCode == 0) { @@ -5534,10 +5512,7 @@ bool updateBash() string cmd = "echo . " + installDir + "/bin/" + DataFileEnvFile + " >> " + fileName; system(cmd.c_str()); - if ( rootUser) - cmd = "su - hdfs -c 'hadoop fs -mkdir -p " + installDir + ";hadoop fs -chown root:root " + installDir + "' >/dev/null 2>&1"; - else - cmd = "sudo su - hdfs -c 'hadoop fs -mkdir -p " + installDir + ";hadoop fs -chown " + USER + ":" + USER + " " + installDir + "' >/dev/null 2>&1"; + cmd = "su - hdfs -c 'hadoop fs -mkdir -p " + installDir + ";hadoop fs -chown root:root " + installDir + "' >/dev/null 2>&1"; system(cmd.c_str()); } @@ -6425,67 +6400,7 @@ bool glusterSetup(string password) cout << "ERROR: failed to make directory(" << DataRedundancyConfigs[pm].pmIpAddr << "): '" << command << "'" << endl; exit(1); } - - /* - if (dataRedundancyStorage == 2) - { - //walk data storage locations and modify fstab to reflect the storage locations entered by user - vector::iterator storageSetupIter=DataRedundancyConfigs[pm].storageLocations.begin(); - for (; storageSetupIter < DataRedundancyConfigs[pm].storageLocations.end(); storageSetupIter++ ) - { - if (rootUser) - { - command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + - " 'echo " + (*storageSetupIter).storageLocation + " " + - installDir + "/gluster/brick" + oam.itoa(brick) + " " + - (*storageSetupIter).storageFilesytemType + " defaults 1 2 >> /etc/fstab'"; - } - else - { - command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + - " 'sudo bash -c `sudo echo " + (*storageSetupIter).storageLocation + " " + - installDir + "/gluster/brick" + oam.itoa(brick) + " " + - (*storageSetupIter).storageFilesytemType + " defaults 1 2 >> /etc/fstab`'"; - } - status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: command failed: " << command << endl; - exit(1); - } - if (rootUser) - { - command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + - " 'mount " + installDir + "/gluster/brick" + oam.itoa(brick) + "'"; - } - else - { - command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + - " 'sudo bash -c `sudo mount " + installDir + "/gluster/brick" + oam.itoa(brick) + "`'"; - } - status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: command failed: " << command << endl; - exit(1); - } - if (!rootUser) - { - int user; - user = getuid(); - command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + - "'sudo bash -c `sudo chown -R " + oam.itoa(user) + ":" + oam.itoa(user) + " " + installDir + "/gluster/brick" + oam.itoa(brick) + "`'"; - status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR(" << status <<"): command failed: " << command << endl; - } - } - - } - } - */ - } + } string errmsg1; string errmsg2; @@ -6499,14 +6414,7 @@ bool glusterSetup(string password) sleep(5); - if (rootUser) - { - command = "gluster peer status >> /tmp/glusterCommands.txt 2>&1"; - } - else - { - command = "sudo gluster peer status >> /tmp/glusterCommands.txt 2>&1"; - } + command = "gluster peer status >> /tmp/glusterCommands.txt 2>&1"; status = system(command.c_str()); @@ -6531,14 +6439,7 @@ bool glusterSetup(string password) { int dbrootID = db + 1; - if (rootUser) - { - command = "gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; - } - else - { - command = "sudo gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; - } + command = "gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; vector::iterator dbrootPmIter = dbrootPms[db].begin(); @@ -6597,7 +6498,7 @@ bool glusterSetup(string password) { int user = getuid(); int group = getgid(); - command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; + command = "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -6606,7 +6507,7 @@ bool glusterSetup(string password) exit(1); } - command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; + command = "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -6615,7 +6516,7 @@ bool glusterSetup(string password) exit(1); } - command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; + command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) diff --git a/oamapps/serverMonitor/msgProcessor.cpp b/oamapps/serverMonitor/msgProcessor.cpp index d76dc65b3..bfcf6afcc 100644 --- a/oamapps/serverMonitor/msgProcessor.cpp +++ b/oamapps/serverMonitor/msgProcessor.cpp @@ -101,9 +101,6 @@ void msgProcessor() int user; user = getuid(); - if (user != 0) - cmd = "sudo fuser -k " + port + "/tcp"; - system(cmd.c_str()); } catch (...) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 4d99b4372..256bd1bd4 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -475,10 +475,6 @@ static void messageThread(Configuration config) string port = sysConfig->getConfig("ProcMgr", "Port"); string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; - if ( !rootUser) - cmd = "sudo fuser -k " + port + "/tcp >/dev/null 2>&1"; - - system(cmd.c_str()); } catch (...) @@ -563,9 +559,6 @@ static void alarmMessageThread(Configuration config) string port = sysConfig->getConfig("ProcMgr_Alarm", "Port"); string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; - if ( !rootUser) - cmd = "sudo fuser -k " + port + "/tcp >/dev/null 2>&1"; - system(cmd.c_str()); } catch (...) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 6a87495ba..fd6798e12 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -4937,7 +4937,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str //clear out the known_host file, sometimes causes a failure on amazon during addModule if ( amazon ) { - string cmd = "sudo unlink " + homedir + ".ssh/know_hosts > /dev/null 2>&1"; + string cmd = "unlink " + homedir + ".ssh/know_hosts > /dev/null 2>&1"; system(cmd.c_str()); } @@ -6298,7 +6298,7 @@ int ProcessManager::removeModule(oam::DeviceNetworkList devicenetworklist, bool homedir = p; } - string cmd = "sudo unlink " + homedir + ".ssh/know_hosts > /dev/null 2>&1"; + string cmd = "unlink " + homedir + ".ssh/know_hosts > /dev/null 2>&1"; system(cmd.c_str()); } @@ -11057,7 +11057,7 @@ int ProcessManager::mountDBRoot(std::string dbrootID) if ( !rootUser) { - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /dev/null"; + cmd = "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /dev/null"; system(cmd.c_str()); } diff --git a/procmon/main.cpp b/procmon/main.cpp index d6b14abf4..7190bbcd7 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -134,13 +134,6 @@ int main(int argc, char** argv) if (p && *p) USER = p; - // change permissions on /dev/shm - if ( !rootUser) - { - string cmd = "sudo chmod 777 /dev/shm >/dev/null 2>&1"; - system(cmd.c_str()); - } - // get and set locale language string systemLang = "C"; @@ -346,8 +339,8 @@ int main(int argc, char** argv) if ( count >= 120 ) { log.writeLog(__LINE__, "Standby PM not responding, infinidb shutting down", LOG_TYPE_CRITICAL); - //Set the alarm - aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); + //Set the alarm, commented out. alarm require ProcMgr + //aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); sleep (1); string cmd = startup::StartUp::installDir() + "/bin/infinidb stop > /dev/null 2>&1"; system(cmd.c_str()); @@ -444,7 +437,7 @@ int main(int argc, char** argv) } catch (...) { - log.writeLog(__LINE__, "wating for good return from getModuleStatus", LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "waiting for good return from getModuleStatus", LOG_TYPE_DEBUG); sleep (1); } } @@ -560,8 +553,8 @@ int main(int argc, char** argv) if ( retry == 20 ) { log.writeLog(__LINE__, "Check DB mounts failed, shutting down", LOG_TYPE_CRITICAL); - //Set the alarm - aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); + //Set the alarm, commented out. alarm require ProcMgr + //aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); sleep (1); string cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /dev/null 2>&1"; system(cmd.c_str()); @@ -589,7 +582,7 @@ int main(int argc, char** argv) while (!mainResumeFlag) { - log.writeLog(__LINE__, "WATING FOR mainResumeFlag to be set", LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "WAITING FOR mainResumeFlag to be set", LOG_TYPE_DEBUG); sleep(1); } @@ -1138,9 +1131,6 @@ static void messageThread(MonitorConfig config) string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; - if ( !rootUser) - cmd = "sudo fuser -k " + port + "/tcp >/dev/null 2>&1"; - system(cmd.c_str()); for (;;) @@ -2370,10 +2360,7 @@ static void statusControlThread() string port = sysConfig->getConfig(portName, "Port"); string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; - if ( !rootUser) - cmd = "sudo fuser -k " + port + "/tcp >/dev/null 2>&1"; - - system(cmd.c_str()); + system(cmd.c_str()); } catch (...) { diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index cda141e84..16ca6b6e9 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1027,7 +1027,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string cmd = "chmod 755 /dev/shm >/dev/null 2>&1"; if ( !rootUser) - cmd = "sudo chmod 777 /dev/shm >/dev/null 2>&1"; + cmd = "chmod 777 /dev/shm >/dev/null 2>&1"; system(cmd.c_str()); @@ -1729,20 +1729,10 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if (!oam.checkLogStatus("/tmp/umount.txt", "busy")) break; - if ( rootUser) - { - cmd = "lsof " + startup::StartUp::installDir() + "/data" + dbrootID + " >> /tmp/umount.txt 2>&1"; - system(cmd.c_str()); - cmd = "fuser -muvf " + startup::StartUp::installDir() + "/data" + dbrootID + " >> /tmp/umount.txt 2>&1"; - system(cmd.c_str()); - } - else - { - cmd = "sudo lsof " + startup::StartUp::installDir() + "/data" + dbrootID + " >> /tmp/umount.txt 2>&1"; - system(cmd.c_str()); - cmd = "sudo fuser -muvf " + startup::StartUp::installDir() + "/data" + dbrootID + " >> /tmp/umount.txt 2>&1"; - system(cmd.c_str()); - } + cmd = "lsof " + startup::StartUp::installDir() + "/data" + dbrootID + " >> /tmp/umount.txt 2>&1"; + system(cmd.c_str()); + cmd = "fuser -muvf " + startup::StartUp::installDir() + "/data" + dbrootID + " >> /tmp/umount.txt 2>&1"; + system(cmd.c_str()); sleep(2); //Flush the cache @@ -1801,7 +1791,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if ( !rootUser) { - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /dev/null"; + cmd = "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /dev/null"; system(cmd.c_str()); } @@ -1843,15 +1833,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if (WEXITSTATUS(status) != 0 ) { - if ( rootUser) - { - cmd = "echo " + entry + " >> /etc/fstab"; - } - else - { - cmd = "sudo echo " + entry + " >> /etc/fstab"; - } - + cmd = "echo " + entry + " >> /etc/fstab"; system(cmd.c_str()); log.writeLog(__LINE__, "Add line entry to /etc/fstab : " + entry); @@ -1863,14 +1845,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if (WEXITSTATUS(status) != 0 ) { - if ( rootUser) - { - cmd = "echo " + entry + " >> " + startup::StartUp::installDir() + "/local/etc/pm1/fstab"; - } - else - { - cmd = "sudo echo " + entry + " >> " + startup::StartUp::installDir() + "/local/etc/pm1/fstab"; - } + cmd = "echo " + entry + " >> " + startup::StartUp::installDir() + "/local/etc/pm1/fstab"; system(cmd.c_str()); @@ -1882,14 +1857,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string::size_type pos1 = entry.find(" ", pos + 1); string directory = entry.substr(pos + 1, pos1 - pos); - if ( rootUser) - { - cmd = "mkdir " + directory; - } - else - { - cmd = "sudo mkdir " + directory; - } + cmd = "mkdir " + directory; system(cmd.c_str()); log.writeLog(__LINE__, "create directory: " + directory); @@ -3123,7 +3091,7 @@ int ProcessMonitor::updateLog(std::string action, std::string level) //if non-root, change file permissions so we can update it if ( !rootUser) { - string cmd = "sudo chmod 666 " + fileName + " > /dev/null"; + string cmd = "chmod 666 " + fileName + " > /dev/null"; system(cmd.c_str()); } @@ -5208,16 +5176,10 @@ int ProcessMonitor::changeMyCnf(std::string type) // set owner and permission string cmd = "chmod 664 " + mycnfFile + " >/dev/null 2>&1"; - if ( !rootUser) - cmd = "sudo chmod 644 " + mycnfFile + " >/dev/null 2>&1"; - system(cmd.c_str()); cmd = "chown mysql:mysql " + mycnfFile + " >/dev/null 2>&1"; - if ( !rootUser) - cmd = "sudo chown mysql:mysql " + mycnfFile + " >/dev/null 2>&1"; - system(cmd.c_str()); // restart mysql @@ -5972,10 +5934,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) { string cmd; - if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; - else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); @@ -6384,7 +6343,7 @@ int ProcessMonitor::checkDataMount() if ( !rootUser) { - cmd = "sudo chown -R " + USER + ":" + USER + " " + dbroot + " > /dev/null"; + cmd = "chown -R " + USER + ":" + USER + " " + dbroot + " > /dev/null"; system(cmd.c_str()); } @@ -6558,16 +6517,8 @@ int ProcessMonitor::glusterAssign(std::string dbrootID) moduleIPAddr = sysConfig->getConfig("SystemModuleConfig", dataDupIPaddr); } - if ( rootUser) - { - command = "mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + + command = "mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterAssign.txt 2>&1"; - } - else - { - command = "sudo mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + - dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterAssign.txt 2>&1"; - } int ret = system(command.c_str()); @@ -6607,14 +6558,7 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) log.writeLog(__LINE__, "glusterUnassign called: " + dbrootID, LOG_TYPE_DEBUG); - if ( rootUser) - { - command = "umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterUnassign.txt 2>&1"; - } - else - { - command = "sudo umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterUnassign.txt 2>&1"; - } + command = "umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterUnassign.txt 2>&1"; int ret = system(command.c_str()); diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 341e37748..9ee4797ec 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -268,7 +268,7 @@ checkLocalDir() echo "" #remove any check tmp files from previous runs - `sudo rm -f /tmp/*_check > /dev/null 2>&1` + `rm -f /tmp/*_check > /dev/null 2>&1` #check /tmp and /dev/shm pass=true @@ -346,10 +346,10 @@ checkRemoteDir() # # remove old _check tmp files from remote servers - `sudo rm -f /tmp/*_check > /dev/null 2>&1` + `rm -f /tmp/*_check > /dev/null 2>&1` for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'sudo rm -f /tmp/*_check > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f /tmp/*_check > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1` done if [ "$USER" != "root" ]; then @@ -625,7 +625,7 @@ checkPorts() pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `sudo nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > /tmp/port_test` + `nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > /tmp/port_test` if [ "$?" -ne 0 ]; then echo $ipadd " Node Passed port test" else @@ -736,7 +736,7 @@ checkPackages() echo "** Run MariaDB ColumnStore Dependent Package Check" echo "" - declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools") + declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "libaio" "rsync" "snappy" "net-tools") declare -a CENTOS_PKG_NOT=("mariadb-libs") if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then @@ -855,7 +855,7 @@ checkPackages() fi fi - declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools") + declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "libaio1" "rsync" "libsnappy1" "net-tools") declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18") if [ "$OS" == "suse12" ]; then @@ -946,7 +946,7 @@ checkPackages() fi fi - declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools") + declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools") declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") if [ "$OS" == "ubuntu16" ] ; then @@ -1063,7 +1063,7 @@ checkPackages() fi fi - declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools") + declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline-dev" "rsync" "libsnappy1" "net-tools") declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian8" ]; then @@ -1180,7 +1180,7 @@ checkPackages() fi fi - declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") + declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") declare -a DEBIAN9_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian9" ]; then From 6613a9f7e7d955e7b7c19666d0f69fd94162b222 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 2 May 2018 10:37:25 -0500 Subject: [PATCH 002/127] MCOL-520 - nonroot changes without sudo --- oam/install_scripts/columnstore | 2 +- oam/install_scripts/module_installer.sh | 39 ++++++++----------------- oam/install_scripts/post-install | 2 -- oam/install_scripts/pre-uninstall | 30 ++----------------- oamapps/alarmmanager/alarmmanager.cpp | 33 +-------------------- oamapps/postConfigure/postConfigure.cpp | 5 +++- 6 files changed, 21 insertions(+), 90 deletions(-) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index 4568c98ec..bf50e7cc0 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -76,7 +76,7 @@ start() { exit 0 fi - (mkdir -p /var/lock/subsys && chmod 777 /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1 + (mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1 if [ -x $InstallDir/bin/columnstore.pre-start ]; then $InstallDir/bin/columnstore.pre-start diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 457edb9ac..3a690a229 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -67,7 +67,6 @@ if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then rm -f /etc/fstab.columnstoreSave cp /etc/fstab /etc/fstab.columnstoreSave cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab - fi fi fi fi @@ -88,7 +87,6 @@ if [ $module = "um" ]; then mkdir -p $COLUMNSTORE_INSTALL_DIR/mysql/db > /dev/null 2>&1 mount $device $COLUMNSTORE_INSTALL_DIR/mysql/db -t ext2 -o noatime,nodiratime,noauto chown mysql:mysql -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1 - fi fi fi fi @@ -150,16 +148,18 @@ if [ $module = "um" ] || ( [ $module = "pm" ] && [ $PMwithUM = "y" ] ) || [ $Ser fi fi -$COLUMNSTORE_INSTALL_DIR/bin/syslogSetup.sh check > /tmp/syslogSetup-check.log 2>&1 -if [ $? -ne 0 ]; then - # try setup again - $COLUMNSTORE_INSTALL_DIR/bin/syslogSetup.sh install > /tmp/syslogSetup-install.log 2>&1 - if [ $? -ne 0 ]; then - echo "WARNING: syslogSetup.sh check failed: check /tmp/syslogSetup-check.log" - exit 2 - fi +if [ $user == "root" ]; then + $COLUMNSTORE_INSTALL_DIR/bin/syslogSetup.sh check > /tmp/syslogSetup-check.log 2>&1 + if [ $? -ne 0 ]; then + # try setup again + $COLUMNSTORE_INSTALL_DIR/bin/syslogSetup.sh install > /tmp/syslogSetup-install.log 2>&1 + if [ $? -ne 0 ]; then + echo "WARNING: syslogSetup.sh check failed: check /tmp/syslogSetup-check.log" + exit 2 + fi + fi fi - + #setup rc.local if [ -f /etc/rc.d ]; then RCFILE=/etc/rc.d/rc.local @@ -168,24 +168,9 @@ else fi touch $RCFILE -echo "add deadline to rc.local" -if [ $module = "um" ]; then - echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - echo "done" >> $RCFILE -else - echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - echo "done" >> $RCFILE - - echo "for scsi_dev in \`mount | awk '/columnstore\\/data/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE - echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE - echo "done" >> $RCFILE -fi - if [ $user != "root" ]; then echo "uncomment runuser in rc.local" - sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 + sed -i -e 's/#MCS//g' /etc/rc.d/rc.local >/dev/null 2>&1 fi echo "!!!Module Installation Successfully Completed!!!" diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 757116788..e69689126 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -239,8 +239,6 @@ else sed -i -e s@prefix=/home/guest@prefix=$prefix@g $installdir/bin/* chown $user:$user $installdir/etc/Columnstore.xml - mkdir /var/lock/subsys > /dev/null 2>&1 - chmod 777 /var/lock/subsys > /dev/null 2>&1 rm -f /var/lock/subsys/mysql-Columnstore fi diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 56d2fbb3e..29f3915a2 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -59,17 +59,10 @@ rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1 # remove Mariab Columnstore Log Rotate File abd Transaction Log File rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1 - rm -f /etc/cron.d/transactionLog > /dev/null 2>&1 # delete Mariab Columnstore shared memory segments $installdir/bin/clearShm > /dev/null 2>&1 -# delete prat files - rm -f /etc/cron.d/ps > /dev/null 2>&1 - rm -f /etc/pscollect > /dev/null 2>&1 - /etc/init.d/crond reload > /dev/null 2>&1 - systemctl reload crond.service > /dev/null 2>&1 - # delete tmp files rm -f $installdir/local/*.columnstore rm -rf $installdir/local/etc/ @@ -77,7 +70,7 @@ rm -rf /tmp/bucketreuse rm -f /tmp/columnstore.txt rm -f /tmp/dbbuilder.* rm -f /tmp/dbrmfiles - rm -f /var/lock/subsys/columnstore +rm -f /var/lock/subsys/columnstore rm -f /tmp/pkgcheck rm -f /tmp/upgrade-status.log.* rm -f /tmp/mount.log @@ -85,13 +78,6 @@ rm -f $installdir/data/bulk/tmpjob/* >/dev/null 2>&1 rm -rf /tmp/columnstore_tmp_files rm -f $installdir/local/moveDbrootTransactionLog -#delete any old setenv files under home directory -if [ $user = "root" ]; then - rm -f /root/setenv-hdfs* -else - rm -f /home/$user/setenv-hdfs* -fi - # delete core files #rm -f /var/log/mariadb/columnstore/corefiles/* > /dev/null 2>&1 @@ -99,6 +85,7 @@ fi if [ -x $installdir/bin/syslogSetup.sh ]; then if [ $user = "root" ]; then $installdir/bin/syslogSetup.sh uninstall >/dev/null 2>&1 + fi fi #remove the start service command @@ -135,11 +122,7 @@ if [ $quiet != 1 ]; then fi #umount file systems -if [ $user = "root" ]; then - rm -f $installdir/data*/OAMdbrootCheck > /dev/null 2>&1 -else - rm -f $installdir/data*/OAMdbrootCheck > /dev/null 2>&1 -fi +rm -f $installdir/data*/OAMdbrootCheck > /dev/null 2>&1 #remove library paths if [ $user = "root" ]; then @@ -149,13 +132,6 @@ else rm -f /etc/defaults/columnstore fi -#cleanup profile file -if [ $user = "root" ]; then - rm -f /etc/profile.d/columnstore* -else - rm -f /etc/profile.d/columnstore* -fi - #tell user to run post configure script echo " " echo "Mariab Columnstore uninstall completed" diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index 6cea84596..1775bafc7 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -505,39 +505,8 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st // shutdown connection procmgr.shutdown(); } - catch (std::runtime_error& e) - { - LoggingID lid(11); - MessageLog ml(lid); - Message msg; - Message::Args args; - args.add("sendAlarmReport error:"); - args.add(e.what()); - msg.format(args); - ml.logErrorMessage(msg); - } - catch (std::exception& e) - { - LoggingID lid(11); - MessageLog ml(lid); - Message msg; - Message::Args args; - args.add("sendAlarmReport error:"); - args.add(e.what()); - msg.format(args); - ml.logErrorMessage(msg); - } catch (...) - { - LoggingID lid(11); - MessageLog ml(lid); - Message msg; - Message::Args args; - args.add("sendAlarmReport error:"); - args.add("general failure"); - msg.format(args); - ml.logErrorMessage(msg); - } + {} return; #endif //SKIP_ALARM diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 7fb1a0fce..281ef1c39 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1190,7 +1190,7 @@ int main(int argc, char* argv[]) // setup to start on reboot, for non-root amazon installs if ( !rootUser ) { - system("sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); + system("sed -i -e 's/#MCS//g' /etc/rc.d/rc.local >/dev/null 2>&1"); system("chmod 666 /etc/fstab >/dev/null 2>&1"); } @@ -4265,6 +4265,9 @@ bool uncommentCalpontXml( string entry) */ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall) { + + return true; + vector lines; string mount1; From faef2f820a8bf7373a2a731812adf096958feea3 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 23 May 2018 10:16:58 -0500 Subject: [PATCH 003/127] change lock directory --- dbcon/mysql/mysql-Columnstore | 11 ++++++++++- oam/install_scripts/columnstore | 24 +++++++++++++++++------- oam/oamcpp/liboamcpp.cpp | 21 ++++++++++++++++++--- oam/oamcpp/liboamcpp.h | 1 + 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore index 0aef31751..5b7e69cd7 100755 --- a/dbcon/mysql/mysql-Columnstore +++ b/dbcon/mysql/mysql-Columnstore @@ -62,9 +62,18 @@ datadir=$basedir/db # 0 means don't wait at all # Negative numbers mean to wait indefinitely service_startup_timeout=90 +user=`whoami 2>/dev/null` -# Lock directory for RedHat / SuSE. +# Lock directory for root user lockdir='/var/lock/subsys' +# Lock directory for non-root user +if $user != "root" +then + lockdir='$HOME/.lock' +fi + +mkdir $lockdir + lock_file_path="$lockdir/mysql-Columnstore" # The following variables are only set for letting mysql.server find things. diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index bf50e7cc0..5addd4c6a 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -58,12 +58,22 @@ fi user=`whoami 2>/dev/null` +# Lock directory for root user +lockdir='/var/lock/subsys' +# Lock directory for non-root user +if $user != "root" +then + lockdir='$HOME/.lock' +fi + +mkdir $lockdir + checkInstallSetup() { InitialInstallFlag=`$InstallDir/bin/getConfig -c $InstallDir/etc/Columnstore.xml Installation InitialInstallFlag` if [ $InitialInstallFlag != "y" ]; then echo "Please run the postConfigure install script, check the Installation Guide" echo "for additional details" - rm -f /var/lock/subsys/columnstore + rm -f $lockdir/columnstore exit 1 fi } @@ -71,18 +81,18 @@ checkInstallSetup() { [ -f $InstallDir/bin/ProcMon ] || exit 0 start() { - if [ -f /var/lock/subsys/columnstore ]; then + if [ -f $lockdir/columnstore ]; then echo "MariaDB Columnstore Database Platform already running" exit 0 fi - (mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1 + (mkdir -p $lockdir//subsys && touch $lockdir/columnstore) >/dev/null 2>&1 if [ -x $InstallDir/bin/columnstore.pre-start ]; then $InstallDir/bin/columnstore.pre-start if [ $? -ne 0 ]; then echo "Error running MariaDB Columnstore pre-start script, not starting MariaDB Columnstore" - rm -f /var/lock/subsys/columnstore + rm -f $lockdir/columnstore exit 1 fi fi @@ -111,7 +121,7 @@ stop() { sleep 1 $InstallDir/bin/clearShm RETVAL=$? - rm -f /var/lock/subsys/columnstore + rm -f $lockdir/columnstore fuser -k 8604/tcp > /dev/null 2>&1 test -f $InstallDir/mysql/mysql-Columnstore || return $RETVAL $InstallDir/mysql/mysql-Columnstore stop > /dev/null 2>&1 @@ -127,7 +137,7 @@ restart() { status() { isrunning=0 if [ $EUID -eq 0 ]; then - if [ -f /var/lock/subsys/columnstore ]; then + if [ -f $lockdir/columnstore ]; then isrunning=1 fi else @@ -155,7 +165,7 @@ restart) restart ;; condrestart) - [ -f /var/lock/subsys/columnstore ] && restart || : + [ -f $lockdir/columnstore ] && restart || : ;; status) status diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index dfe3822be..9453409a9 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -147,6 +147,19 @@ Oam::Oam() } catch (...) {} // defaulted to false } + + //get user + string USER = "root"; + char* p = getenv("USER"); + + if (p && *p) + USER = p; + + userDir = USER; + + if ( USER != "root") + userDir = "home/" + USER; + } Oam::~Oam() @@ -10669,10 +10682,12 @@ bool Oam::checkSystemRunning() // string cmd = startup::StartUp::installDir() + "/bin/columnstore status > /tmp/status.log"; // system(cmd.c_str()); struct stat st; - - if (stat("/var/lock/subsys/columnstore", &st) == 0) + + string lockFile = userDir + "/columnstore"; + + if (stat(lockFile.c_str(), &st) == 0) { - return true; + return true; } if (geteuid() != 0) diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 66482bc25..e82771e47 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -2523,6 +2523,7 @@ private: std::string ProcessConfigFile; std::string InstallDir; static int UseHdfs; + std::string userDir; }; // end of class From e70c9479775ee7c940cb08eb0506ec0d39694d85 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 24 May 2018 10:33:46 -0500 Subject: [PATCH 004/127] change to use local lock file directory for nonroot --- dbcon/mysql/mysql-Columnstore | 11 ++--------- oam/etc/Columnstore.xml | 1 + oam/etc/Columnstore.xml.singleserver | 1 + oam/install_scripts/columnstore | 12 +----------- oam/install_scripts/post-install | 14 ++++++++++++++ oam/install_scripts/pre-uninstall | 5 ++++- oam/oamcpp/liboamcpp.cpp | 12 +++++++++++- tools/configMgt/autoConfigure.cpp | 21 +++++++++++++++++++++ 8 files changed, 55 insertions(+), 22 deletions(-) diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore index 5b7e69cd7..9f130d5dc 100755 --- a/dbcon/mysql/mysql-Columnstore +++ b/dbcon/mysql/mysql-Columnstore @@ -64,15 +64,8 @@ datadir=$basedir/db service_startup_timeout=90 user=`whoami 2>/dev/null` -# Lock directory for root user -lockdir='/var/lock/subsys' -# Lock directory for non-root user -if $user != "root" -then - lockdir='$HOME/.lock' -fi - -mkdir $lockdir +# Lock directory +lockdir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation LockFileDirectory` lock_file_path="$lockdir/mysql-Columnstore" diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 70a0add6d..cb91f9348 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -469,6 +469,7 @@ 3306 /dev/xvd y + /var/lock/subsys - - - - 0.0.0.0 - 8601 - unassigned - - - 0.0.0.0 - 8602 - - - 0.0.0.0 - 8603 - - - 0.0.0.0 - 8604 - - - 0.0.0.0 - 8605 - - - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8800 - - - 0.0.0.0 - 8800 - - - 0.0.0.0 - 8800 - - - 0.0.0.0 - 8622 - - - 0.0.0.0 - 8622 - - - 0.0.0.0 - 8622 - - - 0.0.0.0 - 8630 - - - 0.0.0.0 - 8612 - - - 0.0.0.0 - 8614 - - - 10000 - - - 1 - 8 - 128 - 10K - 0 - 13 - 512 - 512 - - 1 - 0 - y - - - - y - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - 0.0.0.0 - 8620 - - - C - columnstore-1 - pm1 - unassigned - unassigned - 1 - 3 - 6 // 2.5 minutes - - - 1 - $INSTALLDIR/data1 - $INSTALLDIR/data1/systemFiles/dbrm/BRM_saves - $INSTALLDIR/data1/systemFiles/dbrm/tablelocks - 20 - 100000 - 90 - 80 - 70 - /tmp - /tmp - 10 - 0.0.0.0 - 128M - 10 - /tmp/columnstore_tmp_files - $INSTALLDIR - 10 - 120 - restartSystem - n - 95 - OFF - - /tmp/rdwrscratch - - - dm - Director Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned - um - User Module - SIMPLEX - 1 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned - pm - Performance Module - SIMPLEX - 1 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - 1 - 1 - - - 0 - unassigned - 0.0.0.0 - ENABLED - - - 1000 - $INSTALLDIR/data1/systemFiles/dbrm/SMTxnID - - - - 1GB - - - - $INSTALLDIR/data1/systemFiles/dbrm/oidbitmap - - 3000 - - - $INSTALLDIR/data/bulk - $INSTALLDIR/data1/systemFiles/bulkRollback - 98 - 1 - - - 1 - 0.0.0.0 - 8616 - - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - - - - 1 - 0 - 0 - 65536 - 2K - 200 - 0 - - - n - n - n - 2 - n - n - internal - internal - rpm - unassigned - mariadb1 - unassigned - unassigned - unassigned - us-east-1 - unassigned - unassigned - autoassign - unassigned - unassigned - unassigned - standard - unassigned - unassigned - unassigned - standard - unassigned - y - y - 0 - unassigned - unassigned - n - 0 - unassigned - n - unassigned - unassigned - 3306 - unassigned - /dev/xvd - - - - 4 - 2 - 0x0 - - - 128 - 128K - 128 - 64M - 50% - 10% - 100 - N - - $INSTALLDIR/tmp - Y - - - 16K - 32 - 1 - - - 20 - - - 1M - 1M - 512 - - - - - - - - unassigned - 3306 - unassigned - - - - N - - - N - - - Y - - - 127.0.0.1 - 0 - - - diff --git a/oam/etc/Calpont.xml.singleserver b/oam/etc/Calpont.xml.singleserver deleted file mode 100644 index eaed66037..000000000 --- a/oam/etc/Calpont.xml.singleserver +++ /dev/null @@ -1,530 +0,0 @@ - - - - - 127.0.0.1 - 8601 - pm1 - - - 0.0.0.0 - 8602 - - - 127.0.0.1 - 8603 - - - 127.0.0.1 - 8604 - - - 0.0.0.0 - 8605 - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8800 - - - 0.0.0.0 - 8800 - - - 127.0.0.1 - 8800 - - - 0.0.0.0 - 8622 - - - 0.0.0.0 - 8622 - - - 127.0.0.1 - 8622 - - - 127.0.0.1 - 8630 - - - 127.0.0.1 - 8612 - - - 127.0.0.1 - 8614 - - - 10000 - - - 1 - 8 - 128 - - 10K - 0 - 13 - 512 - 512 - - 1 - 0 - n - - - - 95 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - 127.0.0.1 - 8620 - - - C - columnstore-1 - pm1 - unassigned - - - 1 - $INSTALLDIR/data1 - $INSTALLDIR/data1/systemFiles/dbrm/BRM_saves - $INSTALLDIR/data1/systemFiles/dbrm/tablelocks - 20 - 100000 - 90 - 80 - 70 - /tmp - /tmp - 10 - 0.0.0.0 - 128M - 10 - /tmp/columnstore_tmp_files - $INSTALLDIR - 10 - 3 - 10 - 120 - restartSystem - n - 95 - OFF - - /tmp/rdwrscratch - - - dm - Director Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned - um - User Module - SIMPLEX - 0 - 0.0.0.0 - unassigned - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - unassigned - unassigned - pm - Performance Module - SIMPLEX - 1 - 127.0.0.1 - localhost - ENABLED - 0 - 0 - 0 - 0 - 90 - 80 - 70 - 90 - 0 - 0 - 90 - 80 - 70 - / - 1 - 1 - - - 0 - unassigned - 0.0.0.0 - ENABLED - - - 1000 - $INSTALLDIR/data1/systemFiles/dbrm/CalpontShm - $INSTALLDIR/data1/systemFiles/dbrm/SMTxnID - - - $INSTALLDIR/data1/systemFiles/dbrm/CalpontSessionMonitorShm - 10 - - - - 1GB - - - - - $INSTALLDIR/data1/systemFiles/dbrm/oidbitmap - - 3000 - - - $INSTALLDIR/data/bulk - $INSTALLDIR/data/bulk/rollback - 98 - 1 - - - 1 - 127.0.0.1 - 8616 - - - - 127.0.0.1 - 8700 - pm1 - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - 0.0.0.0 - 8700 - unassigned - - - - 50 - - 1 - 0 - 0 - 65536 - 2K - 200 - 0 - - - n - y - y - 2 - n - n - internal - internal - unassigned - unassigned - unassigned - rpm - unassigned - unassigned - us-east-1 - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - unassigned - n - n - 0 - unassigned - unassigned - n - unassigned - unassigned - 3306 - unassigned - - - - 4 - 2 - 0x0 - - - 128 - 128K - 128 - 64M - 25% - 10% - 100 - N - - $INSTALLDIR/tmp - Y - - - 16K - 32 - 1 - - - 20 - - - 1M - 1M - 512 - - - - - - - - unassigned - 3306 - unassigned - - - - N - - - N - - - Y - - - 127.0.0.1 - 0 - - diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 8cc52a1c7..8474b67d0 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -469,6 +469,7 @@ /dev/xvd y /var/lock/subsys + unassigned 10 0.0.0.0 128M 10 - /tmp/columnstore_tmp_files $INSTALLDIR 10 120 @@ -266,7 +267,8 @@ - /tmp/rdwrscratch + rdwrscratch + /columnstore_tmp_files dm diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index c18b0913e..9b2b939a8 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -174,8 +174,11 @@ if [ $user != "root" ]; then mkdir $tmpDir >/dev/null 2>&1 fi +TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` +tmpDir=$tmpDir+$TempFileDir + #create columnstore temp file directory -mkdir -p $tmpDir/columnstore_tmp_files >/dev/null 2>&1 +mkdir -p $tmpDir >/dev/null 2>&1 #create mount directories mkdir /mnt/tmp > /dev/null 2>&1 diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 5b82a1275..52571aacf 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -69,6 +69,9 @@ if [ $user == "root" ]; then fi rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1\ +# delete Mariab Columnstore shared memory segments +$installdir/bin/clearShm > /dev/null 2>&1 + #get temp directory tmpDir="/tmp" if [ $user != "root" ]; then @@ -76,21 +79,14 @@ if [ $user != "root" ]; then mkdir $tmpDir >/dev/null 2>&1 fi -# delete Mariab Columnstore shared memory segments -$installdir/bin/clearShm > /dev/null 2>&1 +TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` +tmpDir=$tmpDir+$TempFileDir # delete tmp files +rm -rf $tmpDir rm -f $installdir/local/*.columnstore rm -rf $installdir/local/etc/ -rm -rf $tmpDir/bucketreuse -rm -f $tmpDir/columnstore.txt -rm -f $tmpDir/dbbuilder.* -rm -f $tmpDir/dbrmfiles -rm -f $tmpDir/pkgcheck -rm -f $tmpDir/upgrade-status.log.* -rm -f $tmpDir/mount.log rm -f $installdir/data/bulk/tmpjob/* >/dev/null 2>&1 -rm -rf $tmpDir/columnstore_tmp_files rm -f $installdir/local/moveDbrootTransactionLog lockdir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation LockFileDirectory` diff --git a/oamapps/serverMonitor/main.cpp b/oamapps/serverMonitor/main.cpp index 6908cbe86..e3fb8e511 100644 --- a/oamapps/serverMonitor/main.cpp +++ b/oamapps/serverMonitor/main.cpp @@ -42,11 +42,8 @@ int main (int argc, char** argv) { ServerMonitor serverMonitor; Oam oam; - - string TempFileDir; - oam.getSystemConfig("TempFileDir", TempFileDir); - tmpDir = startup::StartUp::tmpDir() + TempFileDir; + tmpDir = startup::StartUp::tmpDir(); struct sigaction ign; diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 7822bf891..4992a1d15 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -217,7 +217,9 @@ void IDBPolicy::configIDBPolicy() // Directory in which to place file buffer temporary files. string tmpDir = startup::StartUp::tmpDir(); - string hdfsRdwrScratch = tmpDir + "/rdwrscratch"; + + string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); + hdfsRdwrScratch = tmpDir + hdfsRdwrScratch; IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); diff --git a/utils/startup/installdir.cpp b/utils/startup/installdir.cpp index 19583c6e5..663ac74b3 100644 --- a/utils/startup/installdir.cpp +++ b/utils/startup/installdir.cpp @@ -31,6 +31,10 @@ using namespace boost; #include "idbregistry.h" #endif #include "installdir.h" +#include "configcpp.h" + +using namespace config; + namespace startup { @@ -92,6 +96,18 @@ const string StartUp::tmpDir() *fTmpDirp = cfStr; #else + + Config* sysConfig = Config::makeConfig(); + + string TempFileDir; + + try + { + TempFileDir = sysConfig->getConfig("SystemConfig", "TempFileDir"); + } + catch (...) + {} + fTmpDirp = new string("/tmp"); //See if we can figure out the tmp dir in Linux... //1. env var COLUMNSTORE_INSTALL_DIR @@ -108,6 +124,8 @@ const string StartUp::tmpDir() *fTmpDirp = homedir + "/.tmp"; } + + *fTmpDirp = *fTmpDirp + TempFileDir; #endif diff --git a/writeengine/shared/we_config.cpp b/writeengine/shared/we_config.cpp index 07554283e..63393cd9c 100644 --- a/writeengine/shared/we_config.cpp +++ b/writeengine/shared/we_config.cpp @@ -263,7 +263,10 @@ void Config::checkReload( ) } // Directory in which to place file buffer temporary files. + string TmpFileDir = startup::StartUp::tmpDir(); + string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); + hdfsRdwrScratch = TmpFileDir + hdfsRdwrScratch if ( hdfsRdwrScratch.length() == 0 ) { diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index e64d9a4f9..83044fa45 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -1418,7 +1418,7 @@ std::string WECmdArgs::getTmpFileDir() { if (!fTmpFileDir.empty()) return fTmpFileDir; - fTmpFileDir = startup::StartUp::tmpDir() + "columnstore_tmp_files"; + fTmpFileDir = startup::StartUp::tmpDir(); if (fTmpFileDir.empty()) throw( runtime_error("Config ERROR: TmpFileDir not found!!")); From 72f514ca540b66d29c5480b757c1946e0145e2b9 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 21 Sep 2018 14:44:12 -0500 Subject: [PATCH 021/127] MCOL-520 --- dbcon/execplan/calpontsystemcatalog.cpp | 5 +- dbcon/execplan/sessionmonitor.cpp | 11 ++- dbcon/mysql/ha_calpont_execplan.cpp | 8 +- dbcon/mysql/ha_calpont_impl.cpp | 18 ++-- dbcon/mysql/install_calpont_mysql.sh | 4 +- dbcon/mysql/sm.cpp | 7 +- oam/etc/Columnstore.xml | 2 +- oam/install_scripts/post-mysql-install | 8 +- oamapps/postConfigure/helpers.cpp | 40 ++++++--- oamapps/postConfigure/postConfigure.cpp | 109 ++++++++++++++---------- 10 files changed, 138 insertions(+), 74 deletions(-) diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 15ccd73aa..cf11b6018 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -66,6 +66,8 @@ using namespace logging; #include "rowgroup.h" using namespace rowgroup; +#include "installdir.h" + #include #include #include @@ -92,7 +94,8 @@ throw runtime_error("CALPONT_INTERNAL_ERROR"); \ #if CSC_DEBUG namespace { -std::ofstream csclog("/tmp/csc.log", std::ios::app); + string tmpDir = startup::StartUp::installDir() + "csc.log"; + std::ofstream csclog(tmpDir, std::ios::app); } #define DEBUG csclog #else diff --git a/dbcon/execplan/sessionmonitor.cpp b/dbcon/execplan/sessionmonitor.cpp index c3a89c4ce..0a27b5925 100644 --- a/dbcon/execplan/sessionmonitor.cpp +++ b/dbcon/execplan/sessionmonitor.cpp @@ -54,6 +54,9 @@ using namespace messageqcpp; using namespace BRM; +#include "installdir.h" + + namespace execplan { @@ -102,8 +105,12 @@ SessionMonitor::SessionMonitor() if (stmp != "") fSegmentFilename = strdup(stmp.c_str()); else - fSegmentFilename = strdup("/tmp/CalpontSessionMonitorShm"); - + { + string tmpdir = startup::StartUp::installDir() + "CalpontSessionMonitorShm"; + + fSegmentFilename = strdup(tmpdir); + } + try { madeSems = getSems(); diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 789d85ad3..2fa2150bf 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -6195,7 +6195,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i if (filters) { csep->filters(filters); - filters->drawTree("/tmp/filter1.dot"); + std::string aTmpDir(startup::StartUp::tmpDir()); + aTmpDir = aTmpDir + "/filter1.dot"; + filters->drawTree(aTmpDir); } gwi.clauseType = SELECT; @@ -8620,7 +8622,9 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro { csep->filters(filters); #ifdef DEBUG_WALK_COND - filters->drawTree("/tmp/filter1.dot"); + std::string aTmpDir(startup::StartUp::tmpDir()); + aTmpDir = aTmpDir + "/filter1.dot"; + filters->drawTree(aTmpDir); #endif } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index ad83f48db..792114c9b 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -3056,7 +3056,9 @@ int ha_calpont_impl_rnd_init(TABLE* table) #ifdef PLAN_HEX_FILE // plan serialization - ifstream ifs("/tmp/li1-plan.hex"); + string tmpDir = aTmpDir + "/li1-plan.hex"; + + ifstream ifs(tmpDir); ByteStream bs1; ifs >> bs1; ifs.close(); @@ -3982,6 +3984,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) //set up for cpimport std::vector Cmds; std::string aCmdLine(startup::StartUp::installDir()); + std::string aTmpDir(startup::StartUp::tmpDir()); + //If local module type is not PM and Local PM query is set, error out char escapechar[2] = ""; @@ -4390,6 +4394,8 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) { THD* thd = current_thd; + std::string aTmpDir(startup::StartUp::tmpDir()); + if (!thd->infinidb_vtable.cal_conn_info) thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); @@ -4516,7 +4522,7 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) rc = 1; ifstream dmlFile; ostringstream oss; - oss << "/tmp/" << ci->tableOid << ".txt"; + oss << aTmpDir << ci->tableOid << ".txt"; dmlFile.open(oss.str().c_str()); if (dmlFile.is_open()) @@ -4533,7 +4539,7 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) rc = 1; ifstream dmlFile; ostringstream oss; - oss << "/tmp/" << ci->tableOid << ".txt"; + oss << aTmpDir << ci->tableOid << ".txt"; dmlFile.open(oss.str().c_str()); if (dmlFile.is_open()) @@ -4567,7 +4573,7 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) //get extra warning count if any ifstream dmlFile; ostringstream oss; - oss << "/tmp/" << ci->tableOid << ".txt"; + oss << aTmpDir << ci->tableOid << ".txt"; dmlFile.open(oss.str().c_str()); int totalWarnCount = 0; int colWarns = 0; @@ -5235,7 +5241,9 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE #ifdef PLAN_HEX_FILE // plan serialization - ifstream ifs("/tmp/li1-plan.hex"); + string tmpDir = aTmpDir + "/li1-plan.hex"; + + ifstream ifs(tmpDir); ByteStream bs1; ifs >> bs1; ifs.close(); diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 259e2d182..1080e69bc 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -17,6 +17,8 @@ for arg in "$@"; do elif [ `expr -- "$arg" : '--installdir='` -eq 13 ]; then installdir="`echo $arg | awk -F= '{print $2}'`" prefix=`dirname $installdir` + elif [ `expr -- "$arg" : '--tmpdir='` -eq 9 ]; then + tmpdir="`echo $arg | awk -F= '{print $2}'`" else echo "ignoring unknown argument: $arg" 1>&2 fi @@ -24,7 +26,7 @@ done df=$installdir/mysql/my.cnf -$installdir/mysql/bin/mysql --defaults-extra-file=$df --force --user=root mysql 2>/tmp/mysql_install.log < ${tmpdir}mysql_install.log <8G --> - rdwrscratch + /rdwrscratch /columnstore_tmp_files diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 9163c3833..215f7d695 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -7,7 +7,7 @@ # check log for error checkForError() { # check for password error - grep "ERROR 1045" /tmp/mysql_install.log > /tmp/error.check + grep "ERROR 1045" ${tmpdir}/mysql_install.log > /tmp/error.check if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then echo "MySQL Password file missing or incorrect, check .my.cnf file" rm -f /tmp/error.check @@ -55,6 +55,8 @@ for arg in "$@"; do elif [ $(expr -- "$arg" : '--installdir=') -eq 13 ]; then installdir="$(echo $arg | awk -F= '{print $2}')" prefix=$(dirname $installdir) + elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then + tmpdir="$(echo $arg | awk -F= '{print $2}')" else echo "ignoring unknown argument: $arg" 1>&2 fi @@ -100,11 +102,11 @@ if [ -x $installdir/mysql/mysql-Columnstore ]; then sleep 5 # Install various Calpont stuff... - $installdir/mysql/install_calpont_mysql.sh --installdir=$installdir + $installdir/mysql/install_calpont_mysql.sh --installdir=$installdir --tmpdir=$tmpdir checkForError if [ $? -ne 0 ]; then # retry - $installdir/mysql/install_calpont_mysql.sh --installdir=$installdir + $installdir/mysql/install_calpont_mysql.sh --installdir=$installdir --tmpdir=$tmpdir checkForError if [ $? -ne 0 ]; then echo "ERROR: Invalid password in .my.cnf, or Columnstore plugin install missing" diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index f7d418144..0392979ee 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -35,6 +35,10 @@ using namespace oam; #include "helpers.h" +using namespace installer; + +#include "installdir.h" + extern string mysqlpw; string pwprompt = " "; @@ -45,6 +49,7 @@ string prompt; const char* pcommand = 0; extern string installDir; + extern bool noPrompting; namespace installer @@ -247,12 +252,15 @@ void mysqlSetup() { Oam oam; string cmd; - cmd = installDir + "/bin/post-mysqld-install --installdir=" + installDir + " > /tmp/post-mysqld-install.log 2>&1"; + + string tmpDir = startup::StartUp::tmpDir(); + + cmd = installDir + "/bin/post-mysqld-install --installdir=" + installDir + " > " + tmpDir + "/post-mysqld-install.log 2>&1"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) { - cout << "Error running post-mysqld-install, check /tmp/post-mysqld-install.log" << endl; + cout << "Error running post-mysqld-install, check " << tmpDir << "/post-mysqld-install.log" << endl; cout << "Exiting..." << endl; exit (1); } @@ -276,7 +284,7 @@ void mysqlSetup() HOME = p; } - cmd = installDir + "/bin/post-mysql-install --installdir=" + installDir + " > /tmp/post-mysql-install.log";; + cmd = installDir + "/bin/post-mysql-install --installdir=" + installDir + " --tmpdir=" + tmpDir + " > " + tmpDir + "/post-mysql-install.log"; rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) == 2) @@ -287,7 +295,7 @@ void mysqlSetup() } else if (WEXITSTATUS(rtnCode) == 1) { - cout << "Error running post-mysql-install, /tmp/post-mysql-install.log" << endl; + cout << "Error running post-mysql-install, " + tmpDir + "/post-mysql-install.log" << endl; cout << "Exiting..." << endl; exit (1); } @@ -375,7 +383,9 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) if ( returnStatus != API_SUCCESS) { - cout << "ERROR: Error return in running the MariDB Columnstore Upgrade, check /tmp/upgrade*.logs on " << (*pt).DeviceName << endl; + string tmpDir = startup::StartUp::tmpDir(); + + cout << "ERROR: Error return in running the MariDB Columnstore Upgrade, check " + tmpDir + "/upgrade*.logs on " << (*pt).DeviceName << endl; return returnStatus; } } @@ -404,6 +414,8 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm SystemModuleTypeConfig systemmoduletypeconfig; + string tmpDir = startup::StartUp::tmpDir(); + try { oam.getSystemConfig(systemmoduletypeconfig); @@ -478,7 +490,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm if ( returnStatus != API_SUCCESS) { - cout << endl << "ERROR: Error return in running the MariaDB ColumnStore Master DB Distribute, check /tmp/master-dist*.logs on " << masterModule << endl; + cout << endl << "ERROR: Error return in running the MariaDB ColumnStore Master DB Distribute, check " + tmpDir + "/master-dist*.logs on " << masterModule << endl; return returnStatus; } @@ -491,7 +503,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm if ( returnStatus != API_SUCCESS) { - cout << endl << "ERROR: Error return in running the MariaDB ColumnStore Master replication, check /tmp/master-rep*.logs on " << masterModule << endl; + cout << endl << "ERROR: Error return in running the MariaDB ColumnStore Master replication, check " + tmpDir + "master-rep*.logs on " << masterModule << endl; return returnStatus; } @@ -526,7 +538,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm if ( returnStatus != API_SUCCESS) { - cout << endl << "ERROR: Error return in running the MariaDB ColumnStore Slave replication, check /tmp/slave-rep*.logs on " << (*pt).DeviceName << endl; + cout << endl << "ERROR: Error return in running the MariaDB ColumnStore Slave replication, check " + tmpDir + "/slave-rep*.logs on " << (*pt).DeviceName << endl; return returnStatus; } @@ -728,12 +740,14 @@ void checkMysqlPort( std::string& mysqlPort, Config* sysConfig ) { Oam oam; + string tmpDir = startup::StartUp::tmpDir(); + while (true) { - string cmd = "netstat -na | grep -e :" + mysqlPort + "[[:space:]] | grep LISTEN > /tmp/mysqlport"; + string cmd = "netstat -na | grep -e :" + mysqlPort + "[[:space:]] | grep LISTEN > " + tmpDir + "/mysqlport"; system(cmd.c_str()); - string fileName = "/tmp/mysqlport"; + string fileName = tmpDir + "/mysqlport"; ifstream oldFile (fileName.c_str()); if (oldFile) @@ -813,9 +827,11 @@ void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string bool inUse = false; + string tmpDir = startup::StartUp::tmpDir(); + while (true) { - string localnetstat = "netstat -na | grep -e :" + mysqlPort + "[[:space:]] | grep LISTEN > /tmp/mysqlport"; + string localnetstat = "netstat -na | grep -e :" + mysqlPort + "[[:space:]] | grep LISTEN > " + tmpDir + "/mysqlport"; string remotenetstat = "netstat -na | grep -e :" + mysqlPort + "[[:space:]] | grep LISTEN"; //first check local mysql, if needed @@ -823,7 +839,7 @@ void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string ( ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM ) && pmwithum ) ) { system(localnetstat.c_str()); - string fileName = "/tmp/mysqlport"; + string fileName = tmpDir + "/mysqlport"; ifstream oldFile (fileName.c_str()); if (oldFile) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index e146417c9..6ffef9012 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -80,6 +80,7 @@ using namespace config; #include "helpers.h" using namespace installer; +#include "installdir.h" typedef struct DBRoot_Module_struct { @@ -195,6 +196,7 @@ bool amazon_quick_install = false; string DataFileEnvFile; string installDir; +string tmpDir; string HOME = "/root"; extern string pwprompt; @@ -287,7 +289,8 @@ int main(int argc, char* argv[]) if (p && *p) HOME = p; } - + + tmpDir = startup::StartUp::tmpDir(); for ( int i = 1; i < argc; i++ ) { @@ -568,18 +571,20 @@ int main(int argc, char* argv[]) EEPackageType = "binary"; else { - int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + string cmd = "rpm -qi mariadb-columnstore-platform > " + tmpDir + "/columnstore.txt 2>&1"; + int rtnCode = system(cmd.c_str()); + + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "rpm"; + else { + string cmd = "dpkg -s mariadb-columnstore-platform > " + tmpDir + "/columnstore.txt 2>&1"; + int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "rpm"; - else { - rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "deb"; - else - EEPackageType = "binary"; - } + EEPackageType = "deb"; + else + EEPackageType = "binary"; + } } try { @@ -701,11 +706,11 @@ int main(int argc, char* argv[]) // run my.cnf upgrade script if ( reuseConfig == "y" ) { - cmd = installDir + "/bin/mycnfUpgrade > /tmp/mycnfUpgrade.log 2>&1"; + cmd = installDir + "/bin/mycnfUpgrade > " + tmpDir + "/mycnfUpgrade.log 2>&1"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) - cout << "Error: Problem upgrade my.cnf, check /tmp/mycnfUpgrade.log" << endl; + cout << "Error: Problem upgrade my.cnf, check " << tmpDir << "/mycnfUpgrade.log" << endl; else cout << "NOTE: my.cnf file was upgraded based on my.cnf.rpmsave" << endl; } @@ -1336,19 +1341,21 @@ int main(int argc, char* argv[]) if (!multi_server_quick_install) { - system("aws --version > /tmp/amazon.log 2>&1"); + string amazonLog = tmpDir + "/amazon.log"; + string cmd = "aws --version > " + amazonLog + " 2>&1"; + int rtnCode = system(cmd.c_str()); - ifstream in("/tmp/amazon.log"); + ifstream in(amazonLog); in.seekg(0, std::ios::end); int size = in.tellg(); - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) + if ( size == 0 || oam.checkLogStatus(amazonLog, "not found")) { // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, bu the Amazon CLI API packages isnt installed, exiting" << endl; exit(1); } @@ -1356,7 +1363,7 @@ int main(int argc, char* argv[]) } else { - if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) + if ( size == 0 || oam.checkLogStatus(amazonLog, "not installed")) { // not running on amazon with ec2-api-tools if (amazon_quick_install) @@ -3614,7 +3621,7 @@ int main(int argc, char* argv[]) if ( remote_installer_debug == "1" ) { - logfile = "/tmp/"; + logfile = tmpDir; logfile += remoteModuleName + "_" + EEPackageType + "_install.log"; debug_logfile = " > " + logfile; } @@ -3921,13 +3928,15 @@ int main(int argc, char* argv[]) if (hdfs && !nonDistribute ) { + string postConfigurePsdhLog = tmpDir + "postConfigure.pdsh.log"; + cout << endl << "----- Starting MariaDB ColumnStore Service on all Modules -----" << endl << endl; - string cmd = "pdsh -a '" + installDir + "/bin/columnstore restart' > /tmp/postConfigure.pdsh 2>&1"; + string cmd = "pdsh -a '" + installDir + "/bin/columnstore restart' > " + postConfigurePsdhLog + " 2>&1"; system(cmd.c_str()); - if (oam.checkLogStatus("/tmp/postConfigure.pdsh", "exit") ) + if (oam.checkLogStatus(postConfigurePsdhLog, "exit") ) { - cout << endl << "ERROR: Starting MariaDB ColumnStore Service failue, check /tmp/postConfigure.pdsh. exit..." << endl; + cout << endl << "ERROR: Starting MariaDB ColumnStore Service failue, check " + postConfigurePsdhLog + ". exit..." << endl; exit (1); } } @@ -4012,27 +4021,28 @@ int main(int argc, char* argv[]) } } } + + string dbbuilderLog = tmpDir + "dbbuilder.log"; if (hdfs) - cmd = "bash -c '. " + installDir + "/bin/" + DataFileEnvFile + ";" + installDir + "/bin/dbbuilder 7 > /tmp/dbbuilder.log'"; + cmd = "bash -c '. " + installDir + "/bin/" + DataFileEnvFile + ";" + installDir + "/bin/dbbuilder 7 > " + dbbuilderLog; else - cmd = installDir + "/bin/dbbuilder 7 > /tmp/dbbuilder.log"; + cmd = installDir + "/bin/dbbuilder 7 > " + dbbuilderLog; system(cmd.c_str()); - if (oam.checkLogStatus("/tmp/dbbuilder.log", "System Catalog created") ) + if (oam.checkLogStatus(dbbuilderLog, "System Catalog created") ) cout << endl << "System Catalog Successfully Created" << endl; else { - if ( oam.checkLogStatus("/tmp/dbbuilder.log", "System catalog appears to exist") ) + if ( oam.checkLogStatus(dbbuilderLog, "System catalog appears to exist") ) { - cout.flush(); } else { cout << endl << "System Catalog Create Failure" << endl; - cout << "Check latest log file in /tmp/dbbuilder.log.*" << endl; + cout << "Check latest log file in " << dbbuilderLog << endl; cout << " IMPORTANT: Once issue has been resolved, rerun postConfigure" << endl << endl; exit (1); @@ -4622,7 +4632,7 @@ bool createDbrootDirs(string DBRootStorageType) // mount data1 and create directories if configured with storage if ( DBRootStorageType == "external" ) { - string cmd = "mount " + installDir + "/data1 > /tmp/mount.txt 2>&1"; + string cmd = "mount " + installDir + "/data1 > " + tmpDir + "/mount.txt 2>&1"; system(cmd.c_str()); if ( !rootUser) @@ -4655,11 +4665,12 @@ bool pkgCheck(string columnstorePackage) { while (true) { - string cmd = "ls " + columnstorePackage + " > /tmp/calpontpkgs"; + string fileName = tmpDir + "/calpontpkgs"; + + string cmd = "ls " + columnstorePackage + " > " + fileName; system(cmd.c_str()); string pkg = columnstorePackage; - string fileName = "/tmp/calpontpkgs"; ifstream oldFile (fileName.c_str()); if (oldFile) @@ -4770,7 +4781,7 @@ bool storageSetup(bool amazonInstall) cout.flush(); string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpDir; string hdfslog = logdir + "/hdfsCheck.log1"; @@ -5087,7 +5098,8 @@ bool storageSetup(bool amazonInstall) //check if gluster is installed int rtnCode = 1; - rtnCode = system("gluster --version > /tmp/gluster.log 2>&1"); + string cmd = "gluster --version > " + tmpDir + "/gluster.log 2>&1"; + rtnCode = system(cmd.c_str()); if (rtnCode == 0) { @@ -5099,14 +5111,17 @@ bool storageSetup(bool amazonInstall) } //check if hadoop is installed - system("which hadoop > /tmp/hadoop.log 2>&1"); + string hadoopLog = tmpDir + "/hadoop.log"; + + cmd = "which hadoop > " + hadoopLog + " 2>&1"; + system(cmd.c_str()); - ifstream in("/tmp/hadoop.log"); + ifstream in(hadoopLog); in.seekg(0, std::ios::end); int size = in.tellg(); - if ( size == 0 || oam.checkLogStatus("/tmp/hadoop.log", "no hadoop")) + if ( size == 0 || oam.checkLogStatus(hadoopLog, "no hadoop")) // no hadoop size = 0; else @@ -5577,7 +5592,7 @@ bool storageSetup(bool amazonInstall) cout.flush(); string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpDir; string hdfslog = logdir + "/hdfsCheck.log1"; @@ -6116,14 +6131,14 @@ void remoteInstallThread(void* arg) { //failure pthread_mutex_lock(&THREAD_LOCK); - cout << endl << "Failure with a remote module install, check install log files in /tmp" << endl; + cout << endl << "Failure with a remote module install, check install log files in " << tmpDir << endl; exit(1); } } } pthread_mutex_lock(&THREAD_LOCK); - cout << endl << "Failure with a remote module install, check install log files in /tmp" << endl; + cout << endl << "Failure with a remote module install, check install log files in " << tmpDir << endl; exit(1); } @@ -6680,8 +6695,10 @@ bool glusterSetup(string password) } sleep(5); + + string glusterCommandsLog = tmpDir + "/glusterCommands.log"; - command = "gluster peer status >> /tmp/glusterCommands.txt 2>&1"; + command = "gluster peer status " + glusterCommandsLog + "2>&1"; status = system(command.c_str()); @@ -6717,13 +6734,13 @@ bool glusterSetup(string password) pmnextbrick[pm]++; } - command += "force >> /tmp/glusterCommands.txt 2>&1"; + command += "force >> " + glusterCommandsLog + " 2>&1"; cout << "Gluster create and start volume dbroot" << oam.itoa(dbrootID) << "..."; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { - if (oam.checkLogStatus("/tmp/glusterCommands.txt", "dbroot" + oam.itoa(dbrootID) + " already exists" )) + if (oam.checkLogStatus(glusterCommandsLog, "dbroot" + oam.itoa(dbrootID) + " already exists" )) { string errmsg1; string errmsg2; @@ -6752,7 +6769,7 @@ bool glusterSetup(string password) if (rootUser) { - command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; + command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> " + glusterCommandsLog + " 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -6765,7 +6782,7 @@ bool glusterSetup(string password) { int user = getuid(); int group = getgid(); - command = "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; + command = "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> " + glusterCommandsLog + " 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -6774,7 +6791,7 @@ bool glusterSetup(string password) exit(1); } - command = "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; + command = "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> " + glusterCommandsLog + " 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -6783,7 +6800,7 @@ bool glusterSetup(string password) exit(1); } - command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; + command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> " + glusterCommandsLog + " 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) From 681cfd34ebd89bdbdf2ac4807c93dce969ce8de6 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 24 Sep 2018 11:50:56 -0500 Subject: [PATCH 022/127] MCOL-520 --- exemgr/main.cpp | 4 +- oam/cloud/IDBCloudGetIP.sh | 44 -- oam/cloud/IDBInstanceCmds.sh | 574 ------------------ oam/cloud/IDBVolumeCmds.sh | 337 ---------- oam/cloud/MCSInstanceCmds.sh | 30 +- oam/cloud/MCSVolumeCmds.sh | 39 +- oam/etc/Columnstore.xml | 1 + oam/etc/ConsoleCmds.xml | 3 +- .../disable-rep-columnstore.sh | 30 +- oam/install_scripts/master-rep-columnstore.sh | 46 +- oam/install_scripts/post-install | 6 +- oam/install_scripts/post-mysql-install | 8 +- oam/install_scripts/pre-uninstall | 9 +- oam/install_scripts/slave-rep-columnstore.sh | 48 +- oam/oamcpp/liboamcpp.cpp | 127 ++-- oamapps/mcsadmin/mcsadmin.cpp | 9 +- procmgr/main.cpp | 5 + procmgr/processmanager.cpp | 32 +- procmon/main.cpp | 7 +- procmon/processmonitor.cpp | 178 +++--- 20 files changed, 292 insertions(+), 1245 deletions(-) delete mode 100644 oam/cloud/IDBCloudGetIP.sh delete mode 100644 oam/cloud/IDBInstanceCmds.sh delete mode 100644 oam/cloud/IDBVolumeCmds.sh diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 8742188a0..05fdc600f 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -100,6 +100,8 @@ using namespace querytele; #include "threadpool.h" #include "crashtrace.h" +#include "installdir.h" + namespace { @@ -1320,7 +1322,7 @@ void setupSignalHandlers() void setupCwd(ResourceManager* rm) { - string workdir = rm->getScWorkingDir(); + string workdir = startup::StartUp::tmpDir(); (void)chdir(workdir.c_str()); if (access(".", W_OK) != 0) diff --git a/oam/cloud/IDBCloudGetIP.sh b/oam/cloud/IDBCloudGetIP.sh deleted file mode 100644 index bca6d41f0..000000000 --- a/oam/cloud/IDBCloudGetIP.sh +++ /dev/null @@ -1,44 +0,0 @@ -#! /bin/sh -# IDBCloudGetIP.sh -# Get IP Address from a Cloud environment -# -# 1. Amazon EC2 - -prefix=/usr/local - -#get instance name from called -instanceName="$1" - -ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME` - -if [ $ec2 == "unassigned" ]; then - echo "stopped" - exit 1 -fi - -java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME` -path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH` - -export PATH=$path -export EC2_HOME=$ec2 -export JAVA_HOME=$java - -# get x509 Certification and Private Key -x509Cert=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonX509Certificate` -x509PriKey=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonX509PrivateKey` - -#get instance info -ec2-describe-instances -C $x509Cert -K $x509PriKey $instanceName > /tmp/instanceInfo 2> /dev/null - -#check if running -cat /tmp/instanceInfo | grep running > /tmp/instanceStatus -if [ `cat /tmp/instanceStatus | wc -c` -eq 0 ]; then - echo "stopped" - exit 1 -fi - -#get priviate IP Address -IpAddr=`cat /tmp/instanceInfo | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $15}'` - -echo $IpAddr -exit 0 diff --git a/oam/cloud/IDBInstanceCmds.sh b/oam/cloud/IDBInstanceCmds.sh deleted file mode 100644 index 1d7351594..000000000 --- a/oam/cloud/IDBInstanceCmds.sh +++ /dev/null @@ -1,574 +0,0 @@ -#! /bin/sh -# IDBInstanceCmds.sh -# get-local-instance-ID, get-zone, getPrivateIP from a Cloud environment -# -# 1. Amazon EC2 - -prefix=/usr/local - -#check command -if [ "$1" = "" ]; then - echo "Enter Command Name: {launchInstance|getInstance|getZone|getPrivateIP|getKey|getAMI|getType|terminateInstance|startInstance|assignElasticIP|deassignElasticIP|getProfile|stopInstance|getGroup|getSubnet} -}" - exit 1 -fi - -if [ "$1" = "getPrivateIP" ]; then - if [ "$2" = "" ]; then - echo "Enter Instance Name" - exit 1 - fi - instanceName="$2" -fi - -if [ "$1" = "launchInstance" ]; then - if [ "$2" = "" ]; then - IPaddress="unassigned" - else - IPaddress="$2" - fi - if [ "$3" = "" ]; then - instanceType="unassigned" - else - instanceType="$3" - fi - if [ "$4" = "" ]; then - group="unassigned" - else - group="$4" - fi -fi - -if [ "$1" = "terminateInstance" ]; then - if [ "$2" = "" ]; then - echo "Enter Instance Name" - exit 1 - fi - instanceName="$2" -fi - -if [ "$1" = "stopInstance" ]; then - if [ "$2" = "" ]; then - echo "Enter Instance Name" - exit 1 - fi - instanceName="$2" -fi - -if [ "$1" = "startInstance" ]; then - if [ "$2" = "" ]; then - echo "Enter Instance Name" - exit 1 - fi - instanceName="$2" -fi - -if [ "$1" = "assignElasticIP" ]; then - if [ "$2" = "" ]; then - echo "Enter Instance Name" - exit 1 - else - instanceName="$2" - fi - if [ "$3" = "" ]; then - echo "Enter Elastic IP Address" - exit 1 - else - IPAddress="$3" - fi -fi - -if [ "$1" = "deassignElasticIP" ]; then - if [ "$2" = "" ]; then - echo "Enter Elastic IP Address" - exit 1 - else - IPAddress="$2" - fi -fi - - -test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions - -ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME` - -if [ $ec2 == "unassigned" ]; then - if [ "$1" = "getPrivateIP" ]; then - echo "stopped" - exit 1 - else - echo "unknown" - exit 1 - fi -fi - -java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME` -path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH` - -export PATH=$path -export EC2_HOME=$ec2 -export JAVA_HOME=$java - -# get Keys and region -AmazonAccessKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonAccessKey` -if [ $AmazonAccessKeyFile == "unassigned" ]; then - echo "FAILED: missing Config Setting AmazonAccessKey : $AmazonAccessKeyfile" - exit 1 -fi - -AmazonSecretKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSecretKey` -if [ $AmazonSecretKeyFile == "unassigned" ]; then - echo "FAILED: missing Config Setting AmazonSecretKeyFile : $AmazonSecretKeyFile" - exit 1 -fi - -AmazonAccessKey=`cat $AmazonAccessKeyFile` -AmazonSecretKey=`cat $AmazonSecretKeyFile` - -Region=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonRegion` -subnet=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSubNetID` - -if test ! -f $AmazonAccessKeyfile ; then - echo "FAILED: missing AmazonAccessKeyfile : $AmazonAccessKeyfile" - exit 1 -fi - -if test ! -f $AmazonSecretKeyfile ; then - echo "FAILED: missing AmazonSecretKeyfile : $AmazonSecretKeyfile" - exit 1 -fi - -#default instance to null -instance="" - -describeInstanceFile="/tmp/describeInstance.txt" -touch $describeInstanceFile - -describeInstance() { - ec2-describe-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region > $describeInstanceFile 2>&1 -} - -#call at start up -describeInstance - -getInstance() { - if [ "$instance" != "" ]; then - echo $instance - return - fi - - # first get local IP Address - localIP=`ifconfig eth0 | grep "inet addr:" | awk '{print substr($2,6,20)}'` - - #get local Instance ID - instance=`cat $describeInstanceFile | grep -m 1 -w $localIP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - if [ "$instance" == "" ]; then - describeInstance - fi - instance=`cat $describeInstanceFile | grep -m 1 -w $localIP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - - echo $instance - return -} - -getInstancePrivate() { - if [ "$instance" != "" ]; then - echo $instance - return - fi - - # first get local IP Address - localIP=`ifconfig eth0 | grep "inet addr:" | awk '{print substr($2,6,20)}'` - - #get local Instance ID - instance=`cat $describeInstanceFile | grep -m 1 -w $localIP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - if [ "$instance" == "" ]; then - describeInstance - fi - instance=`cat $describeInstanceFile | grep -m 1 -w $localIP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - - return -} - - -getZone() { - #get from Columnstore.xml if it's there, if not, get from instance then store - zone=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonZone` - - if [ "$zone" = "unassigned" ] || [ "$zone" = "" ]; then - #get local Instance ID - getInstancePrivate >/dev/null 2>&1 - #get zone - if [ "$subnet" == "unassigned" ]; then - zone=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'` - if [ "$zone" == "" ]; then - describeInstance - fi - zone=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'` - - else - zone=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'` - if [ "$zone" == "" ]; then - describeInstance - fi - zone=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'` - fi - $prefix/mariadb/columnstore/bin/setConfig Installation AmazonZone $zone - fi - - echo $zone - return -} - -getPrivateIP() { - #get instance info - grep -B1 -A4 -m 1 $instanceName $describeInstanceFile > /tmp/instanceInfo_$instanceName 2>&1 - if [ `cat /tmp/instanceInfo_$instanceName | wc -c` -eq 0 ]; then - describeInstance - fi - grep -B1 -A4 -m 1 $instanceName $describeInstanceFile > /tmp/instanceInfo_$instanceName 2>&1 - - #check if running or terminated - cat /tmp/instanceInfo_$instanceName | grep running > /tmp/instanceStatus_$instanceName - if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -eq 0 ]; then - # not running - cat /tmp/instanceInfo_$instanceName | grep pending > /tmp/instanceStatus_$instanceName - if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then - describeInstance - echo "stopped" - exit 1 - else - cat /tmp/instanceInfo_$instanceName | grep terminated > /tmp/instanceStatus_$instanceName - if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then - echo "terminated" - exit 1 - else - cat /tmp/instanceInfo_$instanceName | grep shutting-down > /tmp/instanceStatus_$instanceName - if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then - echo "terminated" - exit 1 - else - echo "stopped" - exit 1 - fi - fi - fi - fi - - #running, get priviate IP Address - if [ "$subnet" == "unassigned" ]; then - IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $15}'` - else - IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $14}'` - fi - - echo $IpAddr - exit 0 -} - -getType() { - #get local Instance ID - getInstancePrivate >/dev/null 2>&1 - #get Type - if [ "$subnet" == "unassigned" ]; then - instanceType=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'` - if [ "$instanceType" == "" ]; then - describeInstance - fi - instanceType=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'` - - else - instanceType=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'` - if [ "$instanceType" == "" ]; then - describeInstance - fi - instanceType=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'` - - fi - - echo $instanceType - return -} - -getKey() { - #get local Instance ID - getInstancePrivate >/dev/null 2>&1 - #get Key - if [ "$subnet" == "unassigned" ]; then - key=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'` - if [ "$key" == "" ]; then - describeInstance - fi - key=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'` - - else - key=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'` - if [ "$key" == "" ]; then - describeInstance - fi - key=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'` - - fi - - echo $key - return -} - -getAMI() { - #get local Instance ID - getInstancePrivate >/dev/null 2>&1 - #get AMI - ami=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $3}'` - if [ "$ami" == "" ]; then - describeInstance - fi - ami=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $3}'` - - echo $ami - return -} - -getGroup() { - #get local Instance ID - getInstancePrivate >/dev/null 2>&1 - #get group - if [ "$subnet" == "unassigned" ]; then - group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile | grep -m 1 RESERVATION | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'` - if [ "$group" == "" ]; then - describeInstance - fi - group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile | grep -m 1 RESERVATION | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'` - if [ "$group" == "" ]; then - group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'` - fi - else - group=`grep -B1 -A6 -m 1 $instance $describeInstanceFile | grep -m 1 GROUP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - if [ "$group" == "" ]; then - describeInstance - fi - group=`grep -B1 -A6 -m 1 $instance $describeInstanceFile | grep -m 1 GROUP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - if [ "$group" == "" ]; then - group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $21}'` - fi - fi - - echo $group - return -} - -getProfile() { - #get local Instance ID - getInstancePrivate >/dev/null 2>&1 - #get Type - if [ "$subnet" == "unassigned" ]; then - instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $23}'` - if [ "$instanceProfile" == "" ]; then - describeInstance - fi - instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $23}'` - - else - instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'` - if [ "$instanceProfile" == "" ]; then - describeInstance - fi - instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'` - - fi - - echo $instanceProfile - return -} - -launchInstance() { - #get publickey - getKey >/dev/null 2>&1 - if [ "$group" = "unassigned" ]; then - #get group - getGroup >/dev/null 2>&1 - fi - #get AMI - getAMI >/dev/null 2>&1 - #get Zone - getZone >/dev/null 2>&1 - if [ "$instanceType" = "unassigned" ]; then - #get type - getType >/dev/null 2>&1 - fi - #get AMI Profile - getProfile >/dev/null 2>&1 - - if [ "$subnet" == "unassigned" ]; then - #NOT VPC - if [ "$instanceProfile" = "" ] || [ "$instanceProfile" = "default" ]; then - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone --region $Region $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - else - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - fi - else # VPC - if [ "$instanceProfile" = "" ] || [ "$instanceProfile" = "default" ]; then - if [ "$group" != "default" ]; then - if [ "$IPaddress" = "autoassign" ]; then - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone --region $Region -s $subnet $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - else - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - fi - else - if [ "$IPaddress" = "autoassign" ]; then - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone --region $Region -s $subnet $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - else - newInstance=`ec2-run-instances --O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - fi - fi - else - if [ "$group" != "default" ]; then - if [ "$IPaddress" = "autoassign" ]; then - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - else - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - fi - else - if [ "$IPaddress" = "autoassign" ]; then - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - else - newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - fi - fi - fi - fi - echo $newInstance - return -} - -terminateInstance() { - #terminate Instance - ec2-terminate-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/termInstanceInfo_$instanceName 2>&1 - return -} - -stopInstance() { - #terminate Instance - ec2-stop-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/stopInstanceInfo_$instanceName 2>&1 - return -} - -startInstance() { - #terminate Instance - ec2-start-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/startInstanceInfo_$instanceName 2>&1 - - cat /tmp/startInstanceInfo_$instanceName | grep INSTANCE > /tmp/startInstanceStatus_$instanceName - if [ `cat /tmp/startInstanceStatus_$instanceName | wc -c` -eq 0 ]; then - echo "Failed, check /tmp/startInstanceInfo_$instanceName" - exit 1 - fi - echo "Success" - exit 0 -} - -assignElasticIP() { - #terminate Instance - - if [ "$subnet" == "unassigned" ]; then - ec2-associate-address -O $AmazonAccessKey -W $AmazonSecretKey -i $instanceName $IPAddress > /tmp/assignElasticIPInfo_$IPAddress 2>&1 - else - EIP=`ec2-describe-addresses -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $IPAddress | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'` - ec2-associate-address -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -i $instanceName -a $EIP > /tmp/assignElasticIPInfo_$IPAddress 2>&1 - fi - - cat /tmp/assignElasticIPInfo_$IPAddress | grep ADDRESS > /tmp/assignElasticIPStatus_$IPAddress - if [ `cat /tmp/assignElasticIPStatus_$IPAddress | wc -c` -eq 0 ]; then - echo "Failed, check /tmp/assignElasticIPInfo_$IPAddress" - exit 1 - fi - - echo "Success" - exit 0 -} - -deassignElasticIP() { - #terminate Instance - ec2-disassociate-address -O $AmazonAccessKey -W $AmazonSecretKey $IPAddress > /tmp/deassignElasticIPInfo_$IPAddress 2>&1 - - cat /tmp/deassignElasticIPInfo_$IPAddress | grep ADDRESS > /tmp/deassignElasticIPStatus_$IPAddress - if [ `cat /tmp/deassignElasticIPStatus_$IPAddress | wc -c` -eq 0 ]; then - echo "Failed, check /tmp/deassignElasticIPStatus_$IPAddress" - exit 1 - fi - - echo "Success" - exit 0 -} - -getSubnet() { - #get local Instance ID - getInstancePrivate >/dev/null 2>&1 - #get Subnet - subnet=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $16}'` - if [ "$subnet" == "" ]; then - describeInstance - fi - subnet=`cat $describeInstanceFile | grep -m 1 $instance | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $16}'` - - if [[ $subnet == *"subnet"* ]] - then - echo $subnet - else - echo "failed" - fi - - return -} - - -case "$1" in - getInstance) - getInstance - ;; - getZone) - getZone - ;; - getPrivateIP) - getPrivateIP - ;; - getKey) - getKey - ;; - getAMI) - getAMI - ;; - getType) - getType - ;; - launchInstance) - launchInstance - ;; - terminateInstance) - terminateInstance - ;; - stopInstance) - stopInstance - ;; - startInstance) - startInstance - ;; - assignElasticIP) - assignElasticIP - ;; - deassignElasticIP) - deassignElasticIP - ;; - getProfile) - getProfile - ;; - getGroup) - getGroup - ;; - getSubnet) - getSubnet - ;; - *) - echo $"Usage: $0 {launchInstance|getInstance|getZone|getPrivateIP|getType|getKey|getAMI|terminateInstance|startInstance|assignElasticIP|deassignElasticIP|getProfile|stopInstance|getGroup|getSubnet}" - exit 1 -esac - -exit $? diff --git a/oam/cloud/IDBVolumeCmds.sh b/oam/cloud/IDBVolumeCmds.sh deleted file mode 100644 index 3d64007b1..000000000 --- a/oam/cloud/IDBVolumeCmds.sh +++ /dev/null @@ -1,337 +0,0 @@ -#! /bin/sh -# IDBVolumeCmds.sh -# describe, detach and attach Volume Storage from a Cloud environment -# -# 1. Amazon EC2 - -prefix=/usr/local - -#check command -if [ "$1" = "" ]; then - echo "Enter Command Name: {create|describe|detach|attach|delete|createTag}" - exit 1 -fi - -if [ "$1" = "create" ]; then - if [ "$2" = "" ]; then - echo "Enter size of the volume, in GiB (1-1024)" - exit 1 - fi - volumeSize="$2" - - #get module-type - if [ "$3" = "" ]; then - echo "Enter Module Type" - exit 1 - fi - moduleType="$3" -fi - -if [ "$1" = "describe" ]; then - if [ "$2" = "" ]; then - echo "Enter Volume Name" - exit 1 - fi - volumeName="$2" -fi - -if [ "$1" = "detach" ]; then - if [ "$2" = "" ]; then - echo "Enter Volume Name" - exit 1 - fi - volumeName="$2" -fi - -if [ "$1" = "attach" ]; then - if [ "$2" = "" ]; then - echo "Enter Volume Name" - exit 1 - fi - volumeName="$2" - - #get instance-name and device-name - if [ "$3" = "" ]; then - echo "Enter Instance Name" - exit 1 - fi - instanceName="$3" - - if [ "$4" = "" ]; then - echo "Enter Device Name" - exit 1 - fi - deviceName="$4" -fi - -if [ "$1" = "delete" ]; then - if [ "$2" = "" ]; then - echo "Enter Volume Name" - exit 1 - fi - volumeName="$2" -fi - -if [ "$1" = "createTag" ]; then - if [ "$2" = "" ]; then - echo "Enter Resource Name" - exit 1 - fi - resourceName="$2" - - if [ "$3" = "" ]; then - echo "Enter Tag Name" - exit 1 - fi - tagName="$3" - - if [ "$4" = "" ]; then - echo "Enter Tag Value" - exit 1 - fi - tagValue="$4" -fi - - -test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions - -ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME` - -if [ $ec2 == "unassigned" ]; then - STATUS="unknown" - RETVAL=1 -fi - -java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME` -path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH` - -export PATH=$path -export EC2_HOME=$ec2 -export JAVA_HOME=$java - -# get Keys and region -AmazonAccessKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonAccessKey` -if [ $AmazonAccessKeyFile == "unassigned" ]; then - echo "FAILED: missing Config Setting AmazonAccessKey : $AmazonAccessKeyfile" - exit 1 -fi - -AmazonSecretKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSecretKey` -if [ $AmazonSecretKeyFile == "unassigned" ]; then - echo "FAILED: missing Config Setting AmazonSecretKeyFile : $AmazonSecretKeyFile" - exit 1 -fi - -AmazonAccessKey=`cat $AmazonAccessKeyFile` -AmazonSecretKey=`cat $AmazonSecretKeyFile` - -if test ! -f $AmazonAccessKeyfile ; then - echo "FAILED: missing AmazonAccessKeyfile : $AmazonAccessKeyfile" - exit 1 -fi - -if test ! -f $AmazonSecretKeyfile ; then - echo "FAILED: missing AmazonSecretKeyfile : $AmazonSecretKeyfile" - exit 1 -fi - -Region=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonRegion` - - -checkInfostatus() { - #check if attached - cat /tmp/volumeInfo_$volumeName | grep attached > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then - STATUS="attached" - RETVAL=0 - return - fi - #check if available - cat /tmp/volumeInfo_$volumeName | grep available > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then - STATUS="available" - RETVAL=0 - return - fi - #check if detaching - cat /tmp/volumeInfo_$volumeName | grep detaching > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then - STATUS="detaching" - RETVAL=0 - return - fi - #check if attaching - cat /tmp/volumeInfo_$volumeName | grep attaching > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then - STATUS="attaching" - RETVAL=0 - return - fi - #check if doesn't exist - cat /tmp/volumeInfo_$volumeName | grep "does not exist" > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then - STATUS="does-not-exist" - RETVAL=1 - return - fi - #check if reports already attached from attach command - cat /tmp/volumeInfo_$volumeName | grep "already attached" > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then - STATUS="already-attached" - RETVAL=1 - return - fi - #any other, unknown error - STATUS="unknown" - RETVAL=1 - return -} - -createvolume() { - # get zone - zone=`$prefix/mariadb/columnstore/bin/IDBInstanceCmds.sh getZone` - - if [ $moduleType == "um" ]; then - # get type - volumeType=`$prefix/mariadb/columnstore/bin/getConfig Installation UMVolumeType` - if [ $volumeType == "io1" ]; then - # get IOPS - volumeIOPS=`$prefix/mariadb/columnstore/bin/getConfig Installation UMVolumeIOPS` - fi - else # pm - # get type - volumeType=`$prefix/mariadb/columnstore/bin/getConfig Installation PMVolumeType` - if [ $volumeType == "io1" ]; then - # get IOPS - volumeIOPS=`$prefix/mariadb/columnstore/bin/getConfig Installation PMVolumeIOPS` - fi - fi - - #create volume - if [ $volumeType == "io1" ]; then - volume=`ec2-create-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -z $zone -s $volumeSize -t $volumeType -iops $volumeIOPS | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - else - volume=`ec2-create-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -z $zone -s $volumeSize -t $volumeType | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'` - fi - - echo $volume - return -} - -describevolume() { - #describe volume - ec2-describe-volumes -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1 - - checkInfostatus - echo $STATUS - return -} - -detachvolume() { - #detach volume - ec2-detach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1 - - checkInfostatus - if [ $STATUS == "detaching" ]; then - retries=1 - while [ $retries -ne 60 ]; do - #retry until it's attached - ec2-detach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1 - - checkInfostatus - if [ $STATUS == "available" ]; then - echo "available" - exit 0 - fi - ((retries++)) - sleep 1 - done - test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions - cplogger -w 100 "detachvolume failed: $STATUS" - echo "failed" - exit 1 - fi - - if [ $STATUS == "available" ]; then - echo "available" - exit 0 - fi - - test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions - cplogger -w 100 "detachvolume failed status: $STATUS" - echo $STATUS - exit 1 -} - -attachvolume() { - - #detach volume - ec2-attach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName -i $instanceName -d $deviceName > /tmp/volumeInfo_$volumeName 2>&1 - - checkInfostatus - if [ $STATUS == "attaching" -o $STATUS == "already-attached" ]; then - retries=1 - while [ $retries -ne 60 ]; do - #check status until it's attached - describevolume - if [ $STATUS == "attached" ]; then - echo "attached" - exit 0 - fi - ((retries++)) - sleep 1 - done - test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions - cplogger -w 100 "attachvolume failed: $STATUS" - echo "failed" - exit 1 - fi - - if [ $STATUS == "attached" ]; then - echo "attached" - exit 0 - fi - - test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions - cplogger -w 100 "attachvolume failed: $STATUS" - echo $STATUS - exit 1 -} - -deletevolume() { - #delete volume - ec2-delete-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/deletevolume_$volumeName 2>&1 - return -} - -createTag() { - #create tag - ec2-create-tags -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $resourceName --tag $tagName=$tagValue > /tmp/createTag_$volumeName 2>&1 - return -} - -case "$1" in - create) - createvolume - ;; - describe) - describevolume - ;; - detach) - detachvolume - ;; - attach) - attachvolume - ;; - delete) - deletevolume - ;; - createTag) - createTag - ;; - *) - echo $"Usage: $0 {create|describe|detach|attach|delete|}" - exit 1 -esac - -exit $? diff --git a/oam/cloud/MCSInstanceCmds.sh b/oam/cloud/MCSInstanceCmds.sh index 32e674f90..4691f5d11 100755 --- a/oam/cloud/MCSInstanceCmds.sh +++ b/oam/cloud/MCSInstanceCmds.sh @@ -10,6 +10,8 @@ fi export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR +#get temp directory +tmpDir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir` #check command if [ "$1" = "" ]; then @@ -310,7 +312,7 @@ terminateInstance() { getRegion >/dev/null 2>&1 #terminate Instance - $AWSCLI terminate-instances --instance-ids $instanceName --region $Region > /tmp/termInstanceInfo_$instanceName 2>&1 + $AWSCLI terminate-instances --instance-ids $instanceName --region $Region > ${tmpdir}/termInstanceInfo_$instanceName 2>&1 return } @@ -319,7 +321,7 @@ stopInstance() { getRegion >/dev/null 2>&1 #terminate Instance - $AWSCLI stop-instances --instance-ids $instanceName --region $Region > /tmp/stopInstanceInfo_$instanceName 2>&1 + $AWSCLI stop-instances --instance-ids $instanceName --region $Region > ${tmpdir}/stopInstanceInfo_$instanceName 2>&1 return } @@ -328,11 +330,11 @@ startInstance() { getRegion >/dev/null 2>&1 #terminate Instance - $AWSCLI start-instances --instance-ids $instanceName --region $Region > /tmp/startInstanceInfo_$instanceName 2>&1 + $AWSCLI start-instances --instance-ids $instanceName --region $Region > ${tmpdir}/startInstanceInfo_$instanceName 2>&1 - cat /tmp/startInstanceInfo_$instanceName | grep InstanceId > /tmp/startInstanceStatus_$instanceName - if [ `cat /tmp/startInstanceStatus_$instanceName | wc -c` -eq 0 ]; then - echo "Failed, check /tmp/startInstanceInfo_$instanceName" + cat ${tmpdir}/startInstanceInfo_$instanceName | grep InstanceId > ${tmpdir}/startInstanceStatus_$instanceName + if [ `cat ${tmpdir}/startInstanceStatus_$instanceName | wc -c` -eq 0 ]; then + echo "Failed, check ${tmpdir}/startInstanceInfo_$instanceName" exit 1 fi echo "Success" @@ -345,11 +347,11 @@ assignElasticIP() { EIP=`$AWSCLI describe-addresses --region $Region --public-ips $IPAddress --query 'Addresses[*].AllocationId' --output text` - $AWSCLI associate-address --region $Region --instance-id $instanceName --allocation-id $EIP > /tmp/assignElasticIPInfo_$IPAddress 2>&1 + $AWSCLI associate-address --region $Region --instance-id $instanceName --allocation-id $EIP > ${tmpdir}/assignElasticIPInfo_$IPAddress 2>&1 - cat /tmp/assignElasticIPInfo_$IPAddress | grep error > /tmp/assignElasticIPStatus_$IPAddress - if [ `cat /tmp/assignElasticIPStatus_$IPAddress | wc -c` -ne 0 ]; then - echo "Failed, check /tmp/assignElasticIPInfo_$IPAddress" + cat ${tmpdir}/assignElasticIPInfo_$IPAddress | grep error > ${tmpdir}/assignElasticIPStatus_$IPAddress + if [ `cat ${tmpdir}/assignElasticIPStatus_$IPAddress | wc -c` -ne 0 ]; then + echo "Failed, check ${tmpdir}/assignElasticIPInfo_$IPAddress" exit 1 fi @@ -363,10 +365,10 @@ deassignElasticIP() { EIP=`$AWSCLI describe-addresses --region $Region --public-ips $IPAddress --query 'Addresses[*].AssociationId' --output text` - $AWSCLI disassociate-address --region $Region --association-id $EIP > /tmp/deassignElasticIPInfo_$IPAddress 2>&1 - cat /tmp/deassignElasticIPInfo_$IPAddress | grep error > /tmp/deassignElasticIPStatus_$IPAddress - if [ `cat /tmp/deassignElasticIPStatus_$IPAddress | wc -c` -ne 0 ]; then - echo "Failed, check /tmp/deassignElasticIPStatus_$IPAddress" + $AWSCLI disassociate-address --region $Region --association-id $EIP > ${tmpdir}/deassignElasticIPInfo_$IPAddress 2>&1 + cat ${tmpdir}/deassignElasticIPInfo_$IPAddress | grep error > ${tmpdir}/deassignElasticIPStatus_$IPAddress + if [ `cat ${tmpdir}/deassignElasticIPStatus_$IPAddress | wc -c` -ne 0 ]; then + echo "Failed, check ${tmpdir}/deassignElasticIPStatus_$IPAddress" exit 1 fi diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh index c7a231261..6228c93e9 100755 --- a/oam/cloud/MCSVolumeCmds.sh +++ b/oam/cloud/MCSVolumeCmds.sh @@ -10,6 +10,9 @@ fi export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR +#get temp directory +tmpDir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir` + #check command if [ "$1" = "" ]; then echo "Enter Command Name: {create|describe|detach|attach|delete|createTag}" @@ -108,43 +111,43 @@ Region=`$COLUMNSTORE_INSTALL_DIR/bin/MCSInstanceCmds.sh getRegion` checkInfostatus() { #check if attached - cat /tmp/volumeInfo_$volumeName | grep attached > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + cat ${tmpdir}/volumeInfo_$volumeName | grep attached > ${tmpdir}/volumeStatus_$volumeName + if [ `cat ${tmpdir}/volumeStatus_$volumeName | wc -c` -ne 0 ]; then STATUS="attached" RETVAL=0 return fi #check if available - cat /tmp/volumeInfo_$volumeName | grep available > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + cat ${tmpdir}/volumeInfo_$volumeName | grep available > ${tmpdir}/volumeStatus_$volumeName + if [ `cat ${tmpdir}/volumeStatus_$volumeName | wc -c` -ne 0 ]; then STATUS="available" RETVAL=0 return fi #check if detaching - cat /tmp/volumeInfo_$volumeName | grep detaching > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + cat ${tmpdir}/volumeInfo_$volumeName | grep detaching > ${tmpdir}/volumeStatus_$volumeName + if [ `cat ${tmpdir}/volumeStatus_$volumeName | wc -c` -ne 0 ]; then STATUS="detaching" RETVAL=0 return fi #check if attaching - cat /tmp/volumeInfo_$volumeName | grep attaching > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + cat ${tmpdir}volumeInfo_$volumeName | grep attaching > ${tmpdir}/volumeStatus_$volumeName + if [ `cat ${tmpdir}/volumeStatus_$volumeName | wc -c` -ne 0 ]; then STATUS="attaching" RETVAL=0 return fi #check if doesn't exist - cat /tmp/volumeInfo_$volumeName | grep "does not exist" > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + cat ${tmpdir}/volumeInfo_$volumeName | grep "does not exist" > ${tmpdir}/volumeStatus_$volumeName + if [ `cat ${tmpdir}/volumeStatus_$volumeName | wc -c` -ne 0 ]; then STATUS="does-not-exist" RETVAL=1 return fi #check if reports already attached from attach command - cat /tmp/volumeInfo_$volumeName | grep "already attached" > /tmp/volumeStatus_$volumeName - if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then + cat ${tmpdir}/volumeInfo_$volumeName | grep "already attached" > ${tmpdir}/volumeStatus_$volumeName + if [ `cat ${tmpdir}/volumeStatus_$volumeName | wc -c` -ne 0 ]; then STATUS="already-attached" RETVAL=1 return @@ -188,7 +191,7 @@ createvolume() { describevolume() { #describe volume - $AWSCLI describe-volumes --volume-ids $volumeName --region $Region > /tmp/volumeInfo_$volumeName 2>&1 + $AWSCLI describe-volumes --volume-ids $volumeName --region $Region > ${tmpdir}/volumeInfo_$volumeName 2>&1 checkInfostatus echo $STATUS @@ -197,14 +200,14 @@ describevolume() { detachvolume() { #detach volume - $AWSCLI detach-volume --volume-id $volumeName --region $Region > /tmp/volumeInfo_$volumeName 2>&1 + $AWSCLI detach-volume --volume-id $volumeName --region $Region > ${tmpdir}/volumeInfo_$volumeName 2>&1 checkInfostatus if [ $STATUS == "detaching" ]; then retries=1 while [ $retries -ne 10 ]; do #retry until it's attached - $AWSCLI detach-volume --volume-id $volumeName --region $Region > /tmp/volumeInfo_$volumeName 2>&1 + $AWSCLI detach-volume --volume-id $volumeName --region $Region > ${tmpdir}/volumeInfo_$volumeName 2>&1 checkInfostatus if [ $STATUS == "available" ]; then @@ -234,7 +237,7 @@ detachvolume() { attachvolume() { #detach volume - $AWSCLI attach-volume --volume-id $volumeName --instance-id $instanceName --device $deviceName --region $Region > /tmp/volumeInfo_$volumeName 2>&1 + $AWSCLI attach-volume --volume-id $volumeName --instance-id $instanceName --device $deviceName --region $Region > ${tmpdir}/volumeInfo_$volumeName 2>&1 checkInfostatus if [ $STATUS == "attaching" -o $STATUS == "already-attached" ]; then @@ -268,13 +271,13 @@ attachvolume() { deletevolume() { #delete volume - $AWSCLI delete-volume --volume-id $volumeName --region $Region > /tmp/deletevolume_$volumeName 2>&1 + $AWSCLI delete-volume --volume-id $volumeName --region $Region > ${tmpdir}/deletevolume_$volumeName 2>&1 return } createTag() { #create tag - $AWSCLI create-tags --resources $resourceName --tags Key=$tagName,Value=$tagValue --region $Region > /tmp/createTag_$volumeName 2>&1 + $AWSCLI create-tags --resources $resourceName --tags Key=$tagName,Value=$tagValue --region $Region > ${tmpdir}/createTag_$volumeName 2>&1 return } diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 3f1811a25..364b46c08 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -269,6 +269,7 @@ --> /rdwrscratch /columnstore_tmp_files + /tmp/columnstore_tmp_files dm diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 4bf12d36b..56a6e678f 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -260,7 +260,8 @@ Optional: Module-name and Parameter-name will display the Module Name Configuration parameter - AVAILABLE + getSystemDirectories + Get System Installation and Temporary Logging Directories AVAILABLE diff --git a/oam/install_scripts/disable-rep-columnstore.sh b/oam/install_scripts/disable-rep-columnstore.sh index 1c730da4c..c93f59f90 100644 --- a/oam/install_scripts/disable-rep-columnstore.sh +++ b/oam/install_scripts/disable-rep-columnstore.sh @@ -8,13 +8,13 @@ # check log for error checkForError() { - grep ERROR /tmp/disable-rep-status.log > /tmp/error.check - if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then - echo "ERROR: check log file: /tmp/disable-rep-status.log" - rm -f /tmp/error.check + grep ERROR ${tmpdir}/disable-rep-status.log > ${tmpdir}/error.check + if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then + echo "ERROR: check log file:${tmpdir}/disable-rep-status.log" + rm -f ${tmpdir}/error.check exit 1 fi - rm -f /tmp/error.check + rm -f ${tmpdir}/error.check } prefix=/usr/local @@ -30,42 +30,44 @@ for arg in "$@"; do elif [ `expr -- "$arg" : '--installdir='` -eq 13 ]; then installdir="`echo $arg | awk -F= '{print $2}'`" prefix=`dirname $installdir` + elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then + tmpdir="$(echo $arg | awk -F= '{print $2}')" fi done test -f $installdir/post/functions && . $installdir/post/functions ->/tmp/disable-rep-status.log +>${tmpdir}/disable-rep-status.log # # Run stop slave command # -echo "Run stop slave command" >>/tmp/disable-rep-status.log -cat >/tmp/idb_disable-rep.sql <>${tmpdir}/disable-rep-status.log +cat >${tmpdir}/idb_disable-rep.sql <>/tmp/disable-rep-status.log +cat ${tmpdir}/idb_disable-rep.sql >>${tmpdir}/disable-rep-status.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root $pwprompt \ - calpontsys >/tmp/disable-rep-status.log 2>&1 + calpontsys <${tmpdir}/idb_disable-rep.sql >>${tmpdir}/disable-rep-status.log 2>&1 checkForError # # Run reset slave command # -echo "Run reset slave command" >>/tmp/disable-rep-status.log -cat >/tmp/idb_disable-rep.sql <>${tmpdir}/disable-rep-status.log +cat >${tmpdir}/idb_disable-rep.sql <>/tmp/disable-rep-status.log +cat ${tmpdir}/idb_disable-rep.sql >>${tmpdir}/disable-rep-status.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root $pwprompt \ - calpontsys >/tmp/disable-rep-status.log 2>&1 + calpontsys <${tmpdir}/idb_disable-rep.sql >>${tmpdir}/disable-rep-status.log 2>&1 checkForError diff --git a/oam/install_scripts/master-rep-columnstore.sh b/oam/install_scripts/master-rep-columnstore.sh index 5d27ee278..313199d62 100644 --- a/oam/install_scripts/master-rep-columnstore.sh +++ b/oam/install_scripts/master-rep-columnstore.sh @@ -8,13 +8,13 @@ # check log for error checkForError() { - grep ERROR /tmp/master-rep-status-$hostipaddr.log > /tmp/error.check - if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then - echo "ERROR: check log file: /tmp/master-rep-status-$hostipaddr.log" - rm -f /tmp/error.check + grep ERROR ${tmpdir}/master-rep-status-$hostipaddr.log > ${tmpdir}/error.check + if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then + echo "ERROR: check log file: ${tmpdir}/master-rep-status-$hostipaddr.log" + rm -f ${tmpdir}/error.check exit 1 fi - rm -f /tmp/error.check + rm -f ${tmpdir}/error.check } prefix=/usr/local @@ -29,6 +29,8 @@ for arg in "$@"; do prefix=`dirname $installdir` elif [ `expr -- "$arg" : '--hostIP='` -eq 9 ]; then hostipaddr="`echo $arg | awk -F= '{print $2}'`" + elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then + tmpdir="$(echo $arg | awk -F= '{print $2}')" fi done @@ -37,69 +39,69 @@ test -f $installdir/post/functions && . $installdir/post/functions repUser="idbrep" password="Calpont1" ->/tmp/master-rep-status-$hostipaddr.log +>${tmpdir}/master-rep-status-$hostipaddr.log # # Create Replication User # -echo "Create Replication User $repUser for node $hostipaddr" >>/tmp/master-rep-status-$hostipaddr.log -cat >/tmp/idb_master-rep.sql <>${tmpdir}/master-rep-status-$hostipaddr.log +cat >${tmpdir}/idb_master-rep.sql <>/tmp/master-rep-status-$hostipaddr.log +cat ${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys >/tmp/master-rep-status-$hostipaddr.log 2>&1 + calpontsys <${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log 2>&1 checkForError # # Grant table access for created user # -echo "Grant table access for $repUser for node $hostipaddr" >>/tmp/master-rep-status-$hostipaddr.log -cat >/tmp/idb_master-rep.sql <>${tmpdir}/master-rep-status-$hostipaddr.log +cat >${tmpdir}/idb_master-rep.sql <>/tmp/master-rep-status-$hostipaddr.log +cat ${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys >/tmp/master-rep-status-$hostipaddr.log 2>&1 + calpontsys <${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log 2>&1 checkForError # # Run SHOW MASTER STATUS # -echo "Run SHOW MASTER STATUS to node log" >>/tmp/master-rep-status-$hostipaddr.log -cat >/tmp/idb_master-rep.sql <>${tmpdir}/master-rep-status-$hostipaddr.log +cat >${tmpdir}/idb_master-rep.sql <>/tmp/master-rep-status-$hostipaddr.log +cat ${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys >/tmp/master-rep-status-$hostipaddr.log 2>&1 + calpontsys <${tmpdir}/idb_master-rep.sql >>${tmpdir}/master-rep-status-$hostipaddr.log 2>&1 checkForError -echo "Run SHOW MASTER STATUS to master status log /tmp/show-master-status.log" >>/tmp/master-rep-status-$hostipaddr.log -cat >/tmp/idb_master-rep.sql <>${tmpdir}/master-rep-status-$hostipaddr.log +cat >${tmpdir}/idb_master-rep.sql </tmp/show-master-status.log +cat ${tmpdir}/idb_master-rep.sql >${tmpdir}/show-master-status.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys >/tmp/show-master-status.log + calpontsys <${tmpdir}/idb_master-rep.sql >>${tmpdir}/show-master-status.log #alls good, 'OK' for success diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 9b2b939a8..57c5427e6 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -167,16 +167,20 @@ mkdir -p $installdir/data/bulk/rollback >/dev/null 2>&1 mkdir -p $installdir/data/bulk/tmpjob >/dev/null 2>&1 rm -f $installdir/data/bulk/tmpjob/* >/dev/null 2>&1 -#get temp directory +#get temp base directory tmpDir="/tmp" if [ $user != "root" ]; then tmpDir=$HOME"./tmp" mkdir $tmpDir >/dev/null 2>&1 fi +#get columnstore temp file directory name TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` tmpDir=$tmpDir+$TempFileDir +#set base columnstore temp file directory +$installdir/bin/setConfig -d SystemConfig SystemTempFileDir $tmpDir + #create columnstore temp file directory mkdir -p $tmpDir >/dev/null 2>&1 diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 215f7d695..d02088fff 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -7,16 +7,16 @@ # check log for error checkForError() { # check for password error - grep "ERROR 1045" ${tmpdir}/mysql_install.log > /tmp/error.check - if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then + grep "ERROR 1045" ${tmpdir}/mysql_install.log > ${tmpdir}/error.check + if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then echo "MySQL Password file missing or incorrect, check .my.cnf file" - rm -f /tmp/error.check + rm -f ${tmpdir}/error.check $installdir/mysql/mysql-Columnstore stop sleep 2 exit 2; fi - rm -f /tmp/error.check + rm -f ${tmpdir}/error.check #--------------------------------------------------------------------------- # See if engine columnstore exist diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 52571aacf..572f70420 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -73,14 +73,7 @@ rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1\ $installdir/bin/clearShm > /dev/null 2>&1 #get temp directory -tmpDir="/tmp" -if [ $user != "root" ]; then - tmpDir=$HOME"./tmp" - mkdir $tmpDir >/dev/null 2>&1 -fi - -TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` -tmpDir=$tmpDir+$TempFileDir +tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` # delete tmp files rm -rf $tmpDir diff --git a/oam/install_scripts/slave-rep-columnstore.sh b/oam/install_scripts/slave-rep-columnstore.sh index ae8914237..8468d903e 100644 --- a/oam/install_scripts/slave-rep-columnstore.sh +++ b/oam/install_scripts/slave-rep-columnstore.sh @@ -8,13 +8,13 @@ # check log for error checkForError() { - grep ERROR /tmp/slave-rep-status.log > /tmp/error.check - if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then - echo "ERROR: check log file: /tmp/slave-rep-status.log" - rm -f /tmp/error.check + grep ERROR ${tmpdir}/slave-rep-status.log > ${tmpdir}/error.check + if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then + echo "ERROR: check log file: ${tmpdir}/slave-rep-status.log" + rm -f ${tmpdir}/error.check exit 1 fi - rm -f /tmp/error.check + rm -f ${tmpdir}/error.check } prefix=/usr/local @@ -35,6 +35,10 @@ for arg in "$@"; do masterlogpos="`echo $arg | awk -F= '{print $2}'`" elif [ `expr -- "$arg" : '--port='` -eq 7 ]; then port="`echo $arg | awk -F= '{print $2}'`" + elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then + tmpdir="$(echo $arg | awk -F= '{print $2}')" + fi + fi done @@ -43,29 +47,29 @@ test -f $installdir/post/functions && . $installdir/post/functions repUser="idbrep" password="Calpont1" ->/tmp/slave-rep-status.log +>${tmpdir}/slave-rep-status.log # # Run stop slave command # -echo "Run stop slave command" >>/tmp/slave-rep-status.log -cat >/tmp/idb_slave-rep.sql <>${tmpdir}slave-rep-status.log +cat >${tmpdir}/idb_slave-rep.sql <>/tmp/slave-rep-status.log +cat ${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys >/tmp/slave-rep-status.log 2>&1 + calpontsys <${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log 2>&1 checkForError # # Run Change Master Command # -echo "Run Change Master Command" >>/tmp/slave-rep-status.log -cat >/tmp/idb_slave-rep.sql <>${tmpdir}/slave-rep-status.log +cat >${tmpdir}/idb_slave-rep.sql <>/tmp/slave-rep-status.log +cat ${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys >/tmp/slave-rep-status.log 2>&1 + calpontsys <${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log 2>&1 checkForError # # Run start slave command # -echo "Run start slave command" >>/tmp/slave-rep-status.log -cat >/tmp/idb_slave-rep.sql <>${tmpdir}/slave-rep-status.log +cat >${tmpdir}/idb_slave-rep.sql <>/tmp/slave-rep-status.log +cat ${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys >/tmp/slave-rep-status.log 2>&1 + calpontsys <${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log 2>&1 checkForError # # Run SHOW SLAVE STATUS # -echo "Run SHOW SLAVE STATUS to node log" >>/tmp/slave-rep-status.log -cat >/tmp/idb_slave-rep.sql <>${tmpdir}/slave-rep-status.log +cat >${tmpdir}/idb_slave-rep.sql <>/tmp/slave-rep-status.log +cat ${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys >/tmp/slave-rep-status.log 2>&1 + calpontsys <${tmpdir}/idb_slave-rep.sql >>${tmpdir}/slave-rep-status.log 2>&1 checkForError diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 47469ac5b..a0bd6103e 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -81,6 +81,8 @@ using namespace oam; using namespace logging; using namespace BRM; +string tmpdir; + namespace oam { // flag to tell us ctrl-c was hit @@ -159,6 +161,8 @@ Oam::Oam() if ( USER != "root") userDir = "home/" + USER; + + tmpdir = startup::StartUp::tmpDir(); } @@ -3570,19 +3574,19 @@ void Oam::getLogFile(const std::string moduleName, const std::string loglevel, c logFile = path + logFile; - string tempLogFile = "/tmp/logs"; + string tempLogFile = tmpdir +"/logs"; //make 1 log file made up of archive and current *.log - (void)system("touch /tmp/logs"); + (void)system(tempLogFile.c_str()); string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpdir; - string cmd = "ls " + path + logdir + "/archive | grep '" + logFileName + "' > /tmp/logfiles"; + string cmd = "ls " + path + logdir + "/archive | grep '" + logFileName + "' > " + tmpdir + "/logfiles"; (void)system(cmd.c_str()); - string fileName = "/tmp/logfiles"; + string fileName = tmpdir + "/logfiles"; ifstream oldFile (fileName.c_str()); @@ -3594,7 +3598,7 @@ void Oam::getLogFile(const std::string moduleName, const std::string loglevel, c while (oldFile.getline(line, 400)) { buf = line; - cmd = "cat " + path + logdir + "/archive/" + buf + " >> /tmp/logs"; + cmd = "cat " + path + logdir + "/archive/" + buf + " >> " + tempLogFile; (void)system(cmd.c_str()); } @@ -3602,7 +3606,7 @@ void Oam::getLogFile(const std::string moduleName, const std::string loglevel, c unlink (fileName.c_str()); } - cmd = "cat " + logFile + " >> /tmp/logs"; + cmd = "cat " + logFile + " >> " + tempLogFile; (void)system(cmd.c_str()); //validate and get mm / dd from incoming date @@ -3720,7 +3724,7 @@ void Oam::getLogFile(const std::string moduleName, const std::string loglevel, c unlink (tempLogFile.c_str()); } - fileName = "/tmp/logsByDate"; + fileName = tmpdir + "/logsByDate"; ofstream newFile (fileName.c_str()); //create new file @@ -6747,7 +6751,7 @@ void Oam::addDbroot(const int dbrootNumber, DBRootConfigList& dbrootlist, string int user; user = getuid(); - cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1"; + cmd = "mkfs.ext2 -F " + amazonDeviceName + " > " + tmpdir + "/format.log 2>&1"; writeLog("addDbroot format cmd: " + cmd, LOG_TYPE_DEBUG ); @@ -7801,17 +7805,19 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) string command; + string pidtmp = tmpdir + "/mysql.pid"; + switch (action) { case MYSQL_START: { - command = "start > /tmp/actionMysqlCalpont.log 2>&1"; + command = "start > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; break; } case MYSQL_STOP: { - command = "stop > /tmp/actionMysqlCalpont.log 2>&1"; + command = "stop > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; //set process status try @@ -7826,25 +7832,25 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) case MYSQL_RESTART: { - command = "restart > /tmp/actionMysqlCalpont.log 2>&1"; + command = "restart > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; break; } case MYSQL_RELOAD: { - command = "reload > /tmp/actionMysqlCalpont.log 2>&1"; + command = "reload > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; break; } case MYSQL_FORCE_RELOAD: { - command = "force-reload > /tmp/actionMysqlCalpont.log 2>&1"; + command = "force-reload > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; break; } case MYSQL_STATUS: { - command = "status > /tmp/mysql.status"; + command = "status > " + tmpdir + "/mysql.status"; break; } @@ -7862,9 +7868,9 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) if (action == MYSQL_START || action == MYSQL_RESTART) { //get pid - cmd = "cat " + InstallDir + "/mysql/db/*.pid > /tmp/mysql.pid"; + cmd = "cat " + InstallDir + "/mysql/db/*.pid > " + pidtmp; system(cmd.c_str()); - ifstream oldFile ("/tmp/mysql.pid"); + ifstream oldFile (pidtmp); //fail if file size 0 oldFile.seekg(0, std::ios::end); @@ -7915,14 +7921,15 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) int state = procstat.ProcessOpState; pid_t pidStatus = procstat.ProcessID; - if (checkLogStatus("/tmp/mysql.status", "MySQL running")) + string mysqlStatus = tmpdir + "/mysql.status"; + if (checkLogStatus(mysqlStatus, "MySQL running")) { if ( state != ACTIVE ) { //get pid - cmd = "cat " + InstallDir + "/mysql/db/*.pid > /tmp/mysql.pid"; + cmd = "cat " + InstallDir + "/mysql/db/*.pid > " + pidtmp; system(cmd.c_str()); - ifstream oldFile ("/tmp/mysql.pid"); + ifstream oldFile (pidtmp); char line[400]; string pid; @@ -7947,9 +7954,9 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) else { //check if pid has changed - cmd = "cat " + InstallDir + "/mysql/db/*.pid > /tmp/mysql.pid"; + cmd = "cat " + InstallDir + "/mysql/db/*.pid > " + pidtmp; system(cmd.c_str()); - ifstream oldFile ("/tmp/mysql.pid"); + ifstream oldFile (pidtmp); char line[400]; string pid; @@ -8224,19 +8231,19 @@ int Oam::validateModule(const std::string name) std::string Oam::getEC2InstanceIpAddress(std::string instanceName) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getPrivateIP " + instanceName + " > /tmp/getCloudIP_" + instanceName; + string tmplog = tmpdir + "/getCloudIP_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getPrivateIP " + instanceName + " > " + tmplog; system(cmd.c_str()); - if (checkLogStatus("/tmp/getCloudIP_" + instanceName, "stopped") ) + if (checkLogStatus(tmplog, "stopped") ) return "stopped"; - if (checkLogStatus("/tmp/getCloudIP_" + instanceName, "terminated") ) + if (checkLogStatus(tmplog, "terminated") ) return "terminated"; // get IP Address string IPAddr; - string file = "/tmp/getCloudIP_" + instanceName; - ifstream oldFile (file.c_str()); + ifstream oldFile (tmplog.c_str()); char line[400]; while (oldFile.getline(line, 400)) @@ -8263,7 +8270,8 @@ std::string Oam::getEC2InstanceIpAddress(std::string instanceName) std::string Oam::getEC2LocalInstance(std::string name) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getInstance > /tmp/getInstanceInfo_" + name; + string file = tmpdir + "/getInstanceInfo_" + name; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getInstance > " + file; int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -8271,7 +8279,6 @@ std::string Oam::getEC2LocalInstance(std::string name) // get Instance Name string instanceName; - string file = "/tmp/getInstanceInfo_" + name; ifstream oldFile (file.c_str()); char line[400]; @@ -8297,7 +8304,8 @@ std::string Oam::getEC2LocalInstance(std::string name) std::string Oam::getEC2LocalInstanceType(std::string name) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getType > /tmp/getInstanceType_" + name; + string file = tmpdir + "/getInstanceType_" + name; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getType > " + file; int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -8305,7 +8313,6 @@ std::string Oam::getEC2LocalInstanceType(std::string name) // get Instance Name string instanceType; - string file = "/tmp/getInstanceType_" + name; ifstream oldFile (file.c_str()); char line[400]; @@ -8331,7 +8338,8 @@ std::string Oam::getEC2LocalInstanceType(std::string name) std::string Oam::getEC2LocalInstanceSubnet(std::string name) { // run script to get Instance Subnet - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getSubnet > /tmp/getInstanceSubnet_" + name; + string file = tmpdir + "/getInstanceSubnet_" + name; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getSubnet > " + file; int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -8339,7 +8347,6 @@ std::string Oam::getEC2LocalInstanceSubnet(std::string name) // get Instance Name string instanceSubnet; - string file = "/tmp/getInstanceSubnet_" + name; ifstream oldFile (file.c_str()); char line[400]; @@ -8366,18 +8373,18 @@ std::string Oam::getEC2LocalInstanceSubnet(std::string name) std::string Oam::launchEC2Instance( const std::string name, const std::string IPAddress, const std::string type, const std::string group) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh launchInstance " + IPAddress + " " + type + " " + group + " > /tmp/getInstance_" + name; + string file = tmpdir + "/getInstance_" + name; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh launchInstance " + IPAddress + " " + type + " " + group + " > " + file; int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) return "failed"; - if (checkLogStatus("/tmp/getInstance", "Required") ) + if (checkLogStatus(file, "Required") ) return "failed"; // get Instance ID string instance; - string file = "/tmp/getInstance_" + name; ifstream oldFile (file.c_str()); char line[400]; @@ -8411,7 +8418,7 @@ std::string Oam::launchEC2Instance( const std::string name, const std::string IP void Oam::terminateEC2Instance(std::string instanceName) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh terminateInstance " + instanceName + " > /tmp/terminateEC2Instance_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh terminateInstance " + instanceName + " > " + tmpdir + "/terminateEC2Instance_" + instanceName; system(cmd.c_str()); return; @@ -8428,7 +8435,7 @@ void Oam::terminateEC2Instance(std::string instanceName) void Oam::stopEC2Instance(std::string instanceName) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh stopInstance " + instanceName + " > /tmp/stopEC2Instance_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh stopInstance " + instanceName + " > " + tmpdir + "/stopEC2Instance_" + instanceName; system(cmd.c_str()); return; @@ -8445,7 +8452,7 @@ void Oam::stopEC2Instance(std::string instanceName) bool Oam::startEC2Instance(std::string instanceName) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh startInstance " + instanceName + " > /tmp/startEC2Instance_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh startInstance " + instanceName + " > " + tmpdir + "/startEC2Instance_" + instanceName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) @@ -8465,7 +8472,7 @@ bool Oam::startEC2Instance(std::string instanceName) bool Oam::assignElasticIP(std::string instanceName, std::string IpAddress) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh assignElasticIP " + instanceName + " " + IpAddress + " > /tmp/assignElasticIP_" + instanceName; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh assignElasticIP " + instanceName + " " + IpAddress + " > " + tmpdir + "/assignElasticIP_" + instanceName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) @@ -8485,7 +8492,7 @@ bool Oam::assignElasticIP(std::string instanceName, std::string IpAddress) bool Oam::deassignElasticIP(std::string IpAddress) { // run script to get Instance status and IP Address - string cmd = InstallDir + "/bin/MCSInstanceCmds.sh deassignElasticIP " + IpAddress + " > /tmp/deassignElasticIP_" + IpAddress; + string cmd = InstallDir + "/bin/MCSInstanceCmds.sh deassignElasticIP " + IpAddress + " > " + tmpdir + "/deassignElasticIP_" + IpAddress; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) @@ -8505,7 +8512,7 @@ bool Oam::deassignElasticIP(std::string IpAddress) std::string Oam::getEC2VolumeStatus(std::string volumeName) { // run script to get Volume Status - string cmd = InstallDir + "/bin/MCSVolumeCmds.sh describe " + volumeName + " > /tmp/getVolumeStatus_" + volumeName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh describe " + volumeName + " > " + tmpdir + "/getVolumeStatus_" + volumeName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) @@ -8513,7 +8520,7 @@ std::string Oam::getEC2VolumeStatus(std::string volumeName) // get status string status; - string file = "/tmp/getVolumeStatus_" + volumeName; + string file = tmpdir + "/getVolumeStatus_" + volumeName; ifstream oldFile (file.c_str()); char line[400]; @@ -8539,7 +8546,8 @@ std::string Oam::getEC2VolumeStatus(std::string volumeName) std::string Oam::createEC2Volume(std::string size, std::string name) { // run script to get Volume Status - string cmd = InstallDir + "/bin/MCSVolumeCmds.sh create " + size + " " + name + " > /tmp/createVolumeStatus_" + name; + string file = tmpdir + "/createVolumeStatus_" + name; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh create " + size + " " + name + " > " + file; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) @@ -8547,7 +8555,6 @@ std::string Oam::createEC2Volume(std::string size, std::string name) // get status string volumeName; - string file = "/tmp/createVolumeStatus_" + name; ifstream oldFile (file.c_str()); char line[400]; @@ -8587,7 +8594,7 @@ bool Oam::attachEC2Volume(std::string volumeName, std::string deviceName, std::s for ( int retry = 0 ; retry < 2 ; retry++ ) { // run script to attach Volume - string cmd = InstallDir + "/bin/MCSVolumeCmds.sh attach " + volumeName + " " + instanceName + " " + deviceName + " > /tmp/attachVolumeStatus_" + volumeName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh attach " + volumeName + " " + instanceName + " " + deviceName + " > " + tmpdir + "/attachVolumeStatus_" + volumeName; ret = system(cmd.c_str()); if (WEXITSTATUS(ret) == 0 ) @@ -8614,7 +8621,7 @@ bool Oam::attachEC2Volume(std::string volumeName, std::string deviceName, std::s bool Oam::detachEC2Volume(std::string volumeName) { // run script to attach Volume - string cmd = InstallDir + "/bin/MCSVolumeCmds.sh detach " + volumeName + " > /tmp/detachVolumeStatus_" + volumeName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh detach " + volumeName + " > " + tmpdir + "/detachVolumeStatus_" + volumeName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) @@ -8634,7 +8641,7 @@ bool Oam::detachEC2Volume(std::string volumeName) bool Oam::deleteEC2Volume(std::string volumeName) { // run script to delete Volume - string cmd = InstallDir + "/bin/MCSVolumeCmds.sh delete " + volumeName + " > /tmp/deleteVolumeStatus_" + volumeName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh delete " + volumeName + " > " + tmpdir + "/deleteVolumeStatus_" + volumeName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) @@ -8654,7 +8661,7 @@ bool Oam::deleteEC2Volume(std::string volumeName) bool Oam::createEC2tag(std::string resourceName, std::string tagName, std::string tagValue) { // run script to create a tag - string cmd = InstallDir + "/bin/MCSVolumeCmds.sh createTag " + resourceName + " " + tagName + " " + tagValue + " > /tmp/createTagStatus_" + resourceName; + string cmd = InstallDir + "/bin/MCSVolumeCmds.sh createTag " + resourceName + " " + tagName + " " + tagValue + " > " + tmpdir + "createTagStatus_" + resourceName; int ret = system(cmd.c_str()); if (WEXITSTATUS(ret) != 0 ) @@ -9009,7 +9016,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string } sleep(5); - command = glustercmd + "peer status " + " >> /tmp/glusterCommands.txt 2>&1"; + command = glustercmd + "peer status " + " >> " + tmpdir + "/glusterCommands.log 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9042,7 +9049,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string pmnextbrick[pm]++; } - command += "force >> /tmp/glusterCommands.txt 2>&1"; + command += "force >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << "Gluster create and start volume dbroot" << itoa(newDbrootID) << "..."; status = system(command.c_str()); @@ -9054,7 +9061,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string if (user != 0) { - command = "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-uid " + itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; + command = "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-uid " + itoa(user) + " >> " + tmpdir + "/glusterCommands.log 2>&1";; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9063,7 +9070,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string exceptionControl("GLUSTER_ADD", API_FAILURE); } - command = "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-gid " + itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; + command = "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-gid " + itoa(group) + " >> " + tmpdir + "/glusterCommands.log 2>&1";; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9073,7 +9080,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string } } - command = glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; + command = glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> " + tmpdir + "/glusterCommands.log 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9120,7 +9127,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string int status; writeLog("glusterctl: GLUSTER_DELETE: dbroot = " + dbrootID, LOG_TYPE_DEBUG ); - command = glustercmd + "--mode=script volume stop dbroot" + dbrootID + " >> /tmp/glusterCommands.txt 2>&1"; + command = glustercmd + "--mode=script volume stop dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; status = system(command.c_str()); @@ -9133,7 +9140,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string // give time for transaction to finish after stopping sleep(10); - command = glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> /tmp/glusterCommands.txt 2>&1"; + command = glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; status = system(command.c_str()); @@ -9153,7 +9160,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string string command = ""; int status; - command = glustercmd + "peer probe " + ipAddress + " >> /tmp/glusterCommands.txt 2>&1"; + command = glustercmd + "peer probe " + ipAddress + " >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << "gluster peer probe " + ipAddress << endl; status = system(command.c_str()); @@ -9309,7 +9316,7 @@ int Oam::checkGlusterLog(std::string logFile, std::string& msg) { if (checkLogStatus(logFile, "OK")) { - if ( logFile == "/tmp/gluster_howhas.log" ) + if ( logFile == tmpdir + "/gluster_howhas.log" ) { ifstream File(logFile.c_str()); @@ -9364,7 +9371,7 @@ int Oam::checkGlusterLog(std::string logFile, std::string& msg) writeLog("checkGlusterLog: FAILURE", LOG_TYPE_ERROR); - if ( logFile == "/tmp/gluster_howhas.log" ) + if ( logFile == tmpdir + "/gluster_howhas.log" ) return 2; else exceptionControl("glusterctl", API_FAILURE); @@ -10875,8 +10882,6 @@ int Oam::readHdfsActiveAlarms(AlarmList& alarmList) bool Oam::checkSystemRunning() { - // string cmd = startup::StartUp::installDir() + "/bin/columnstore status > /tmp/status.log"; - // system(cmd.c_str()); struct stat st; string lockFileDir = "/var/subsys/lock"; diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 7cbe161ba..ccd74511e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -188,9 +188,6 @@ int main(int argc, char* argv[]) string ccHistoryFile = HOME + "/.cc_history"; - cout << startup::StartUp::installDir() << endl; - cout << startup::StartUp::tmpDir() << endl; - string cf = startup::StartUp::installDir() + "/etc/" + ConsoleCmdsFile; fConfig = Config::makeConfig(cf); @@ -4919,8 +4916,12 @@ int processCommand(string* arguments) } break; - case 39: + case 39: // getSystemDirectories { + cout << "System Installation and Temporary Logging Directories" << endl << endl; + + cout << "System Installation Directory = " << startup::StartUp::installDir() << endl; + cout << "System Temporary Logging Directory = " << startup::StartUp::tmpDir() << endl; } break; diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 60b9f0060..83f41f561 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -61,6 +61,8 @@ bool HDFS = false; string localHostName; string PMwithUM = "n"; string MySQLRep = "n"; +string tmpLogDir; + // pushing the ACTIVE_ALARMS_FILE to all nodes every 10 seconds. const int ACTIVE_ALARMS_PUSHING_INTERVAL = 10; @@ -256,6 +258,9 @@ int main(int argc, char** argv) freeifaddrs(addrs); log.writeLog(__LINE__, "Main Ethernet Port = " + iface_name, LOG_TYPE_DEBUG); + //get tmp log directory + tmpLogDir = startup::StartUp::tmpDir(); + // //start a thread to ping all system modules // diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index fadb63ea4..76c46d733 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -56,6 +56,7 @@ extern bool HDFS; extern string localHostName; extern string PMwithUM; extern string AmazonPMFailover; +extern string tmpLogDir; typedef map moduleList; extern moduleList moduleInfoList; @@ -2727,7 +2728,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // Save the BRM. This command presages a system backup. Best to have a current BRM on disk string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpLogDir; string cmd = startup::StartUp::installDir() + "/bin/save_brm > " + logdir + "/save_brm.log1 2>&1"; int rtnCode = system(cmd.c_str()); @@ -5134,17 +5135,17 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str continue; } - string cmd = installDir + "/bin/remote_command.sh " + IPAddr + " " + password + " 'ls' 1 > /tmp/login_test.log"; + string loginTmp = tmpLogDir + "/login_test.log"; + string cmd = installDir + "/bin/remote_command.sh " + IPAddr + " " + password + " 'ls' 1 > " + loginTmp; system(cmd.c_str()); - if (!oam.checkLogStatus("/tmp/login_test.log", "README")) + if (!oam.checkLogStatus(loginTmp, "README")) { //check for RSA KEY ISSUE and fix - if (oam.checkLogStatus("/tmp/login_test.log", "Offending")) + if (oam.checkLogStatus(loginTmp, "Offending")) { log.writeLog(__LINE__, "addModule - login failed, Offending key issue, try fixing: " + moduleName, LOG_TYPE_DEBUG); - string file = "/tmp/login_test.log"; - oam.fixRSAkey(file); + oam.fixRSAkey(loginTmp); } log.writeLog(__LINE__, "addModule - login failed, retry login test: " + moduleName, LOG_TYPE_DEBUG); @@ -5551,7 +5552,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str //set root password if (amazon) { - cmd = startup::StartUp::installDir() + "/bin/remote_command.sh " + remoteModuleIP + " " + password + " '/root/.scripts/updatePassword.sh " + password + "' > /tmp/password_change.log"; + cmd = startup::StartUp::installDir() + "/bin/remote_command.sh " + remoteModuleIP + " " + password + " '/root/.scripts/updatePassword.sh " + password + "' > " + tmpLogDir + "/password_change.log"; log.writeLog(__LINE__, "addModule - cmd: " + cmd, LOG_TYPE_DEBUG); int rtnCode = system(cmd.c_str()); @@ -5572,7 +5573,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str //run remote installer script if ( packageType != "binary" ) { - string logFile = "/tmp/" + remoteModuleName + "_user_installer.log"; + string logFile = tmpLogDir + "/" + remoteModuleName + "_user_installer.log"; log.writeLog(__LINE__, "addModule - user_installer run for " + remoteModuleName, LOG_TYPE_DEBUG); string cmd = installDir + "/bin/package_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + " --nodeps 1 > " + logFile; @@ -5643,7 +5644,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str else { // do a binary package install - string logFile = "/tmp/" + remoteModuleName + "_binary_installer.log"; + string logFile = tmpLogDir + "/" + remoteModuleName + "_binary_installer.log"; log.writeLog(__LINE__, "addModule - binary_installer run for " + remoteModuleName, LOG_TYPE_DEBUG); string binservertype = oam.itoa(config.ServerInstallType()); @@ -5723,7 +5724,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str { if ( packageType != "binary" ) { - string logFile = "/tmp/" + remoteModuleName + "_package_installer.log"; + string logFile = tmpLogDir + "/" + remoteModuleName + "_package_installer.log"; log.writeLog(__LINE__, "addModule - package_installer run for " + remoteModuleName, LOG_TYPE_DEBUG); string cmd = installDir + "/bin/package_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + + " --nodeps 1 > " + logFile; log.writeLog(__LINE__, "addModule cmd: " + cmd, LOG_TYPE_DEBUG); @@ -5792,7 +5793,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str else { // do a binary package install - string logFile = "/tmp/" + remoteModuleName + "_binary_installer.log"; + string logFile = tmpLogDir + "/" + remoteModuleName + "_binary_installer.log"; log.writeLog(__LINE__, "addModule - binary_installer run for " + remoteModuleName, LOG_TYPE_DEBUG); string binservertype = oam.itoa(config.ServerInstallType()); @@ -7038,7 +7039,7 @@ void ProcessManager::saveBRM(bool skipSession, bool clearshm) Oam oam; string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpLogDir; log.writeLog(__LINE__, "Running reset_locks", LOG_TYPE_DEBUG); @@ -11096,7 +11097,8 @@ int ProcessManager::mountDBRoot(std::string dbrootID) //send msg to ProcMon if not local module if ( config.moduleName() == moduleName ) { - string cmd = "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/mount.txt"; + string tmpMount = tmpLogDir + "/mount.txt"; + string cmd = "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpMount; system(cmd.c_str()); if ( !rootUser) @@ -11105,14 +11107,14 @@ int ProcessManager::mountDBRoot(std::string dbrootID) system(cmd.c_str()); } - ifstream in("/tmp/mount.txt"); + ifstream in(tmpMount); in.seekg(0, std::ios::end); int size = in.tellg(); if ( size != 0 ) { - if (!oam.checkLogStatus("/tmp/mount.txt", "already")) + if (!oam.checkLogStatus(tmpMount, "already")) { log.writeLog(__LINE__, "mount failed, dbroot: " + dbrootID); return API_FAILURE; diff --git a/procmon/main.cpp b/procmon/main.cpp index 9e27f805d..6047f2dfa 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -61,7 +61,7 @@ bool mainResumeFlag; string USER = "root"; string PMwithUM = "n"; bool startProcMon = false; - +string tmpLogDir; //extern std::string gOAMParentModuleName; extern bool gOAMParentModuleFlag; @@ -138,6 +138,9 @@ int main(int argc, char** argv) string systemLang = "C"; setlocale(LC_ALL, systemLang.c_str()); + + //get tmp log directory + tmpLogDir = startup::StartUp::tmpDir(); // create message thread pthread_t MessageThread; @@ -196,7 +199,7 @@ int main(int argc, char** argv) string modType = config.moduleType(); //run the module install script - string cmd = startup::StartUp::installDir() + "/bin/module_installer.sh " + " --installdir=" + startup::StartUp::installDir() + " --module=" + modType + " > /tmp/module_installer.log 2>&1"; + string cmd = startup::StartUp::installDir() + "/bin/module_installer.sh " + " --installdir=" + startup::StartUp::installDir() + " --module=" + modType + " > " + tmpLogDir + "/module_installer.log 2>&1"; log.writeLog(__LINE__, "run module_installer.sh", LOG_TYPE_DEBUG); log.writeLog(__LINE__, cmd, LOG_TYPE_DEBUG); diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 2adf793be..7a636c487 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -53,6 +53,7 @@ extern string USER; extern bool HDFS; extern string PMwithUM; extern bool startProcMon; +extern string tmpLogDir; //std::string gOAMParentModuleName; bool gOAMParentModuleFlag; @@ -914,7 +915,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO { string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpLogDir; string cmd = startup::StartUp::installDir() + "/bin/reset_locks > " + logdir + "/reset_locks.log1 2>&1"; system(cmd.c_str()); @@ -1545,14 +1546,15 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO // install mysqld rpms if being reconfigured as a um if ( reconfigureModuleName.find("um") != string::npos ) { - string cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install >> /tmp/rpminstall"; + string cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install >> " + tmpLogDir + "/rpminstall"; system(cmd.c_str()); - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install >> /tmp/rpminstall"; + cmd = startup::StartUp::installDir() + "/bin/post-mysql-install >> " + tmpLogDir + "/rpminstall"; system(cmd.c_str()); - cmd = startup::StartUp::installDir() + "/mysql/mysql-Columnstore start > /tmp/mysqldstart"; + cmd = startup::StartUp::installDir() + "/mysql/mysql-Columnstore start > " + tmpLogDir + "/mysqldstart"; system(cmd.c_str()); - ifstream file ("/tmp/mysqldstart"); + string tmpFile = tmpLogDir + "/mysqldstart"; + ifstream file (tmpFile); if (!file) { @@ -1752,21 +1754,22 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO { int retry = 1; + string tmpUmount = tmpLogDir + "/umount.log"; for ( ; retry < 5 ; retry++) { - string cmd = "export LC_ALL=C;umount " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/umount.txt 2>&1"; + string cmd = "export LC_ALL=C;umount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpUmount + " 2>&1"; system(cmd.c_str()); return_status = API_SUCCESS; - if (!oam.checkLogStatus("/tmp/umount.txt", "busy")) + if (!oam.checkLogStatus(tmpUmount, "busy")) break; - cmd = "lsof " + startup::StartUp::installDir() + "/data" + dbrootID + " >> /tmp/umount.txt 2>&1"; - system(cmd.c_str()); - cmd = "fuser -muvf " + startup::StartUp::installDir() + "/data" + dbrootID + " >> /tmp/umount.txt 2>&1"; - system(cmd.c_str()); + cmd = "lsof " + startup::StartUp::installDir() + "/data" + dbrootID + " >> " + tmpUmount + " 2>&1"; + system(cmd.c_str()); + cmd = "fuser -muvf " + startup::StartUp::installDir() + "/data" + dbrootID + " >> " + tmpUmount + " 2>&1"; + system(cmd.c_str()); sleep(2); //Flush the cache @@ -1779,7 +1782,8 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO { log.writeLog(__LINE__, "unmount failed, device busy, DBRoot: " + dbrootID, LOG_TYPE_ERROR); return_status = API_FAILURE; - system("mv -f /tmp/umount.txt /tmp/umount_failed.txt"); + string cmd = "mv -f " + tmpUmount + " " + tmpUmount + "failed"; + system(cmd.c_str()); } } else @@ -1820,7 +1824,8 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if (DataRedundancyConfig == "n") { - string cmd = "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/mount.txt 2>&1"; + string tmpMount = tmpLogDir + "/mount.log"; + string cmd = "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpMount + "2>&1"; system(cmd.c_str()); if ( !rootUser) @@ -1830,18 +1835,19 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO } return_status = API_SUCCESS; - ifstream in("/tmp/mount.txt"); + ifstream in(tmpMount); in.seekg(0, std::ios::end); int size = in.tellg(); if ( size != 0 ) { - if (!oam.checkLogStatus("/tmp/mount.txt", "already")) + if (!oam.checkLogStatus(tmpMount, "already")) { log.writeLog(__LINE__, "mount failed, DBRoot: " + dbrootID, LOG_TYPE_ERROR); return_status = API_FAILURE; - system("mv -f /tmp/mount.txt /tmp/mount_failed.txt"); + string cmd = "mv -f " + tmpMount + " " + tmpMount + "failed"; + system(cmd.c_str()); } } } @@ -2582,7 +2588,7 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpLogDir; string cmd = startup::StartUp::installDir() + "/bin/reset_locks > " + logdir + "/reset_locks.log1 2>&1"; system(cmd.c_str()); @@ -2654,7 +2660,7 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpLogDir; string outFileName = logdir + "/" + processName + ".out"; string errFileName = logdir + "/" + processName + ".err"; @@ -3830,7 +3836,7 @@ int ProcessMonitor::buildSystemTables() { string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpLogDir; string cmd = startup::StartUp::installDir() + "/bin/dbbuilder 7 > " + logdir + "/dbbuilder.log &"; system(cmd.c_str()); @@ -4683,7 +4689,7 @@ int ProcessMonitor::runStartupTest() //run startup test script string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpLogDir; string cmd = startup::StartUp::installDir() + "/bin/startupTests.sh > " + logdir + "/startupTests.log1 2>&1"; system(cmd.c_str()); @@ -4752,7 +4758,7 @@ int ProcessMonitor::runHDFSTest() string logdir("/var/log/mariadb/columnstore"); - if (access(logdir.c_str(), W_OK) != 0) logdir = "/tmp"; + if (access(logdir.c_str(), W_OK) != 0) logdir = tmpLogDir; string hdfslog = logdir + "/hdfsCheck.log1"; @@ -5015,48 +5021,6 @@ void ProcessMonitor::checkModuleFailover( std::string processName) } } -/****************************************************************************************** -* @brief runUpgrade -* -* purpose: run upgrade script -* -******************************************************************************************/ -/*int ProcessMonitor::runUpgrade(std::string mysqlpw) -{ - Oam oam; - - for ( int i = 0 ; i < 10 ; i++ ) - { - //run upgrade script - string cmd = startup::StartUp::installDir() + "/bin/upgrade-columnstore.sh doupgrade --password=" + - mysqlpw + " --installdir=" + startup::StartUp::installDir() + " > " + "/tmp/upgrade-columnstore.log 2>&1"; - system(cmd.c_str()); - - cmd = "/tmp/upgrade-columnstore.log"; - if (oam.checkLogStatus(cmd, "OK")) { - log.writeLog(__LINE__, "upgrade-columnstore.sh: Successful return", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - else { - if (oam.checkLogStatus(cmd, "ERROR 1045") ) { - log.writeLog(__LINE__, "upgrade-columnstore.sh: Missing Password error, return success", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - - log.writeLog(__LINE__, "upgrade-columnstore.sh: Error return, check log /tmp/upgrade-status.log", LOG_TYPE_ERROR); - //restart mysqld and retry - try { - oam.actionMysqlCalpont(MYSQL_RESTART); - } - catch(...) - {} - sleep(1); - } - } - return oam::API_FAILURE; -} -*/ - /****************************************************************************************** * @brief changeMyCnf * @@ -5239,13 +5203,13 @@ int ProcessMonitor::runMariaDBCommandLine(std::string command) if ( MySQLPort.empty() ) MySQLPort = "3306"; - string cmd = startup::StartUp::installDir() + "/bin/mariadb-command-line.sh --installdir=" + startup::StartUp::installDir() + " --command='" + command + "' --port=" + MySQLPort + " > /tmp/mariadb-command-line.sh.log 2>&1"; + string cmd = startup::StartUp::installDir() + "/bin/mariadb-command-line.sh --installdir=" + startup::StartUp::installDir() + " --command='" + command + "' --port=" + MySQLPort + " > " + tmpLogDir + "/mariadb-command-line.sh.log 2>&1"; log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); system(cmd.c_str()); - string logFile = "/tmp/mariadb-command-line.sh.log"; + string logFile = tmpLogDir + "/mariadb-command-line.sh.log"; if (oam.checkLogStatus(logFile, "ERROR 1045") ) { @@ -5261,7 +5225,7 @@ int ProcessMonitor::runMariaDBCommandLine(std::string command) } else { - log.writeLog(__LINE__, "mariadb-command-line.sh: Error return, check log /tmp/mariadb-command-line.sh.log", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "mariadb-command-line.sh: Error return, check log " + tmpLogDir + "/mariadb-command-line.sh.log", LOG_TYPE_ERROR); return oam::API_FAILURE; } } @@ -5349,8 +5313,8 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master { string ipAddr = (*pt1).IPAddr; - string logFile = "/tmp/master-rep-columnstore-" + moduleName + ".log"; - string cmd = startup::StartUp::installDir() + "/bin/master-rep-columnstore.sh --installdir=" + startup::StartUp::installDir() + " --hostIP=" + ipAddr + " --port=" + MySQLPort + " > " + logFile + " 2>&1"; + string logFile = tmpLogDir + "/master-rep-columnstore-" + moduleName + ".log"; + string cmd = startup::StartUp::installDir() + "/bin/master-rep-columnstore.sh --installdir=" + startup::StartUp::installDir() + " --hostIP=" + ipAddr + " --port=" + MySQLPort + " --tmpdir=" + tmpLogDir + " > " + logFile + " 2>&1"; log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); system(cmd.c_str()); @@ -5388,13 +5352,14 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master // File Position Binlog_Do_DB Binlog_Ignore_DB // mysql-bin.000006 2921 // - // in log - /tmp/show-master-status.log + // in log - show-master-status.log - ifstream file ("/tmp/show-master-status.log"); + string masterLog = tmpLogDir + "/show-master-status.log"; + ifstream file (masterLog); if (!file) { - log.writeLog(__LINE__, "runMasterRep - show master status log file doesn't exist - /tmp/show-master-status.log", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "runMasterRep - show master status log file doesn't exist - " + masterLog, LOG_TYPE_ERROR); return oam::API_FAILURE; } else @@ -5439,7 +5404,7 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master file.close(); } - log.writeLog(__LINE__, "runMasterRep - 'mysql-bin not found in log file - /tmp/show-master-status.log", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "runMasterRep - 'mysql-bin not found in log file - " + masterLog, LOG_TYPE_ERROR); return oam::API_FAILURE; } @@ -5491,14 +5456,14 @@ int ProcessMonitor::runSlaveRep(std::string& masterLogFile, std::string& masterL while (true) { - string cmd = startup::StartUp::installDir() + "/bin/slave-rep-columnstore.sh --installdir=" + startup::StartUp::installDir() + " --masteripaddr=" + masterIPAddress + " --masterlogfile=" + masterLogFile + " --masterlogpos=" + masterLogPos + " --port=" + MySQLPort + " > /tmp/slave-rep-columnstore.log 2>&1"; + string logFile = tmpLogDir + "/slave-rep-columnstore.log"; + + string cmd = startup::StartUp::installDir() + "/bin/slave-rep-columnstore.sh --installdir=" + startup::StartUp::installDir() + " --masteripaddr=" + masterIPAddress + " --masterlogfile=" + masterLogFile + " --masterlogpos=" + masterLogPos + " --port=" + MySQLPort + " --tmpdir=" + tmpLogDir + " > " + logFile + " 2>&1"; log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); system(cmd.c_str()); - string logFile = "/tmp/slave-rep-columnstore.log"; - if (oam.checkLogStatus(logFile, "ERROR 1045") ) { if ( passwordError ) @@ -5519,7 +5484,7 @@ int ProcessMonitor::runSlaveRep(std::string& masterLogFile, std::string& masterL } else { - log.writeLog(__LINE__, "slave-rep-columnstore.sh: Error return, check log /tmp/slave-rep-columnstore.log", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "slave-rep-columnstore.sh: Error return, check log " + logFile, LOG_TYPE_ERROR); return oam::API_FAILURE; } } @@ -5554,16 +5519,16 @@ int ProcessMonitor::runDisableRep() if ( MySQLPort.empty() ) MySQLPort = "3306"; + + string logFile = tmpLogDir + "/disable-rep-columnstore.log"; - string cmd = startup::StartUp::installDir() + "/bin/disable-rep-columnstore.sh --installdir=" + startup::StartUp::installDir() + " > /tmp/disable-rep-columnstore.log 2>&1"; + string cmd = startup::StartUp::installDir() + "/bin/disable-rep-columnstore.sh --installdir=" + startup::StartUp::installDir() + " --tmpdir=" + tmpLogDir + " > " + logFile + " 2>&1"; log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); system(cmd.c_str()); - cmd = "/tmp/disable-rep-columnstore.log"; - - if (oam.checkLogStatus(cmd, "OK")) + if (oam.checkLogStatus(logFile, "OK")) { log.writeLog(__LINE__, "disable-rep-columnstore.sh: Successful return", LOG_TYPE_DEBUG); return oam::API_SUCCESS; @@ -5576,7 +5541,7 @@ int ProcessMonitor::runDisableRep() return oam::API_SUCCESS; } - log.writeLog(__LINE__, "disable-rep-columnstore.sh: Error return, check log /tmp/disable-rep-columnstore.log", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "disable-rep-columnstore.sh: Error return, check log " + logFile, LOG_TYPE_ERROR); return oam::API_FAILURE; } @@ -5647,12 +5612,12 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul { string ipAddr = (*pt1).IPAddr; - string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + moduleName + ".log"; + string logFile = tmpLogDir + "/master-dist_" + moduleName + ".log"; + string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > " + logFile; log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); system(cmd.c_str()); - string logFile = "/tmp/master-dist_" + moduleName + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) { @@ -5686,10 +5651,10 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); string ipAddr = (*pt1).IPAddr; - string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; - system(cmd.c_str()); + string logFile = tmpLogDir + "/master-dist_" + slaveModule + ".log"; - string logFile = "/tmp/master-dist_" + slaveModule + ".log"; + string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > " + logFile; + system(cmd.c_str()); if (!oam.checkLogStatus(logFile, "FAILED")) log.writeLog(__LINE__, "runMasterDist: Success rsync to module: " + slaveModule, LOG_TYPE_DEBUG); @@ -5722,7 +5687,8 @@ bool ProcessMonitor::amazonIPCheck() log.writeLog(__LINE__, "amazonIPCheck function called", LOG_TYPE_DEBUG); // delete description file so it will create a new one - unlink("/tmp/describeInstance.txt"); + string tmpLog = tmpLogDir + "/describeInstance.log"; + unlink(tmpLog.c_str()); // // Get Module Info @@ -5949,8 +5915,8 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; - - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + string mountLog = tmpLogDir + "/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > " + mountLog; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); @@ -6318,7 +6284,7 @@ int ProcessMonitor::checkDataMount() if ( DataRedundancyConfig == "n" ) { //remove any local check flag for starters - string cmd = "umount " + dbroot + " > /tmp/umount.txt 2>&1"; + string cmd = "umount " + dbroot + " > " + tmpLogDir + "/umount.log 2>&1"; system(cmd.c_str()); unlink(fileName.c_str()); @@ -6331,17 +6297,18 @@ int ProcessMonitor::checkDataMount() if (WEXITSTATUS(status) != 0 ) { // not mounted, mount - cmd = "export LC_ALL=C;mount " + dbroot + " > /tmp/mount.txt 2>&1"; + string mountLog = tmpLogDir + "/mount.log"; + cmd = "export LC_ALL=C;mount " + dbroot + " > " + mountLog + " 2>&1"; system(cmd.c_str()); - ifstream in("/tmp/mount.txt"); + ifstream in(mountLog); in.seekg(0, std::ios::end); int size = in.tellg(); if ( size != 0 ) { - if (!oam.checkLogStatus("/tmp/mount.txt", "already")) + if (!oam.checkLogStatus(mountLog, "already")) { log.writeLog(__LINE__, "checkDataMount: mount failed, DBRoot: " + dbroot, LOG_TYPE_ERROR); @@ -6533,8 +6500,8 @@ int ProcessMonitor::glusterAssign(std::string dbrootID) moduleIPAddr = sysConfig->getConfig("SystemModuleConfig", dataDupIPaddr); } - command = "mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + - dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterAssign.txt 2>&1"; + string tmpLog = tmpLogDir + "/glusterAssign.log"; + command = "mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpLog + " 2>&1"; int ret = system(command.c_str()); @@ -6542,16 +6509,17 @@ int ProcessMonitor::glusterAssign(std::string dbrootID) { log.writeLog(__LINE__, "glusterAssign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); - ifstream in("/tmp/glusterAssign.txt"); + ifstream in(tmpLog); in.seekg(0, std::ios::end); int size = in.tellg(); if ( size != 0 ) { - if (!oam.checkLogStatus("/tmp/glusterAssign.txt", "already")) + if (!oam.checkLogStatus(tmpLog, "already")) { log.writeLog(__LINE__, "glusterAssign failed.", LOG_TYPE_ERROR); - system("mv -f /tmp/glusterAssign.txt /tmp/glusterAssign_failed.txt"); + string cmd = "mv -f " + tmpLog + " " + tmpLog + "failed"; + system(cmd.c_str()); return oam::API_FAILURE; } } @@ -6575,8 +6543,10 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) std::string errmsg = ""; log.writeLog(__LINE__, "glusterUnassign called: " + dbrootID, LOG_TYPE_DEBUG); - - command = "umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterUnassign.txt 2>&1"; + + string tmpLog = tmpLogDir + "/glusterUnassign.log"; + + command = "umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpLog + " 2>&1"; int ret = system(command.c_str()); @@ -6584,16 +6554,18 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) { log.writeLog(__LINE__, "glusterUnassign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); - ifstream in("/tmp/glusterUnassign.txt"); + ifstream in(tmpLog); in.seekg(0, std::ios::end); int size = in.tellg(); if ( size != 0 ) { - if (!oam.checkLogStatus("/tmp/glusterUnassign.txt", "not mounted")) + if (!oam.checkLogStatus(tmpLog, "not mounted")) { log.writeLog(__LINE__, "glusterUnassign failed.", LOG_TYPE_ERROR); - system("mv -f /tmp/glusterUnassign.txt /tmp/glusterUnassign_failed.txt"); + + string cmd = "mv -f " + tmpLog + " " + tmpLog + "failed"; + system(cmd.c_str()); return oam::API_FAILURE; } } From 148eb6bf58e1534f863c18f9fbec150b976bc1e3 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 24 Sep 2018 14:20:17 -0500 Subject: [PATCH 023/127] MCOL-520 --- oam/install_scripts/columnstore | 7 +++-- oam/install_scripts/mariadb-command-line.sh | 30 ++++++++++--------- oam/install_scripts/module_installer.sh | 21 +++++++------ oam/install_scripts/remote_command.sh | 6 ++-- oam/install_scripts/remote_scp_get.sh | 6 ++-- oam/install_scripts/remote_scp_put.sh | 6 ++-- oam/install_scripts/run.sh | 5 +++- oam/install_scripts/syslogSetup.sh | 11 ++++--- oamapps/alarmmanager/alarmmanager.cpp | 16 +++++----- oamapps/columnstoreSupport/bulklogReport.sh | 7 +++-- .../columnstoreSupport/columnstoreSupport.cpp | 22 ++++++++------ oamapps/columnstoreSupport/configReport.sh | 7 +++-- oamapps/columnstoreSupport/hadoopReport.sh | 7 +++-- oamapps/columnstoreSupport/hardwareReport.sh | 7 +++-- oamapps/columnstoreSupport/logReport.sh | 7 +++-- oamapps/columnstoreSupport/resourceReport.sh | 7 +++-- oamapps/columnstoreSupport/softwareReport.sh | 7 +++-- procmgr/processmanager.cpp | 2 +- procmon/processmonitor.cpp | 2 +- 19 files changed, 112 insertions(+), 71 deletions(-) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index 2e1932d97..71dc7ee5c 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -58,6 +58,9 @@ fi lockdir=`$InstallDir/bin/getConfig Installation LockFileDirectory` +#get temp directory +tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` + checkInstallSetup() { InitialInstallFlag=`$InstallDir/bin/getConfig -c $InstallDir/etc/Columnstore.xml Installation InitialInstallFlag` if [ $InitialInstallFlag != "y" ]; then @@ -99,14 +102,14 @@ start() { RETVAL=0 echo "Starting MariaDB Columnstore Database Platform" - rm -f /tmp/StopColumnstore + rm -f ${tmpDir}/StopColumnstore exec $InstallDir/bin/run.sh $InstallDir/bin/ProcMon > /dev/null 2>&1 & return $RETVAL } stop() { echo "Shutting down MariaDB Columnstore Database Platform" - touch /tmp/StopColumnstore + touch ${tmpDir}/StopColumnstore pkill ProcMon pkill ProcMgr sleep 1 diff --git a/oam/install_scripts/mariadb-command-line.sh b/oam/install_scripts/mariadb-command-line.sh index efdbbddd5..86719deb1 100755 --- a/oam/install_scripts/mariadb-command-line.sh +++ b/oam/install_scripts/mariadb-command-line.sh @@ -8,20 +8,20 @@ # check log for error checkForError() { - grep "ERROR 1045" /tmp/mariadb-command-line.log > /tmp/error.check - if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then - echo "ERROR - PASSWORD: check log file: /tmp/mariadb-command-line.log" - rm -f /tmp/error.check + grep "ERROR 1045" ${tmpdir}/mariadb-command-line.log > ${tmpdir}/error.check + if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then + echo "ERROR - PASSWORD: check log file: ${tmpdir}/mariadb-command-line.log" + rm -f ${tmpdir}/error.check exit 2 fi - grep ERROR /tmp/mariadb-command-line.log > /tmp/error.check - if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then - echo "ERROR: check log file: /tmp/mariadb-command-line.log" - rm -f /tmp/error.check + grep ERROR ${tmpdir}/mariadb-command-line.log > ${tmpdir}/error.check + if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then + echo "ERROR: check log file: ${tmpdir}/mariadb-command-line.log" + rm -f ${tmpdir}/error.check exit 1 fi - rm -f /tmp/error.check + rm -f ${tmpdir}/error.check } prefix=/usr/local @@ -37,27 +37,29 @@ for arg in "$@"; do prefix=`dirname $installdir` elif [ `expr -- "$arg" : '--port='` -eq 7 ]; then port="`echo $arg | awk -F= '{print $2}'`" + elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then + tmpdir="$(echo $arg | awk -F= '{print $2}')" fi done test -f $installdir/post/functions && . $installdir/post/functions ->/tmp/mariadb-command-line.log +>${tmpdir}/mariadb-command-line.log # # Run command # -echo "Run command" >>/tmp/mariadb-command-line.log -cat >/tmp/mariadb-command-line.sql <>${tmpdir}/mariadb-command-line.log +cat >${tmpdir}/mariadb-command-line.sql <> /tmp/mariadb-command-line.log +cat${tmpdir}/mariadb-command-line.sql >> ${tmpdir}/mariadb-command-line.log $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ --user=root \ - calpontsys < /tmp/mariadb-command-line.sql >> /tmp/mariadb-command-line.log 2>&1 + calpontsys < ${tmpdir}/mariadb-command-line.sql >> ${tmpdir}/mariadb-command-line.log 2>&1 checkForError diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 3a690a229..0d9008fd4 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -55,6 +55,9 @@ fi PMwithUM=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation PMwithUM` ServerTypeInstall=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation ServerTypeInstall` +#get temp directory +tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` + cloud=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation Cloud` if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then echo "Amazon setup on Module" @@ -121,40 +124,40 @@ MySQLRep=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation MySQLRep` if [ $MySQLRep = "y" ]; then if test -f $COLUMNSTORE_INSTALL_DIR/mysql/my.cnf ; then echo "Run Upgrade on my.cnf on Module" - $COLUMNSTORE_INSTALL_DIR/bin/mycnfUpgrade > /tmp/mycnfUpgrade.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/mycnfUpgrade > ${tmpDir}/mycnfUpgrade.log 2>&1 fi fi if test -f $COLUMNSTORE_INSTALL_DIR/mysql/my.cnf ; then mysqlPort=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation MySQLPort` echo "Run Mysql Port update on my.cnf on Module" - $COLUMNSTORE_INSTALL_DIR/bin/mycnfUpgrade $mysqlPort > /tmp/mycnfUpgrade_port.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/mycnfUpgrade $mysqlPort > ${tmpDir}/mycnfUpgrade_port.log 2>&1 fi # if um, run mysql install scripts if [ $module = "um" ] || ( [ $module = "pm" ] && [ $PMwithUM = "y" ] ) || [ $ServerTypeInstall = "2" ]; then echo "Run post-mysqld-install" - $COLUMNSTORE_INSTALL_DIR/bin/post-mysqld-install --installdir=$COLUMNSTORE_INSTALL_DIR > /tmp/post-mysqld-install.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/post-mysqld-install --installdir=$COLUMNSTORE_INSTALL_DIR > ${tmpDir}/post-mysqld-install.log 2>&1 if [ $? -ne 0 ]; then - echo "ERROR: post-mysqld-install failed: check /tmp/post-mysqld-install.log" + echo "ERROR: post-mysqld-install failed: check ${tmpDir}/post-mysqld-install.log" exit 1 fi echo "Run post-mysql-install" - $COLUMNSTORE_INSTALL_DIR/bin/post-mysql-install --installdir=$COLUMNSTORE_INSTALL_DIR > /tmp/post-mysql-install.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/post-mysql-install --installdir=$COLUMNSTORE_INSTALL_DIR > ${tmpDir}/post-mysql-install.log 2>&1 if [ $? -ne 0 ]; then - echo "ERROR: post-mysql-install failed: check /tmp/post-mysql-install.log" + echo "ERROR: post-mysql-install failed: check ${tmpDir}/post-mysql-install.log" exit 1 fi fi if [ $user == "root" ]; then - $COLUMNSTORE_INSTALL_DIR/bin/syslogSetup.sh check > /tmp/syslogSetup-check.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/syslogSetup.sh check > ${tmpDir}/syslogSetup-check.log 2>&1 if [ $? -ne 0 ]; then # try setup again - $COLUMNSTORE_INSTALL_DIR/bin/syslogSetup.sh install > /tmp/syslogSetup-install.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/syslogSetup.sh install > ${tmpDir}/syslogSetup-install.log 2>&1 if [ $? -ne 0 ]; then - echo "WARNING: syslogSetup.sh check failed: check /tmp/syslogSetup-check.log" + echo "WARNING: syslogSetup.sh check failed: check ${tmpDir}/syslogSetup-check.log" exit 2 fi fi diff --git a/oam/install_scripts/remote_command.sh b/oam/install_scripts/remote_command.sh index 75095632e..85e76306f 100755 --- a/oam/install_scripts/remote_command.sh +++ b/oam/install_scripts/remote_command.sh @@ -16,9 +16,9 @@ set PASSWORD [lindex $argv 1] set COMMAND [lindex $argv 2] set DEBUG [lindex $argv 3] -exec whoami >/tmp/whoami.tmp -set USERNAME [exec cat /tmp/whoami.tmp] -exec rm -f /tmp/whoami.tmp +exec whoami >whoami.tmp +set USERNAME [exec cat whoami.tmp] +exec rm -f whoami.tmp set UNM [lindex $argv 4] if { $UNM != "" && $UNM != "-" } { diff --git a/oam/install_scripts/remote_scp_get.sh b/oam/install_scripts/remote_scp_get.sh index 61a1de219..96b8d644e 100755 --- a/oam/install_scripts/remote_scp_get.sh +++ b/oam/install_scripts/remote_scp_get.sh @@ -7,9 +7,9 @@ # Argument 2 - Remote Server root password # Argument 3 - Command set timeout 10 -exec whoami >/tmp/whoami.tmp -set USERNAME [exec cat /tmp/whoami.tmp] -exec rm -f /tmp/whoami.tmp +exec whoami > whoami.tmp +set USERNAME [exec cat whoami.tmp] +exec rm -f whoami.tmp set SERVER [lindex $argv 0] set PASSWORD [lindex $argv 1] set FILE [lindex $argv 2] diff --git a/oam/install_scripts/remote_scp_put.sh b/oam/install_scripts/remote_scp_put.sh index c94ea666e..cee9211a7 100644 --- a/oam/install_scripts/remote_scp_put.sh +++ b/oam/install_scripts/remote_scp_put.sh @@ -7,9 +7,9 @@ # Argument 2 - Remote Server root password # Argument 3 - Command set timeout 30 -exec whoami >/tmp/whoami.tmp -set USERNAME [exec cat /tmp/whoami.tmp] -exec rm -f /tmp/whoami.tmp +exec whoami >whoami.tmp +set USERNAME [exec cat whoami.tmp] +exec rm -f whoami.tmp set SERVER [lindex $argv 0] set PASSWORD [lindex $argv 1] set FILE [lindex $argv 2] diff --git a/oam/install_scripts/run.sh b/oam/install_scripts/run.sh index 0cef59ffe..accdd3be2 100755 --- a/oam/install_scripts/run.sh +++ b/oam/install_scripts/run.sh @@ -45,9 +45,12 @@ if [ $vflg -gt 0 ]; then echo "starting $exename $args with sleep=$sopt and tries=$topt" fi +#get temp directory +tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` + while [ $keep_going -ne 0 ]; do $exename $args - if [ -e /tmp/StopColumnstore ]; then + if [ -e ${tmpDir}/StopColumnstore ]; then exit 0 fi if [ $topt -gt 0 -a $retries -ge $topt ]; then diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 9a96cb8ee..347889f4a 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -47,6 +47,9 @@ fi columnstoreSyslogFile=$installdir/bin/columnstoreSyslog columnstoreSyslogFile7=$installdir/bin/columnstoreSyslog7 +#get temp directory +tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` + checkSyslog() { #check which syslog daemon is being used #first check which is running @@ -101,16 +104,16 @@ if [ "$daemon" = "syslog-ng" ]; then fi elif [ "$daemon" = "rsyslog" ]; then #check if rsyslog version 7 or greater - rsyslogd -v > /tmp/rsyslog.ver - cnt=`grep "rsyslogd 7" /tmp/rsyslog.ver | wc -l` + rsyslogd -v > ${tmpDir}/rsyslog.ver + cnt=`grep "rsyslogd 7" ${tmpDir}/rsyslog.ver | wc -l` if [ $cnt -gt 0 ]; then rsyslog7=1 fi - cnt=`grep "rsyslogd 8" /tmp/rsyslog.ver | wc -l` + cnt=`grep "rsyslogd 8" ${tmpDir}/rsyslog.ver | wc -l` if [ $cnt -gt 0 ]; then rsyslog7=1 fi - cnt=`grep "rsyslogd 9" /tmp/rsyslog.ver | wc -l` + cnt=`grep "rsyslogd 9" ${tmpDir}/rsyslog.ver | wc -l` if [ $cnt -gt 0 ]; then rsyslog7=1 fi diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index 8579fcba6..10ca418d1 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -621,15 +621,17 @@ void ALARMManager::getActiveAlarm(AlarmList& alarmList) const *****************************************************************************************/ void ALARMManager::getAlarm(std::string date, AlarmList& alarmList) const { - string alarmFile = "/tmp/alarms"; + + string alarmFile = startup::StartUp::tmpDir() + "/alarms"; //make 1 alarm log file made up of archive and current alarm.log - (void)system("touch /tmp/alarms"); - - string cmd = ("ls " + ALARM_ARCHIVE_FILE + " | grep 'alarm.log' > /tmp/alarmlogfiles"); + string cmd = "touch " + alarmFile; (void)system(cmd.c_str()); - string fileName = "/tmp/alarmlogfiles"; + cmd = "ls " + ALARM_ARCHIVE_FILE + " | grep 'alarm.log' > " + alarmFile; + (void)system(cmd.c_str()); + + string fileName = startup::StartUp::tmpDir() + "/alarmlogfiles"; ifstream oldFile (fileName.c_str()); @@ -641,7 +643,7 @@ void ALARMManager::getAlarm(std::string date, AlarmList& alarmList) const while (oldFile.getline(line, 200)) { buf = line; - string cmd = "cat " + ALARM_ARCHIVE_FILE + "/" + buf + " >> /tmp/alarms"; + string cmd = "cat " + ALARM_ARCHIVE_FILE + "/" + buf + " >> " + alarmFile; (void)system(cmd.c_str()); } @@ -649,7 +651,7 @@ void ALARMManager::getAlarm(std::string date, AlarmList& alarmList) const unlink (fileName.c_str()); } - cmd = "cat " + ALARM_FILE + " >> /tmp/alarms"; + cmd = "cat " + ALARM_FILE + " >> " + alarmFile; (void)system(cmd.c_str()); int fd = open(alarmFile.c_str(), O_RDONLY); diff --git a/oamapps/columnstoreSupport/bulklogReport.sh b/oamapps/columnstoreSupport/bulklogReport.sh index 29398b984..903ab8002 100755 --- a/oamapps/columnstoreSupport/bulklogReport.sh +++ b/oamapps/columnstoreSupport/bulklogReport.sh @@ -14,7 +14,10 @@ else INSTALLDIR="/usr/local/mariadb/columnstore" fi -rm -f /tmp/${MODULE}_bulklogReport.txt +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/${MODULE}_bulklogReport.txt { @@ -27,6 +30,6 @@ if test -d $INSTALLDIR/data/bulk ; then egrep '(ERR|CRIT)' $INSTALLDIR/data/bulk/log/*.err 2>/dev/null fi -} > /tmp/${MODULE}_bulklogReport.txt +} > ${tmpDir}/${MODULE}_bulklogReport.txt exit 0 diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp index 742f81e97..0232cd9c6 100644 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -57,6 +57,7 @@ ChildModule childmodule; string rootPassword = ""; string debug_flag = "0"; string mysqlpw = " "; +string tmpDir; int runningThreads = 0; pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; @@ -139,11 +140,11 @@ void childReportThread(threadInfo_t& st) cout << "Error with running remote_command.sh, exiting..." << endl; } - cmd = installDir + "/bin/remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " /tmp/" + outputFile + " > /dev/null 2>&1"; + cmd = installDir + "/bin/remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile + " > /dev/null 2>&1"; rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) - cout << "ERROR: failed to retrieve /tmp/" << outputFile << " from " + remoteHostName << endl; + cout << "ERROR: failed to retrieve " << tmpDir << "/" << outputFile << " from " + remoteHostName << endl; pthread_mutex_lock( &mutex1 ); runningThreads--; @@ -198,7 +199,7 @@ void reportThread(string reporttype) string cmd = installDir + "/bin/logReport.sh " + localModule + " " + installDir; system(cmd.c_str()); - cmd = "mv -f /tmp/" + localModule + "_logReport.tar.gz ."; + cmd = "mv -f " + tmpDir + "/" + localModule + "_logReport.tar.gz ."; system(cmd.c_str()); cmd = "tar -zcf " + localModule + "_mysqllogReport.tar.gz " + installDir + "/mysql/db/*.err* 2>/dev/null"; @@ -234,7 +235,7 @@ void reportThread(string reporttype) cmd = installDir + "/bin/" + reportType + "Report.sh " + localModule + " " + installDir; system(cmd.c_str()); - cmd = " mv -f /tmp/" + localModule + "_" + reportType + "Report.txt ."; + cmd = " mv -f " + tmpDir + "/" + localModule + "_" + reportType + "Report.txt ."; system(cmd.c_str()); if (reportType == "config" ) @@ -431,6 +432,8 @@ int main(int argc, char* argv[]) cout << "ERROR: Problem accessing Columnstore configuration file" << endl; exit(-1); } + + tmpDir = startup::StartUp::tmpDir(); for ( int i = 1; i < argc; i++ ) { @@ -842,14 +845,15 @@ int main(int argc, char* argv[]) else { // check if mysql is supported and get info + string logFile = tmpDir + "/idbmysql.log"; string columnstoreMysql = installDir + "/mysql/bin/mysql --defaults-extra-file=" + installDir + "/mysql/my.cnf -u root "; - string cmd = columnstoreMysql + " -e 'status' > /tmp/idbmysql.log 2>&1"; + string cmd = columnstoreMysql + " -e 'status' > " + logFile + " 2>&1"; system(cmd.c_str()); //check for mysql password set string pwprompt = " "; - if (oam.checkLogStatus("/tmp/idbmysql.log", "ERROR 1045") ) + if (oam.checkLogStatus(logFile, "ERROR 1045") ) { cout << "NOTE: MariaDB Columnstore root user password is set" << endl; @@ -903,10 +907,10 @@ int main(int argc, char* argv[]) //check for mysql password set pwprompt = "--password=" + mysqlpw; - string cmd = columnstoreMysql + pwprompt + " -e 'status' > /tmp/idbmysql.log 2>&1"; + string cmd = columnstoreMysql + pwprompt + " -e 'status' > " + logFile + " 2>&1"; system(cmd.c_str()); - if (oam.checkLogStatus("/tmp/idbmysql.log", "ERROR 1045") ) + if (oam.checkLogStatus(logFile, "ERROR 1045") ) { cout << "FAILED: Failed login using MariaDB Columnstore root user password '" << mysqlpw << "'" << endl; FAILED = true; @@ -1048,7 +1052,7 @@ int main(int argc, char* argv[]) { cout << "Get hadoop report data" << endl; string cmd = installDir + "/bin/hadoopReport.sh " + localModule + " " + installDir + "\n"; - cmd += " mv -f /tmp/hadoopReport.txt ."; + cmd += " mv -f " + tmpDir + "/hadoopReport.txt ."; FILE* pipe = popen(cmd.c_str(), "r"); if (!pipe) diff --git a/oamapps/columnstoreSupport/configReport.sh b/oamapps/columnstoreSupport/configReport.sh index 2a73479e7..722935b10 100755 --- a/oamapps/columnstoreSupport/configReport.sh +++ b/oamapps/columnstoreSupport/configReport.sh @@ -14,7 +14,10 @@ else INSTALLDIR="/usr/local/mariadb/columnstore" fi -rm -f /tmp/${MODULE}_configReport.txt +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/${MODULE}_configReport.txt { echo " " @@ -79,6 +82,6 @@ echo "################# top -b -n 1 #################" echo " " top -b -n 1 -} > /tmp/${MODULE}_configReport.txt +} > ${tmpDir}/${MODULE}_configReport.txt exit 0 diff --git a/oamapps/columnstoreSupport/hadoopReport.sh b/oamapps/columnstoreSupport/hadoopReport.sh index 7b45b6d9d..eb80de805 100755 --- a/oamapps/columnstoreSupport/hadoopReport.sh +++ b/oamapps/columnstoreSupport/hadoopReport.sh @@ -13,7 +13,10 @@ else INSTALLDIR="/usr/local/mariadb/columnstore" fi -rm -f /tmp/hdfsReport.txt +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/hdfsReport.txt { echo @@ -58,6 +61,6 @@ echo echo "################# hdfs fsck $INSTALLDIR #################" echo hadoop fsck $INSTALLDIR 2>/dev/null -} > /tmp/hadoopReport.txt +} > ${tmpDir}/hadoopReport.txt exit 0 diff --git a/oamapps/columnstoreSupport/hardwareReport.sh b/oamapps/columnstoreSupport/hardwareReport.sh index c1db4bf67..53e5e7ca3 100755 --- a/oamapps/columnstoreSupport/hardwareReport.sh +++ b/oamapps/columnstoreSupport/hardwareReport.sh @@ -14,7 +14,10 @@ else INSTALLDIR="/usr/local/mariadb/columnstore" fi -rm -f /tmp/${MODULE}_hardwareReport.txt +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/${MODULE}_hardwareReport.txt { echo " " @@ -76,6 +79,6 @@ echo "################# ifconfig -a #################" echo " " ifconfig -a 2>/dev/null -} > /tmp/${MODULE}_hardwareReport.txt +} > ${tmpDir}/${MODULE}_hardwareReport.txt exit 0 diff --git a/oamapps/columnstoreSupport/logReport.sh b/oamapps/columnstoreSupport/logReport.sh index 72bbb8da1..8655b664c 100755 --- a/oamapps/columnstoreSupport/logReport.sh +++ b/oamapps/columnstoreSupport/logReport.sh @@ -14,8 +14,11 @@ else INSTALLDIR="/usr/local/mariadb/columnstore" fi -rm -f /tmp/${MODULE}_logReport.tar.gz +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` -tar -zcf /tmp/${MODULE}_logReport.tar.gz /var/log/mariadb/columnstore > /dev/null 2>&1 +rm -f ${tmpDir}/${MODULE}_logReport.tar.gz + +tar -zcf ${tmpDir}/${MODULE}_logReport.tar.gz /var/log/mariadb/columnstore > /dev/null 2>&1 exit 0 diff --git a/oamapps/columnstoreSupport/resourceReport.sh b/oamapps/columnstoreSupport/resourceReport.sh index f1c1be617..24add52c8 100755 --- a/oamapps/columnstoreSupport/resourceReport.sh +++ b/oamapps/columnstoreSupport/resourceReport.sh @@ -14,7 +14,10 @@ else INSTALLDIR="/usr/local/mariadb/columnstore" fi -rm -f /tmp/${MODULE}_resourceReport.txt +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/${MODULE}_resourceReport.txt { echo " " @@ -63,6 +66,6 @@ echo "################# bin/editem -i #################" echo " " $INSTALLDIR/bin/editem -i 2>/dev/null -} > /tmp/${MODULE}_resourceReport.txt +} > ${tmpDir}/${MODULE}_resourceReport.txt exit 0 diff --git a/oamapps/columnstoreSupport/softwareReport.sh b/oamapps/columnstoreSupport/softwareReport.sh index d94f5c7b3..a4f2221c2 100755 --- a/oamapps/columnstoreSupport/softwareReport.sh +++ b/oamapps/columnstoreSupport/softwareReport.sh @@ -14,7 +14,10 @@ else INSTALLDIR="/usr/local/mariadb/columnstore" fi -rm -f /tmp/${MODULE}_softwareReport.txt +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/${MODULE}_softwareReport.txt { echo " " @@ -35,6 +38,6 @@ echo "################# mcsadmin getStorageConfig #################" echo " " $INSTALLDIR/bin/mcsadmin getStorageConfig -} > /tmp/${MODULE}_softwareReport.txt +} > ${tmpDir}/${MODULE}_softwareReport.txt exit 0 diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 76c46d733..30e69c77f 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -11097,7 +11097,7 @@ int ProcessManager::mountDBRoot(std::string dbrootID) //send msg to ProcMon if not local module if ( config.moduleName() == moduleName ) { - string tmpMount = tmpLogDir + "/mount.txt"; + string tmpMount = tmpLogDir + "/mount.log"; string cmd = "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpMount; system(cmd.c_str()); diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 7a636c487..a59d785ed 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -5203,7 +5203,7 @@ int ProcessMonitor::runMariaDBCommandLine(std::string command) if ( MySQLPort.empty() ) MySQLPort = "3306"; - string cmd = startup::StartUp::installDir() + "/bin/mariadb-command-line.sh --installdir=" + startup::StartUp::installDir() + " --command='" + command + "' --port=" + MySQLPort + " > " + tmpLogDir + "/mariadb-command-line.sh.log 2>&1"; + string cmd = startup::StartUp::installDir() + "/bin/mariadb-command-line.sh --installdir=" + startup::StartUp::installDir() + " --command='" + command + "' --port=" + MySQLPort + "--tmpdir=" + tmpLogDir + " > " + tmpLogDir + "/mariadb-command-line.sh.log 2>&1"; log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); From 7142daac5a7a269963546d24b2b97a76e8ec59ed Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 24 Sep 2018 14:21:00 -0500 Subject: [PATCH 024/127] MCOL-520 --- oamapps/columnstoreSupport/minMaxCheck.sh | 55 ++++++++++++----------- oamapps/serverMonitor/msgProcessor.cpp | 2 +- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/oamapps/columnstoreSupport/minMaxCheck.sh b/oamapps/columnstoreSupport/minMaxCheck.sh index 4e36db272..ca6b158e1 100755 --- a/oamapps/columnstoreSupport/minMaxCheck.sh +++ b/oamapps/columnstoreSupport/minMaxCheck.sh @@ -32,10 +32,10 @@ # 2) Two warning.log entries will be logged at the end of the script if one or min/max EM entries were corrected. # Example: # Mar 11 14:16:36 srvqaperf8 oamcpp[16364]: 36.231731 |0|0|0| W 08 CAL0000: min-max-monitor: some values were reset for oids 3004 3005 -# Mar 11 14:16:36 srvqaperf8 oamcpp[16365]: 36.263270 |0|0|0| W 08 CAL0000: min-max-monitor: log files are /tmp/idb_mm_mon.sdiff.*.15190 +# Mar 11 14:16:36 srvqaperf8 oamcpp[16365]: 36.263270 |0|0|0| W 08 CAL0000: min-max-monitor: log files are idb_mm_mon.sdiff.*.15190 # 3) The script outputs the results of the selects in #3 above as it's going through the columns and will echo a line when it finds a col that was corrected. # Example: -# **** Extent map min/max changed on the scan. See results in /tmp/idb_mm_mon.sdiff.3039.15190 +# **** Extent map min/max changed on the scan. See results in idb_mm_mon.sdiff.3039.15190 # Define the cols array. Here's a sql statement that will list the date and datetime cols in the expected format. @@ -56,6 +56,9 @@ if [ -z "$PGMPATH" ]; then PGMPATH=$INSTALLDIR/bin fi +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` + cols=( 1339664:tpch1.orders.o_orderdate 1339718:tpch1.lineitem.l_shipdate @@ -71,18 +74,18 @@ cols=( # If called with "all", run the script against all of the column types that use CP. # if [ $# -eq 1 ] && [ "$1" == "all" ]; then - $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7);" calpontsys --skip-column-names > /tmp/idb_mm_mon.cols - cols=( $( cat /tmp/idb_mm_mon.cols ) ) - rm -f /tmp/idb_mm_mon.cols + $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7);" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols + cols=( $( cat${tmpDir}/idb_mm_mon.cols ) ) + rm -f ${tmpDir}/idb_mm_mon.cols # # Else if one parm passed, run against the columns in the given schema. # elif [ $# -eq 1 ]; then db=$1 - $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7) and \`schema\` = '$db';" calpontsys --skip-column-names > /tmp/idb_mm_mon.cols - cols=( $( cat /tmp/idb_mm_mon.cols ) ) - rm -f /tmp/idb_mm_mon.cols + $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7) and \`schema\` = '$db';" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols + cols=( $( cat ${tmpDir}/idb_mm_mon.cols ) ) + rm -f ${tmpDir}/idb_mm_mon.cols # # Else if two parms passed, run the script against all the columns that use CP for that table. @@ -90,9 +93,9 @@ elif [ $# -eq 1 ]; then elif [ $# -eq 2 ]; then db=$1 tbl=$2 - $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7) and \`schema\` = '$db' and tablename = '$tbl';" calpontsys --skip-column-names > /tmp/idb_mm_mon.cols - cols=( $( cat /tmp/idb_mm_mon.cols ) ) - rm -f /tmp/idb_mm_mon.cols + $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7) and \`schema\` = '$db' and tablename = '$tbl';" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols + cols=( $( cat ${tmpDir}/idb_mm_mon.cols ) ) + rm -f ${tmpDir}/idb_mm_mon.cols # # Else if three parms passed, run the script against the column. @@ -101,9 +104,9 @@ elif [ $# -eq 3 ]; then db=$1 tbl=$2 col=$3 - $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where \`schema\` = '$db' and tablename = '$tbl' and columnname='$col';" calpontsys --skip-column-names > /tmp/idb_mm_mon.cols - cols=( $( cat /tmp/idb_mm_mon.cols ) ) - rm -f /tmp/idb_mm_mon.cols + $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where \`schema\` = '$db' and tablename = '$tbl' and columnname='$col';" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols + cols=( $( cat ${tmpDir}/idb_mm_mon.cols ) ) + rm -f ${tmpDir}/idb_mm_mon.cols fi i=0 @@ -127,29 +130,29 @@ while [ $i -lt ${#cols[@]} ]; do # Look up the oid if the cols array is being used to keep from having to continually update the array if tables are dropped and recreated. # if [ $# -eq 0 ]; then - $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where \`schema\` = '$schema' and tablename='$table' and columnname='$column';" calpontsys --skip-column-names > /tmp/idb_mm_mon.cols - results=`wc -l /tmp/idb_mm_mon.cols | awk '{print $1}'` + $MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where \`schema\` = '$schema' and tablename='$table' and columnname='$column';" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols + results=`wc -l ${tmpDir}/idb_mm_mon.cols | awk '{print $1}'` if [ $results -eq 0 ]; then oid=0 else - oid=`cat /tmp/idb_mm_mon.cols` + oid=`cat ${tmpDir}/idb_mm_mon.cols` fi fi - $PGMPATH/editem -o$oid | awk '{print $1, $6, $8, $12}' >/tmp/idb_mm_mon.$oid.1.$$ + $PGMPATH/editem -o$oid | awk '{print $1, $6, $8, $12}' >${tmpDir}/idb_mm_mon.$oid.1.$$ $PGMPATH/editem -c$oid $MYSQLCMD --execute="select count($column) from $table" $schema -vvv - $PGMPATH/editem -o$oid | awk '{print $1, $6, $8, $12}' >/tmp/idb_mm_mon.$oid.2.$$ - sdiff /tmp/idb_mm_mon.$oid.1.$$ /tmp/idb_mm_mon.$oid.2.$$ --suppress-common-lines | grep -n -v invalid > /tmp/idb_mm_mon.sdiff.$oid.$$ - count=`wc -l /tmp/idb_mm_mon.sdiff.$oid.$$ | awk '{print $1}'` + $PGMPATH/editem -o$oid | awk '{print $1, $6, $8, $12}' >${tmpDir}/idb_mm_mon.$oid.2.$$ + sdiff ${tmpDir}/idb_mm_mon.$oid.1.$$ ${tmpDir}/idb_mm_mon.$oid.2.$$ --suppress-common-lines | grep -n -v invalid > ${tmpDir}/idb_mm_mon.sdiff.$oid.$$ + count=`wc -l ${tmpDir}/idb_mm_mon.sdiff.$oid.$$ | awk '{print $1}'` if [ $count -ne 0 ]; then badoidlist="$badoidlist $oid" ((j++)) - echo "**** Extent map min/max changed on the scan. See results in /tmp/idb_mm_mon.sdiff.$oid.$$" + echo "**** Extent map min/max changed on the scan. See results in ${tmpDir}/idb_mm_mon.sdiff.$oid.$$" else - rm -f /tmp/idb_mm_mon.sdiff.$oid.$$ + rm -f ${tmpDir}/idb_mm_mon.sdiff.$oid.$$ fi - rm -f /tmp/idb_mm_mon.$oid.*.$$ + rm -f ${tmpDir}/idb_mm_mon.$oid.*.$$ ((i++)) done echo "" @@ -160,9 +163,9 @@ if [ $j -eq 0 ]; then exit 0 else $PGMPATH/cplogger -w 0 "min-max-monitor: some values were reset for oids" "$badoidlist" - $PGMPATH/cplogger -w 0 "min-max-monitor: log files are /tmp/idb_mm_mon.sdiff.*.$$" + $PGMPATH/cplogger -w 0 "min-max-monitor: log files are ${tmpDir}/idb_mm_mon.sdiff.*.$$" echo "min-max-monitor: some values were reset for oids" "$badoidlist" - echo "min-max-monitor: log files are /tmp/idb_mm_mon.sdiff.*.$$" + echo "min-max-monitor: log files are ${tmpDir}/idb_mm_mon.sdiff.*.$$" echo "" exit 1 fi diff --git a/oamapps/serverMonitor/msgProcessor.cpp b/oamapps/serverMonitor/msgProcessor.cpp index 9b66e0a6e..c4a7fe53d 100644 --- a/oamapps/serverMonitor/msgProcessor.cpp +++ b/oamapps/serverMonitor/msgProcessor.cpp @@ -629,7 +629,7 @@ void msgProcessor() MessageLog ml(lid); Message msg; Message::Args args; - args.add("RUN_DBHEALTH_CHECK failed, check /tmp/dbhealthTest.log"); + args.add("RUN_DBHEALTH_CHECK failed, check dbhealthTest.log"); msg.format(args); ml.logDebugMessage(msg); } From bb355df98c85005e015643f990d01a5ce29d4e19 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 24 Sep 2018 14:54:09 -0500 Subject: [PATCH 025/127] MCOL-520 --- oamapps/columnstoreSupport/alarmReport.sh | 9 ++- oamapps/columnstoreSupport/sqlLogs.sh | 11 ++-- oamapps/mcsadmin/mcsadmin.cpp | 79 ++++++++++++++--------- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/oamapps/columnstoreSupport/alarmReport.sh b/oamapps/columnstoreSupport/alarmReport.sh index 865ea95a4..098c90705 100755 --- a/oamapps/columnstoreSupport/alarmReport.sh +++ b/oamapps/columnstoreSupport/alarmReport.sh @@ -14,7 +14,10 @@ else DATE=" " fi -rm -f /tmp/logReport.log +#get temp directory +tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/logReport.log { echo " " @@ -31,6 +34,6 @@ if test -f /var/log/mariadb/columnstore/archive/alarm.log-$DATE ; then cat /var/log/mariadb/columnstore/archive/alarm.log-$DATE 2>/dev/null fi -} > /tmp/logReport.log +} > ${tmpDir}/logReport.log -exit 0 \ No newline at end of file +exit 0 diff --git a/oamapps/columnstoreSupport/sqlLogs.sh b/oamapps/columnstoreSupport/sqlLogs.sh index 564973e7e..e34ebe5b9 100755 --- a/oamapps/columnstoreSupport/sqlLogs.sh +++ b/oamapps/columnstoreSupport/sqlLogs.sh @@ -23,6 +23,9 @@ if [ -z "$MYSQLCMD" ]; then export MYSQLCMD fi +#get temp directory +tmpDir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir` + main() { if [ "$option" == "usage" ]; then @@ -111,7 +114,7 @@ create () prevSession=$3; } print $0 "|" val "|" - }' | sort -t '|' -n -k 1 > /tmp/idbtmp.tbl + }' | sort -t '|' -n -k 1 > ${tmpDir}/idbtmp.tbl echo "Step 2 of 4. Populating $DB.start table with Start SQL log entries." sql=" @@ -126,7 +129,7 @@ create () sessionStatementId int ) ENGINE=MyISAM ; create index start_idx on start (sessionid, sessionStatementId); - load data infile '/tmp/idbtmp.tbl' into table start fields terminated by '|'; + load data infile '${tmpDir}/idbtmp.tbl' into table start fields terminated by '|'; " $MYSQLCMD -e "$sql" @@ -148,7 +151,7 @@ create () prevSession=$3; } print $0 "|" val "|" - }' | sort -t '|' -n -k 1 > /tmp/idbtmp.tbl + }' | sort -t '|' -n -k 1 > ${tmpDir}/idbtmp.tbl echo "Step 4 of 4. Populating $DB.stop table with End SQL log entries." sql=" @@ -160,7 +163,7 @@ create () sessionStatementId int ) ENGINE=MyISAM ; create index stop_idx on stop (sessionid, sessionStatementId); - load data infile '/tmp/idbtmp.tbl' into table stop fields terminated by '|'; + load data infile '${tmpDir}/idbtmp.tbl' into table stop fields terminated by '|'; " $MYSQLCMD $DB -e "$sql;" diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index ccd74511e..0750b1abc 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -67,6 +67,7 @@ string localModule; bool rootUser = true; string HOME = "/root"; string SingleServerInstall; +string tmpDir; bool repeatStop; @@ -234,6 +235,8 @@ int main(int argc, char* argv[]) if (user != 0) rootUser = false; + + tmpDir = startup::StartUp::tmpDir(); // create/open command log file if not created @@ -2595,17 +2598,19 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { - cmd = "pdsh -a '/" + startup::StartUp::installDir() + "/bin/columnstore stop' > /tmp/cc-stop.pdsh 2>&1"; + string logFile = tmpDir + "/cc-stop.pdsh"; + + cmd = "pdsh -a '/" + startup::StartUp::installDir() + "/bin/columnstore stop' > " + logFile + " 2>&1"; system(cmd.c_str()); - if (oam.checkLogStatus("/tmp/cc-stop.pdsh", "exit") ) + if (oam.checkLogStatus(logFile, "exit") ) { - cout << endl << "ERROR: Stopping MariaDB ColumnStore Service failure, check /tmp/cc-stop.pdsh. exit..." << endl; + cout << endl << "ERROR: Stopping MariaDB ColumnStore Service failure, check " << logFile << " exit..." << endl; } } else { - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /tmp/status.log"; + cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/status.log"; system(cmd.c_str()); } } @@ -2615,7 +2620,7 @@ int processCommand(string* arguments) if ( gracefulTemp == FORCEFUL ) { - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /tmp/status.log"; + cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/status.log"; system(cmd.c_str()); cout << endl << " Successful shutdown of System (stopped local columnstore service) " << endl << endl; } @@ -2623,7 +2628,7 @@ int processCommand(string* arguments) if (Failed.find("Connection refused") != string::npos) { cout << endl << "**** shutdownSystem Error : ProcessManager not Active, stopping columnstore service" << endl; - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /tmp/status.log"; + cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/status.log"; system(cmd.c_str()); cout << endl << " Successful stop of local columnstore service " << endl << endl; } @@ -2644,12 +2649,13 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { - cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore stop' > /tmp/cc-stop.pdsh 2>&1"; + string logFile = tmpDir + "cc-stop.pdsh"; + cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore stop' > " + logFile + " 2>&1"; system(cmd.c_str()); - if (oam.checkLogStatus("/tmp/cc-stop.pdsh", "exit") ) + if (oam.checkLogStatus(logFile, "exit") ) { - cout << endl << "ERROR: Stopping MariaDB ColumnStore Service failure, check /tmp/cc-stop.pdsh. exit..." << endl; + cout << endl << "ERROR: Stopping MariaDB ColumnStore Service failure, check " + logFile + ". exit..." << endl; break; } } @@ -2750,12 +2756,13 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { - cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore restart' > /tmp/cc-restart.pdsh 2>&1"; + string logFile = tmpDir + "/cc-restart.pdsh"; + cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore restart' > " + logFile + " 2>&1"; system(cmd.c_str()); - if (oam.checkLogStatus("/tmp/cc-restart.pdsh", "exit") ) + if (oam.checkLogStatus(logFile, "exit") ) { - cout << endl << "ERROR: Restart MariaDB ColumnStore Service failure, check /tmp/cc-restart.pdsh. exit..." << endl; + cout << endl << "ERROR: Restart MariaDB ColumnStore Service failure, check " << logFile << ". exit..." << endl; break; } } @@ -2787,7 +2794,7 @@ int processCommand(string* arguments) if ( modulename == localModule ) { - cmd = startup::StartUp::installDir() + "/bin/columnstore restart > /tmp/start.log 2>&1"; + cmd = startup::StartUp::installDir() + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; int rtnCode = system(cmd.c_str()); if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0) @@ -2814,7 +2821,7 @@ int processCommand(string* arguments) cout << endl << "**** startSystem Failed" << endl; // stop local columnstore service - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /tmp/stop.log 2>&1"; + cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; system(cmd.c_str()); FAILED = true; @@ -2828,7 +2835,7 @@ int processCommand(string* arguments) cout << endl << "**** startSystem Failed" << endl; // stop local columnstore service - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /tmp/stop.log 2>&1"; + cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; system(cmd.c_str()); FAILED = true; @@ -2854,7 +2861,7 @@ int processCommand(string* arguments) //just kick off local server cout << endl << " System being started, please wait..."; cout.flush(); - cmd = startup::StartUp::installDir() + "/bin/columnstore restart > /tmp/start.log 2>&1"; + cmd = startup::StartUp::installDir() + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; int rtnCode = system(cmd.c_str()); if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0) @@ -2991,12 +2998,13 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { - cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore restart' > /tmp/cc-restart.pdsh 2>&1"; + string logFile = tmpDir + "/cc-restart.pdsh"; + cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore restart' > " + logFile + " 2>&1"; system(cmd.c_str()); - if (oam.checkLogStatus("/tmp/cc-restart.pdsh", "exit") ) + if (oam.checkLogStatus(logFile, "exit") ) { - cout << endl << "ERROR: Restart MariaDB ColumnStore Service failue, check /tmp/cc-restart.pdsh. exit..." << endl; + cout << endl << "ERROR: Restart MariaDB ColumnStore Service failue, check " << logFile << ". exit..." << endl; break; } } @@ -3044,7 +3052,7 @@ int processCommand(string* arguments) cout << endl << "**** restartSystem Failed" << endl; // stop local columnstore service - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /tmp/stop.log 2>&1"; + cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; system(cmd.c_str()); FAILED = true; @@ -3059,7 +3067,7 @@ int processCommand(string* arguments) FAILED = true; // stop local columnstore service - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /tmp/stop.log 2>&1"; + cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; system(cmd.c_str()); break; @@ -3075,7 +3083,7 @@ int processCommand(string* arguments) break; //RESTART LOCAL HOST - cmd = startup::StartUp::installDir() + "/bin/columnstore restart > /tmp/start.log 2>&1"; + cmd = startup::StartUp::installDir() + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; int rtnCode = system(cmd.c_str()); if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0) @@ -3095,7 +3103,7 @@ int processCommand(string* arguments) //just kick off local server cout << " System being restarted, please wait..."; cout.flush(); - cmd = startup::StartUp::installDir() + "/bin/columnstore restart > /tmp/start.log 2>&1"; + cmd = startup::StartUp::installDir() + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) @@ -4921,7 +4929,7 @@ int processCommand(string* arguments) cout << "System Installation and Temporary Logging Directories" << endl << endl; cout << "System Installation Directory = " << startup::StartUp::installDir() << endl; - cout << "System Temporary Logging Directory = " << startup::StartUp::tmpDir() << endl; + cout << "System Temporary Logging Directory = " << tmpDir << endl; } break; @@ -5563,17 +5571,26 @@ int processCommand(string* arguments) if ( rootUser) { - int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + string logFile = tmpDir + "/columnstore.log"; + string cmd = "rpm -qi mariadb-columnstore-platform > " + logFile + " 2>&1"; + int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) == 0) - system("cat /tmp/columnstore.txt"); + { + string cmd = "cat " + logFile; + system(cmd.c_str()); + } else { - rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + string cmd = "dpkg -s mariadb-columnstore-platform > " + logFile + " 2>&1"; + rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) == 0) - system("cat /tmp/columnstore.txt"); - else + { + string cmd = "cat " + logFile; + system(cmd.c_str()); + } + else { cout << "SoftwareVersion = " << columnstore_version << endl; cout << "SoftwareRelease = " << columnstore_release << endl; @@ -8676,11 +8693,11 @@ int ProcessSupportCommand(int CommandID, std::string arguments[]) catch (exception& e) { cout << endl << "**** checkDBFunctional Failed : " << e.what() << endl; - cout << endl << " can check UM /tmp/dbfunctional.log for possible additional information" << endl << endl; + cout << endl << " can check UM " + tmpDir + "/dbfunctional.log for possible additional information" << endl << endl; } catch (...) { - cout << endl << " checkDBFunctional Failed: check UM /tmp/dbfunctional.log" << endl << endl; + cout << endl << " checkDBFunctional Failed: check UM " + tmpDir + "/dbfunctional.log" << endl << endl; } } break; From b7b98a3e1a21daef192352b07634aaddc14eca5e Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 25 Sep 2018 11:32:56 -0500 Subject: [PATCH 026/127] MCOL-520 --- dbcon/execplan/sessionmanager.h | 2 +- dbcon/mysql/ha_calpont_impl.cpp | 10 +- oam/etc/Columnstore.xml | 10 +- oam/install_scripts/pre-uninstall | 2 +- tools/dbbuilder/dbbuilder.cpp | 30 ++- tools/evalidx/checkidx.py | 2 +- .../clusterTester/columnstoreClusterTester.sh | 218 +++++++++--------- utils/configcpp/configcpp.cpp | 4 +- utils/configcpp/writeonce.cpp | 8 +- utils/idbdatafile/IDBLogger.cpp | 8 +- utils/joiner/joinpartition.cpp | 9 +- versioning/BRM/sessionmanagerserver.h | 2 +- versioning/BRM/slavecomm.cpp | 14 +- writeengine/bulk/bulkload.py | 8 +- writeengine/bulk/checkidx.py | 6 +- writeengine/bulk/qa-bulkload.py | 8 +- writeengine/server/we_dataloader.cpp | 18 -- writeengine/shared/we_config.cpp | 2 +- writeengine/splitter/we_sdhandler.cpp | 8 +- writeengine/splitter/we_splitterapp.cpp | 2 +- 20 files changed, 185 insertions(+), 186 deletions(-) diff --git a/dbcon/execplan/sessionmanager.h b/dbcon/execplan/sessionmanager.h index 97f0aa01b..f1f9ad71e 100644 --- a/dbcon/execplan/sessionmanager.h +++ b/dbcon/execplan/sessionmanager.h @@ -55,7 +55,7 @@ namespace execplan * default value is 1000. * SessionManager/SharedMemoryTmpFile: the file to store the shared memory segment * data in between invocations if DESTROYSHMSEG is defined below. The - * default is /tmp/CalpontShm. + * default is /tmp/ColumnstoreShm. * SessionManager/TxnIDFile: the file to store the last transaction ID issued */ diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 792114c9b..42eb9adb7 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -2720,16 +2720,16 @@ int ha_calpont_impl_rnd_init(TABLE* table) #ifdef DEBUG_SETENV string home(getenv("HOME")); - if (!getenv("CALPONT_HOME")) + if (!getenv("COLUMNSTORE_HOME")) { string calpontHome(home + "/Calpont/etc/"); - setenv("CALPONT_HOME", calpontHome.c_str(), 1); + setenv("COLUMNSTORE_HOME", calpontHome.c_str(), 1); } - if (!getenv("CALPONT_CONFIG_FILE")) + if (!getenv("COLUMNSTORE_CONFIG_FILE")) { - string calpontConfigFile(home + "/Calpont/etc/Columnstore.xml"); - setenv("CALPONT_CONFIG_FILE", calpontConfigFile.c_str(), 1); + string calpontConfigFile(home + "/mariadb/columnstore/etc/Columnstore.xml"); + setenv("COLUMNSTORE_CONFIG_FILE", calpontConfigFile.c_str(), 1); } if (!getenv("CALPONT_CSC_IDENT")) diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 364b46c08..3d4f4383e 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -249,9 +249,9 @@ 90 80 70 - 10 0.0.0.0 @@ -269,7 +269,7 @@ --> /rdwrscratch /columnstore_tmp_files - /tmp/columnstore_tmp_files + ${tmpDir}/columnstore_tmp_files dm @@ -495,7 +495,7 @@ - $INSTALLDIR/tmp + $INSTALLDIR${tmpDir} Y diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 572f70420..9b8dd2782 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -79,7 +79,7 @@ tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` rm -rf $tmpDir rm -f $installdir/local/*.columnstore rm -rf $installdir/local/etc/ -rm -f $installdir/data/bulk/tmpjob/* >/dev/null 2>&1 +rm -f $installdir/data/bulk${tmpDir}job/* >/dev/null 2>&1 rm -f $installdir/local/moveDbrootTransactionLog lockdir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation LockFileDirectory` diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index 2dc73082d..aec094b89 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -43,6 +43,9 @@ using namespace execplan; #include "installdir.h" +string tmpDir; +string logFile; + enum BUILD_OPTION { SYSCATALOG_ONLY = 7, //Create systables only @@ -54,8 +57,10 @@ namespace int setUp() { #ifndef _MSC_VER - (void)system("/bin/rm -f /tmp/dbbuilder.status >/dev/null 2>&1"); - (void)system("/bin/touch /tmp/dbbuilder.status >/dev/null 2>&1"); + string cmd = "/bin/rm -f " + logFile + " >/dev/null 2>&1"; + (void)system(cmd.c_str()); + cmd = "/bin/touch -f " + logFile + " >/dev/null 2>&1"; + (void)system(cmd.c_str()); #endif return 0; } @@ -69,7 +74,8 @@ int checkNotThere(WriteEngine::FID fid) void tearDown() { - unlink("/tmp/oidbitmap"); + string file = tmpDir + "/oidbitmap"; + unlink(file.c_str()); } void usage() @@ -173,6 +179,10 @@ int main(int argc, char* argv[]) return 1; } + //get tmp log directory + tmpDir = startup::StartUp::tmpDir(); + logFile = tmpDir + "/dbbuilder.status"; + buildOption = atoi(argv[optind++]); if (buildOption != 7 && !fFlg) @@ -187,7 +197,7 @@ int main(int argc, char* argv[]) bool canWrite = true; - if (access("/tmp/dbbuilder.status", W_OK) != 0) + if (access(logFile.c_str(), W_OK) != 0) canWrite = false; try @@ -196,7 +206,7 @@ int main(int argc, char* argv[]) { string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + - "' > /tmp/dbbuilder.status"; + "' > " + logFile; if (canWrite) (void)system(cmd.c_str()); @@ -230,7 +240,7 @@ int main(int argc, char* argv[]) } catch (const exception& ex) { - string cmd(string("echo 'FAILED: ") + ex.what() + "' > /tmp/dbbuilder.status"); + string cmd(string("echo 'FAILED: ") + ex.what() + "' > " + logFile); if (canWrite) (void)system(cmd.c_str()); @@ -242,7 +252,7 @@ int main(int argc, char* argv[]) } catch (...) { - string cmd = "echo 'FAILED: HDFS checking.' > /tmp/dbbuilder.status"; + string cmd = "echo 'FAILED: HDFS checking.' > " + logFile; if (canWrite) (void)system(cmd.c_str()); @@ -261,7 +271,7 @@ int main(int argc, char* argv[]) SystemCatalog sysCatalog; sysCatalog.build(); - std::string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > /tmp/dbbuilder.status"; + std::string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; if (canWrite) (void)system(cmd.c_str()); @@ -296,7 +306,7 @@ int main(int argc, char* argv[]) } catch (exception& ex) { - string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > /tmp/dbbuilder.status"; + string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; if (canWrite) (void)system(cmd.c_str()); @@ -307,7 +317,7 @@ int main(int argc, char* argv[]) } catch (...) { - string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > /tmp/dbbuilder.status"; + string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; if (canWrite) (void)system(cmd.c_str()); diff --git a/tools/evalidx/checkidx.py b/tools/evalidx/checkidx.py index 2ca213520..c58b68dd3 100755 --- a/tools/evalidx/checkidx.py +++ b/tools/evalidx/checkidx.py @@ -6,7 +6,7 @@ def find_paths(): """Find DBRoot and BulkRoot.""" try: - config_file = os.environ['CALPONT_CONFIG_FILE'] + config_file = os.environ['COLUMNSTORE_CONFIG_FILE'] except KeyError: try: config_file = '/usr/local/mariadb/columnstore/etc' diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 2534fdd19..b426b8584 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -18,6 +18,9 @@ if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore fi +#get temp directory +tmpDir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir` + checkContinue() { if [ "$CHECK" = false ]; then @@ -69,7 +72,6 @@ helpPrint () { echo " Firewall settings" echo " Date/time settings" echo " Dependent packages installed" - echo " For non-root user install - test permissions on /tmp and /dev/shm" echo "" echo "Usage: $0 [options]" echo "OPTIONS:" @@ -203,23 +205,23 @@ checkLocalOS() echo "" #get local OS - `$COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh > /tmp/os_detect 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh > ${tmpDir}/os_detect 2>&1` if [ "$?" -eq 0 ]; then - localOS=`cat /tmp/os_detect | grep "Operating System name" | cut -f2 -d '"'` + localOS=`cat ${tmpDir}/os_detect | grep "Operating System name" | cut -f2 -d '"'` echo "Local Node OS System Name : $localOS" if [ "$OS" != "" ] ; then echo "" echo "Local Node OS Versions doesn't match the command line OS argument" echo "Contining using the Detected Local Node OS Version" - OS=`cat /tmp/os_detect | grep "Operating System tag" | cut -f4 -d " "` + OS=`cat ${tmpDir}/os_detect | grep "Operating System tag" | cut -f4 -d " "` echo "Local Node OS Version : $OS" else - OS=`cat /tmp/os_detect | grep "Operating System tag" | cut -f4 -d " "` + OS=`cat ${tmpDir}/os_detect | grep "Operating System tag" | cut -f4 -d " "` fi else - localOS=`cat /tmp/os_detect | grep "Operating System name" | cut -f2 -d '"'` + localOS=`cat ${tmpDir}/os_detect | grep "Operating System name" | cut -f2 -d '"'` echo "Local Node OS System Name : $localOS" if [ "$OS" == "" ] ; then @@ -269,16 +271,16 @@ checkLocalDir() echo "" #remove any check tmp files from previous runs - `rm -f /tmp/*_check > /dev/null 2>&1` + `rm -f ${tmpDir}/*_check > /dev/null 2>&1` - #check /tmp and /dev/shm + #check ${tmpDir} and /dev/shm pass=true - `touch /tmp/cs_check > /dev/null 2>&1` + `touch ${tmpDir}/cs_check > /dev/null 2>&1` if [ "$?" -eq 0 ]; then - echo "Local Node permission test on /tmp : Passed" - `rm -f /tmp/cs_check` + echo "Local Node permission test on ${tmpDir} : Passed" + `rm -f ${tmpDir}/cs_check` else - echo "Local Node permission test on /tmp : ${bold}Failed${normal}, change permissions to 777 and re-test" + echo "Local Node permission test on ${tmpDir} : ${bold}Failed${normal}, change permissions to 777 and re-test" exit 1 fi @@ -349,10 +351,10 @@ checkRemoteDir() # # remove old _check tmp files from remote servers - `rm -f /tmp/*_check > /dev/null 2>&1` + `rm -f ${tmpDir}/*_check > /dev/null 2>&1` for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f /tmp/*_check > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f ${tmpDir}/*_check > /dev/null 2>&1' 1 > ${tmpDir}/remote_command_check 2>&1` done if [ "$USER" != "root" ]; then @@ -363,26 +365,26 @@ checkRemoteDir() echo "" for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /tmp/cs_check' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch ${tmpDir}/cs_check' 1 > ${tmpDir}/remote_command_check 2>&1` rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then - `grep "Permission denied" /tmp/remote_command_check > /dev/null 2>&1` + `grep "Permission denied" ${tmpDir}/remote_command_check > /dev/null 2>&1` if [ "$?" -eq 0 ]; then - echo "$ipadd Node permission test on /tmp : ${bold}Failed${normal}, change permissions to 777 and re-test" + echo "$ipadd Node permission test on ${tmpDir} : ${bold}Failed${normal}, change permissions to 777 and re-test" exit 1 else - echo "$ipadd Node permission test on /tmp : Passed" + echo "$ipadd Node permission test on ${tmpDir} : Passed" fi else - echo "Error running remote_command.sh to $ipadd Node, check /tmp/remote_command_check" + echo "Error running remote_command.sh to $ipadd Node, check ${tmpDir}/remote_command_check" pass=false REPORTPASS=false fi - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > ${tmpDir}/remote_command_check 2>&1` rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then - `grep "Permission denied" /tmp/remote_command_check > /dev/null 2>&1` + `grep "Permission denied" ${tmpDir}/remote_command_check > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "$ipadd Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test" pass=false @@ -391,7 +393,7 @@ checkRemoteDir() echo "$ipadd Node permission test on /dev/shm : Passed" fi else - echo "Error running remote_command.sh to $ipadd Node, check /tmp/remote_command_check" + echo "Error running remote_command.sh to $ipadd Node, check ${tmpDir}/remote_command_check" pass=false REPORTPASS=false fi @@ -415,20 +417,20 @@ checkOS() echo "" pass=true - `/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh /tmp/.` + `/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh ${tmpDir}/.` for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD /tmp/os_detect.sh 1 > /tmp/remote_scp_put_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD ${tmpDir}/os_detect.sh 1 > ${tmpDir}/remote_scp_put_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_put.sh to $ipadd Node, check /tmp/remote_scp_put_check" + echo "Error running remote_scp_put.sh to $ipadd Node, check ${tmpDir}/remote_scp_put_check" exit 1 else - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD /tmp/os_detect.sh 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD ${tmpDir}/os_detect.sh 1 > ${tmpDir}/remote_command_check` rc="$?" if [ "$?" -ne 0 ]; then - echo "Error running remote_command.sh /tmp/os_detect.sh on $ipadd Node, check /tmp/remote_command_check" + echo "Error running remote_command.sh ${tmpDir}/os_detect.sh on $ipadd Node, check ${tmpDir}/remote_command_check" exit 1 else - remoteOS=`cat /tmp/remote_command_check | grep "Operating System name" | cut -f2 -d '"'` + remoteOS=`cat ${tmpDir}/remote_command_check | grep "Operating System name" | cut -f2 -d '"'` echo "$ipadd Node OS Version : $remoteOS" if [ $localOS != $remoteOS ]; then echo "${bold}Failed${normal}, $ipadd has a different OS than local node" @@ -453,25 +455,25 @@ checkLocale() echo "" #get local Locale - `locale | grep LANG= > /tmp/locale_check 2>&1` + `locale | grep LANG= > ${tmpDir}/locale_check 2>&1` if [ "$?" -eq 0 ]; then - echo "Local Node Locale : `cat /tmp/locale_check`" + echo "Local Node Locale : `cat ${tmpDir}/locale_check`" else echo "Error running 'locale' command on local node" fi pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > /tmp/locale_check 2>&1' 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > ${tmpDir}/locale_check 2>&1' 1 > ${tmpDir}/remote_command_check` rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/locale_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/locale_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" exit 1 else echo "$ipadd Node Locale : `cat locale_check`" - `diff /tmp/locale_check locale_check > /dev/null 2>&1` + `diff ${tmpDir}/locale_check locale_check > /dev/null 2>&1` if [ "$?" -ne 0 ]; then echo "${bold}Failed${normal}, $ipadd has a different Locale setting than local node" pass=false @@ -480,7 +482,7 @@ checkLocale() `rm -f locale_check` fi else - echo "Error running remote_command.sh to $ipadd Node, check /tmp/remote_command_check" + echo "Error running remote_command.sh to $ipadd Node, check ${tmpDir}/remote_command_check" exit 1 pass=false REPORTPASS=false @@ -538,7 +540,7 @@ checkLocalSELINUX() pass=true #check local SELINUX if [ -f /etc/selinux/config ]; then - `cat /etc/selinux/config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` + `cat /etc/selinux/config | grep SELINUX | grep enforcing > ${tmpDir}/selinux_check 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Warning${normal}, Local Node SELINUX setting is Enabled, check port test results" pass=false @@ -565,9 +567,9 @@ checkUMASK() pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > ${tmpDir}/remote_command_check 2>&1` if [ "$?" -eq 0 ]; then - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> /tmp/remote_scp_get 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> ${tmpDir}/remote_scp_get 2>&1` if [ "$?" -eq 0 ]; then permission=`cat test.log` result=${permission:4:1} @@ -584,11 +586,11 @@ checkUMASK() pass=false fi else - echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check /tmp/remote_scp_get" + echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check ${tmpDir}/remote_scp_get" pass=false fi else - echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check /tmp/remote_command_check" + echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check ${tmpDir}/remote_command_check" pass=false fi `rm -f test.log` @@ -611,11 +613,11 @@ checkSELINUX() pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "$ipadd Node SELINUX setting is Not Enabled" else - `cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` + `cat config | grep SELINUX | grep enforcing > ${tmpDir}/selinux_check 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results" pass=false @@ -644,12 +646,12 @@ checkFirewalls() #check local FIREWALLS for firewall in "${FIREWALL_LIST[@]}"; do pass=true - `service $firewall status > /tmp/firewall1_check 2>&1` + `service $firewall status > ${tmpDir}/firewall1_check 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results" pass=false else - `systemctl status $firewall > /tmp/firewall1_check 2>&1` + `systemctl status $firewall > ${tmpDir}/firewall1_check 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results" pass=false @@ -667,12 +669,12 @@ checkFirewalls() # 'sysconfig not on remote node for firewall in "${FIREWALL_LIST[@]}"; do pass=true - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > ${tmpDir}/firewall_check 2>&1" 1 > ${tmpDir}/remote_command_check` if [ "$?" -eq 0 ]; then echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results" pass=false else - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > ${tmpDir}/firewall_check 2>&1" 1 > ${tmpDir}/remote_command_check` if [ "$?" -eq 0 ]; then echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results" pass=false @@ -696,7 +698,7 @@ checkFirewalls() pass=true #check local IPTABLES - `/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1` + `/sbin/rcSuSEfirewall2 status > ${tmpDir}/rcSuSEfirewall2_check 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, Local Node rcSuSEfirewall2 service is Enabled, check port test results" pass=false @@ -705,7 +707,7 @@ checkFirewalls() fi for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > ${tmpDir}/rcSuSEfirewall2_check 2>&1' 1 > ${tmpDir}/remote_command_check` rc="$?" if [ $rc -eq 0 ] ; then echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, check port test results" @@ -728,12 +730,12 @@ checkPorts() pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > /tmp/port_test` + `nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > ${tmpDir}/port_test` if [ "$?" -ne 0 ]; then echo $ipadd " Node Passed port test" else echo $ipadd " Node ${bold}Failed${normal} port test, check and disable any firewalls or open ports in firewall" - cat /tmp/port_test + cat ${tmpDir}/port_test pass=false REPORTPASS=false fi @@ -756,16 +758,16 @@ checkTime() #get local epoch time localTime=`date +%s` for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'date +%s > /tmp/time_check' > /tmp/time_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'date +%s > ${tmpDir}/time_check' > ${tmpDir}/time_check` rc="$?" if [ $rc -ne 0 ] ; then - echo $ipadd " Node ${bold}Failed${normal} date/time check failed, check /tmp/time_check" + echo $ipadd " Node ${bold}Failed${normal} date/time check failed, check ${tmpDir}/time_check" pass=false REPORTPASS=false else - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/time_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/time_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" else remoteTime=`cat time_check` timeDiff=`echo "$(($remoteTime-$localTime))"` @@ -803,17 +805,17 @@ checkMysqlPassword() fi if [ "$IPADDRESSES" != "" ]; then - `/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/mariadb-command-line.sh /tmp/.` + `/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/mariadb-command-line.sh ${tmpDir}/.` for ipadd in "${NODE_IPADDRESS[@]}"; do `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD $COLUMNSTORE_INSTALL_DIR/mysql/mysql-Columnstore start > /dev/null 2>&1` - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD /tmp/mariadb-command-line.sh 1 > /tmp/remote_scp_put_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD ${tmpDir}/mariadb-command-line.sh 1 > ${tmpDir}/remote_scp_put_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_put.sh to $ipadd Node, check /tmp/remote_scp_put_check" + echo "Error running remote_scp_put.sh to $ipadd Node, check ${tmpDir}/remote_scp_put_check" exit 1 else - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD /tmp/mariadb-command-line.sh 1 > /tmp/remote_command_check` - `cat /tmp/remote_command_check | grep "ERROR - PASSWORD" > /dev/null 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD ${tmpDir}/mariadb-command-line.sh 1 > ${tmpDir}/remote_command_check` + `cat ${tmpDir}/remote_command_check | grep "ERROR - PASSWORD" > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node MariaDB login failed with missing password file, /root/.my.cnf" pass=false @@ -860,8 +862,8 @@ checkPackages() REPORTPASS=false fi else - `yum list installed "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep Installed > /dev/null 2>&1` + `yum list installed "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep Installed > /dev/null 2>&1` if [ "$?" -ne 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install" pass=false @@ -880,8 +882,8 @@ checkPackages() #check for package that shouldnt be installed pass=true for PKG in "${CENTOS_PKG_NOT[@]}"; do - `yum list installed "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep Installed > /dev/null 2>&1` + `yum list installed "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep Installed > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" pass=false @@ -901,14 +903,14 @@ checkPackages() for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${CENTOS_PKG[@]}"; do if [ $OS == "centos6" ] && [ $PKG == "boost" ]; then - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'ls /usr/lib/libboost_regex.so > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'ls /usr/lib/libboost_regex.so > /dev/null 2>&1' 1 > ${tmpDir}/remote_command_check 2>&1` if [ $? -ne 0 ] ; then echo "${bold}Failed${normal}, $ipadd Node ${bold}boost libraries${normal} not installed" pass=false REPORTPASS=false fi else - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` rc="$?" if [ $rc -eq 2 ] ; then echo "${bold}Failed${normal}, $ipadd Node, 'yum' not installed" @@ -933,7 +935,7 @@ checkPackages() #check for package that shouldnt be installed for PKG in "${CENTOS_PKG_NOT[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` rc="$?" if [ $rc -eq 2 ] ; then echo "${bold}Failed${normal}, $ipadd Node, 'yum' not installed" @@ -972,8 +974,8 @@ checkPackages() pass=true #check centos packages on local node for PKG in "${SUSE_PKG[@]}"; do - `rpm -qi "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep "not installed" > /dev/null 2>&1` + `rpm -qi "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep "not installed" > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install" pass=false @@ -990,8 +992,8 @@ checkPackages() #check for package that shouldnt be installed pass=true for PKG in "${SUSE_PKG_NOT[@]}"; do - `rpm -qi "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep "not installed" > /dev/null 2>&1` + `rpm -qi "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep "not installed" > /dev/null 2>&1` if [ "$?" -ne 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" pass=false @@ -1011,7 +1013,7 @@ checkPackages() if [ "$IPADDRESSES" != "" ]; then for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${SUSE_PKG[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` rc="$?" if [ $rc -ne 0 ] ; then echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is not installed, please install" @@ -1030,7 +1032,7 @@ checkPackages() #check for package that shouldnt be installed for PKG in "${SUSE_PKG_NOT[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` rc="$?" if [ $rc -eq 0 ] ; then echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" @@ -1063,8 +1065,8 @@ checkPackages() pass=true #check centos packages on local node for PKG in "${UBUNTU_PKG[@]}"; do - `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + `dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1` if [ "$?" -ne 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install" pass=false @@ -1081,8 +1083,8 @@ checkPackages() #check for package that shouldnt be installed pass=true for PKG in "${UBUNTU_PKG_NOT[@]}"; do - `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + `dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" pass=false @@ -1102,12 +1104,12 @@ checkPackages() if [ "$IPADDRESSES" != "" ]; then for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${UBUNTU_PKG[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" else - `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + `cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" pass=false @@ -1134,12 +1136,12 @@ checkPackages() #check for package that shouldnt be installed for PKG in "${UBUNTU_PKG_NOT[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" else - `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + `cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" pass=false @@ -1181,8 +1183,8 @@ checkPackages() pass=true #check centos packages on local node for PKG in "${DEBIAN_PKG[@]}"; do - `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + `dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1` if [ "$?" -ne 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install" pass=false @@ -1199,8 +1201,8 @@ checkPackages() #check for package that shouldnt be installed pass=true for PKG in "${DEBIAN_PKG_NOT[@]}"; do - `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + `dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" pass=false @@ -1220,12 +1222,12 @@ checkPackages() if [ "$IPADDRESSES" != "" ]; then for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${DEBIAN_PKG[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" else - `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + `cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" pass=false @@ -1252,12 +1254,12 @@ checkPackages() #check for package that shouldnt be installed for PKG in "${DEBIAN_PKG_NOT[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" else - `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + `cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" pass=false @@ -1299,8 +1301,8 @@ checkPackages() pass=true #check centos packages on local node for PKG in "${DEBIAN9_PKG[@]}"; do - `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + `dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1` if [ "$?" -ne 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install" pass=false @@ -1317,8 +1319,8 @@ checkPackages() #check for package that shouldnt be installed pass=true for PKG in "${DEBIAN9_PKG_NOT[@]}"; do - `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` - `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + `dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1` + `cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" pass=false @@ -1339,12 +1341,12 @@ checkPackages() if [ "$IPADDRESSES" != "" ]; then for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${DEBIAN9_PKG[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" else - `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + `cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" pass=false @@ -1371,12 +1373,12 @@ checkPackages() #check for package that shouldnt be installed for PKG in "${DEBIAN9_PKG_NOT[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" else - `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + `cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" pass=false diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index 3236cc5d2..459a6242b 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -97,7 +97,7 @@ Config* Config::makeConfig(const char* cf) cf = cfStr.c_str(); #else - cf = getenv("CALPONT_CONFIG_FILE"); + cf = getenv("COLUMNSTORE_CONFIG_FILE"); #endif if (cf == 0 || *cf == 0) @@ -419,7 +419,7 @@ void Config::writeConfig(const string& configFile) const value = c1->getConfig("SystemConfig", "SystemName"); //good read, save copy, copy temp file tp tmp then to Columnstore.xml - //move to /tmp to get around a 'same file error' in mv command + //move to get around a 'same file error' in mv command try { if (exists(scft)) fs::remove(scft); diff --git a/utils/configcpp/writeonce.cpp b/utils/configcpp/writeonce.cpp index c4c22623f..01ed162bd 100644 --- a/utils/configcpp/writeonce.cpp +++ b/utils/configcpp/writeonce.cpp @@ -50,12 +50,16 @@ namespace config void WriteOnceConfig::initializeDefaults() { + string tmpDir = startup::StartUp::tmpDir(); + fLBID_Shift = make_pair("13", false); fDBRootCount = make_pair("1", false); fDBRMRoot = make_pair("/mnt/OAM/dbrm/BRM_saves", false); - fSharedMemoryTmpFile1 = make_pair("/tmp/CalpontShm,", false); + string file = tmpDir + "/ColumnstoreShm"; + fSharedMemoryTmpFile1 = make_pair(file, false); fTxnIDFile = make_pair("/mnt/OAM/dbrm/SMTxnID", false); - fSharedMemoryTmpFile2 = make_pair("/tmp/CalpontSessionMonitorShm", false); + file = tmpDir + "/CalpontSessionMonitorShm"; + fSharedMemoryTmpFile2 = make_pair(file, false); } void WriteOnceConfig::setup() diff --git a/utils/idbdatafile/IDBLogger.cpp b/utils/idbdatafile/IDBLogger.cpp index 81d1cfd48..11ce65029 100644 --- a/utils/idbdatafile/IDBLogger.cpp +++ b/utils/idbdatafile/IDBLogger.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2018 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -28,6 +29,8 @@ #include "IDBLogger.h" +#include "installdir.h" + #include #include #include @@ -55,7 +58,10 @@ void IDBLogger::writeLog( const std::string& logmsg ) pthread_t threadid = pthread_self(); #endif ostringstream fname; - fname << "/tmp/idbdf-log-" << pid << "-" << threadid << ".csv"; + + string tmpDir = startup::StartUp::tmpDir(); + + fname << tmpDir + "/idbdf-log-" << pid << "-" << threadid << ".csv"; ofstream output; output.open( fname.str().c_str(), ios::out | ios::app ); diff --git a/utils/joiner/joinpartition.cpp b/utils/joiner/joinpartition.cpp index 688489730..962143aac 100644 --- a/utils/joiner/joinpartition.cpp +++ b/utils/joiner/joinpartition.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2018 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -22,6 +23,7 @@ #include "joinpartition.h" #include "tuplejoiner.h" #include "atomicops.h" +#include "installdir.h" using namespace std; using namespace utils; @@ -128,12 +130,9 @@ JoinPartition::JoinPartition(const JoinPartition& jp, bool splitMode) : // Instead, each will double in size, giving a capacity of 8GB -> 16 -> 32, and so on. // bucketCount = jp.bucketCount; bucketCount = 2; - filenamePrefix = config->getConfig("HashJoin", "TempFilePath"); + filenamePrefix = startup::StartUp::tmpDir(); - if (filenamePrefix.empty()) - filenamePrefix = "/tmp/infinidb"; - - filenamePrefix += "/Infinidb-join-data-"; + filenamePrefix += "/Columnstore-join-data-"; uniqueID = atomicops::atomicInc(&uniqueNums); uint32_t tmp = uniqueID; diff --git a/versioning/BRM/sessionmanagerserver.h b/versioning/BRM/sessionmanagerserver.h index c7319fc0d..b25de41f9 100644 --- a/versioning/BRM/sessionmanagerserver.h +++ b/versioning/BRM/sessionmanagerserver.h @@ -73,7 +73,7 @@ namespace BRM * default value is 1000. * SessionManager/SharedMemoryTmpFile: the file to store the shared memory segment * data in between invocations if DESTROYSHMSEG is defined below. The - * default is /tmp/CalpontShm. + * default is /tmp/ColumnstoreShm. * SessionManager/TxnIDFile: the file to store the last transaction ID issued */ diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index cc708781a..fab93d605 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -43,6 +43,8 @@ #include "slavecomm.h" #undef SLAVECOMM_DLLEXPORT +#include "installdir.h" + using namespace std; using namespace messageqcpp; using namespace idbdatafile; @@ -98,6 +100,8 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) : } } } + + string tmpDir = startup::StartUp::tmpDir(); /* NOTE: this string has to match whatever is designated as the first slave */ if (hostname == "DBRM_Worker1") @@ -108,11 +112,11 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) : } catch (exception& e) { - savefile = "/tmp/BRM_SaveFiles"; + savefile = tmpDir + "/BRM_SaveFiles"; } if (savefile == "") - savefile = "/tmp/BRM_SaveFiles"; + savefile = tmpDir + "/BRM_SaveFiles"; tmp = ""; @@ -181,17 +185,19 @@ SlaveComm::SlaveComm() { config::Config* config = config::Config::makeConfig(); + string tmpDir = startup::StartUp::tmpDir(); + try { savefile = config->getConfig("SystemConfig", "DBRMRoot"); } catch (exception& e) { - savefile = "/tmp/BRM_SaveFiles"; + savefile = tmpDir + "/BRM_SaveFiles"; } if (savefile == "") - savefile = "/tmp/BRM_SaveFiles"; + savefile = tmpDir + "/BRM_SaveFiles"; journalName = savefile + "_journal"; diff --git a/writeengine/bulk/bulkload.py b/writeengine/bulk/bulkload.py index e4d8c0366..9a6ab0e06 100644 --- a/writeengine/bulk/bulkload.py +++ b/writeengine/bulk/bulkload.py @@ -52,10 +52,10 @@ def find_paths(): """Find DBRoot and BulkRoot.""" try: - config_file = os.environ['CALPONT_CONFIG_FILE'] + config_file = os.environ['COLUMNSTORE_CONFIG_FILE'] except KeyError: try: - logger.info("Environment variable CALPONT_CONFIG_FILE not set, looking for system Columnstore.xml") + logger.info("Environment variable COLUMNSTORE_CONFIG_FILE not set, looking for system Columnstore.xml") config_file = '/usr/local/mariadb/columnstore/etc/Columnstore.xml' os.lstat(config_file) except: @@ -185,10 +185,6 @@ def main(): clean up old files, sort the index inserts and generally rock and roll """ start_dir = curdir=os.getcwd() # remember where we started - - if not os.access('.', os.W_OK): - os.chdir('/tmp') - logger.warn('Changing to /tmp to have permission to write files') if not os.environ.has_key('LD_LIBRARY_PATH'): logger.info('No environment variable LD_LIBRARY_PATH') diff --git a/writeengine/bulk/checkidx.py b/writeengine/bulk/checkidx.py index 0fe0a4ed9..be22a674b 100755 --- a/writeengine/bulk/checkidx.py +++ b/writeengine/bulk/checkidx.py @@ -6,7 +6,7 @@ def find_paths(): """Find DBRoot and BulkRoot.""" try: - config_file = os.environ['CALPONT_CONFIG_FILE'] + config_file = os.environ['COLUMNSTORE_CONFIG_FILE'] except KeyError: try: config_file = '/usr/local/mariadb/columnstore/etc/Columnstore.xml' @@ -71,10 +71,6 @@ def main(): """ Validate indexes.. """ - - if not os.access('.', os.W_OK): - os.chdir('/tmp') - print 'Changing to /tmp to have permission to write files' if len(os.getenv('LD_LIBRARY_PATH'))<5: print 'Suspicous LD_LIBRARY_PATH: %s'%os.getenv('LD_LIBRARY_PATH') diff --git a/writeengine/bulk/qa-bulkload.py b/writeengine/bulk/qa-bulkload.py index 599183436..ea5436fd8 100644 --- a/writeengine/bulk/qa-bulkload.py +++ b/writeengine/bulk/qa-bulkload.py @@ -49,10 +49,10 @@ def find_paths(): """Find DBRoot and BulkRoot.""" try: - config_file = os.environ['CALPONT_CONFIG_FILE'] + config_file = os.environ['COLUMNSTORE_CONFIG_FILE'] except KeyError: try: - logger.info("Environment variable CALPONT_CONFIG_FILE not set, looking for system Columnstore.xml") + logger.info("Environment variable COLUMNSTORE_CONFIG_FILE not set, looking for system Columnstore.xml") config_file = '/usr/local/mariadb/columnstore/etc/Columnstore.xml' os.lstat(config_file) except: @@ -182,10 +182,6 @@ def main(): clean up old files, sort the index inserts and generally rock and roll """ start_dir = curdir=os.getcwd() # remember where we started - - if not os.access('.', os.W_OK): - os.chdir('/tmp') - logger.warn('Changing to /tmp to have permission to write files') if not os.environ.has_key('LD_LIBRARY_PATH'): logger.info('No environment variable LD_LIBRARY_PATH') diff --git a/writeengine/server/we_dataloader.cpp b/writeengine/server/we_dataloader.cpp index b3845ad9f..d1d735a3b 100644 --- a/writeengine/server/we_dataloader.cpp +++ b/writeengine/server/we_dataloader.cpp @@ -1331,15 +1331,6 @@ void WEDataLoader::onReceiveBrmRptFileName(ByteStream& Ibs) cout << "Creating directory : " << dirname << endl; boost::filesystem::create_directories(dirname.c_str()); } - - /* - #ifdef _MSC_VER - mkdir(dirname.c_str()); - #else - mkdir(dirname.c_str(), 0777); - boost::filesystem::create_directories("/tmp/boby/test"); - #endif - */ } if (fpSysLog) @@ -1568,15 +1559,6 @@ void WEDataLoader::onReceiveJobId(ByteStream& Ibs) cout << "Creating directory : " << dirname << endl; boost::filesystem::create_directories(dirname.c_str()); } - - /* - #ifdef _MSC_VER - mkdir(dirname.c_str()); - #else - mkdir(dirname.c_str(), 0777); - boost::filesystem::create_directories("/tmp/boby/test"); - #endif - */ } fJobFile.open(aJobFileName.c_str()); diff --git a/writeengine/shared/we_config.cpp b/writeengine/shared/we_config.cpp index 63393cd9c..070e5d918 100644 --- a/writeengine/shared/we_config.cpp +++ b/writeengine/shared/we_config.cpp @@ -270,7 +270,7 @@ void Config::checkReload( ) if ( hdfsRdwrScratch.length() == 0 ) { - hdfsRdwrScratch = "/tmp/hdfsscratch"; + hdfsRdwrScratch = TmpFileDir + "/hdfsscratch"; } IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 61fe45239..a1db530ae 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -72,6 +72,8 @@ using namespace batchloader; #include "we_tablelockgrabber.h" #include "we_simplesyslog.h" +#include "installdir.h" + namespace WriteEngine { //------------------------------------------------------------------------------ @@ -1402,7 +1404,7 @@ void WESDHandler::onBrmReport(int PmId, messageqcpp::SBS& Sbs) if (!fRef.fCmdArgs.getConsoleOutput()) { ostringstream oss; - oss << "/tmp/" << fTableOId << ".txt"; + oss << startup::StartUp::tmpDir() << fTableOId << ".txt"; ofstream dmlFile(oss.str().c_str(), std::ofstream::app); if (dmlFile.is_open()) @@ -1883,10 +1885,10 @@ void WESDHandler::onCleanupResult(int PmId, messageqcpp::SBS& Sbs) WEColOorVec::iterator aIt = fImportRslt.fColOorVec.begin(); ofstream dmlFile; - if (!fRef.fCmdArgs.getConsoleOutput()) //for DML to use file /tmp/ + if (!fRef.fCmdArgs.getConsoleOutput()) //for DML to use file { ostringstream oss; - oss << "/tmp/" << fTableOId << ".txt"; + oss << startup::StartUp::tmpDir() << fTableOId << ".txt"; dmlFile.open(oss.str().c_str()); } diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index 0804baf1b..dac0c7387 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -111,7 +111,7 @@ WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) : { ofstream dmlFile; ostringstream oss; - oss << "/tmp/" << fDh.getTableOID() << ".txt"; + oss << startup::StartUp::tmpDir() << fDh.getTableOID() << ".txt"; dmlFile.open(oss.str().c_str()); if (dmlFile.is_open()) From df77de334d95ef2b665766383c8f7f7d99e282f1 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 25 Sep 2018 13:11:20 -0500 Subject: [PATCH 027/127] MCOL-520 --- oam/install_scripts/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index d450d7f27..4f80ae0b0 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -11,7 +11,6 @@ install(PROGRAMS post-install remote_scp_get.sh columnstoreAlias run.sh - upgrade-columnstore.sh post-mysql-install post-mysqld-install binary_installer.sh From 9cfe199154aeeba559672cffa4c68728474a54d9 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 25 Sep 2018 13:35:06 -0500 Subject: [PATCH 028/127] MCOL-520 --- dbcon/execplan/sessionmonitor.h | 1 - ddlproc/ddlproc.cpp | 3 --- exemgr/main.cpp | 3 --- oamapps/postConfigure/installer.cpp | 13 +++++++++---- oamapps/resourceMonitor/hardwareMonitor.cpp | 12 +++++++++--- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/dbcon/execplan/sessionmonitor.h b/dbcon/execplan/sessionmonitor.h index af7a9fb07..42a9ec899 100644 --- a/dbcon/execplan/sessionmonitor.h +++ b/dbcon/execplan/sessionmonitor.h @@ -58,7 +58,6 @@ namespace execplan * SessionMonitor/SharedMemoryTmpFile: the file to store the shared memory segment * data in. This needs to be a different file than * file used for the SessionManager/SharedMemoryTmpFile. - * The default is /tmp/CalpontMonShm. */ /* diff --git a/ddlproc/ddlproc.cpp b/ddlproc/ddlproc.cpp index e721462fe..4fbee7843 100644 --- a/ddlproc/ddlproc.cpp +++ b/ddlproc/ddlproc.cpp @@ -79,9 +79,6 @@ void setupCwd() workdir = "."; (void)chdir(workdir.c_str()); - - if (access(".", W_OK) != 0) - (void)chdir("/tmp"); } void added_a_pm(int) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 05fdc600f..47f71140c 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1324,9 +1324,6 @@ void setupCwd(ResourceManager* rm) { string workdir = startup::StartUp::tmpDir(); (void)chdir(workdir.c_str()); - - if (access(".", W_OK) != 0) - (void)chdir("/tmp"); } void startRssMon(size_t maxPct, int pauseSeconds) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index f472ef494..3cde4f24e 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -56,6 +56,8 @@ using namespace config; #include "helpers.h" using namespace installer; +#include "installdir.h" + typedef struct Module_struct { std::string moduleName; @@ -169,6 +171,8 @@ int main(int argc, char* argv[]) if (p && *p) USER = p; + + string tmpDir = startup::StartUp::tmpDir(); // setup to start on reboot, for non-root amazon installs if ( !rootUser ) @@ -882,17 +886,18 @@ int main(int argc, char* argv[]) { cout << " DONE" << endl; - cmd = installDir + "/bin/dbbuilder 7 > /tmp/dbbuilder.log"; + string logFile = tmpDir + "/dbbuilder.log"; + cmd = installDir + "/bin/dbbuilder 7 > " + logFile; system(cmd.c_str()); - if (oam.checkLogStatus("/tmp/dbbuilder.log", "System Catalog created") ) + if (oam.checkLogStatus(logFile, "System Catalog created") ) cout << endl << "System Catalog Successfull Created" << endl; else { - if ( !oam.checkLogStatus("/tmp/dbbuilder.log", "System catalog appears to exist") ) + if ( !oam.checkLogStatus(logFile, "System catalog appears to exist") ) { cout << endl << "System Catalog Create Failure" << endl; - cout << "Check latest log file in /tmp/dbbuilder.log.*" << endl; + cout << "Check latest log file in " << logFile" << endl; cout << " IMPORTANT: Once issue has been resolved, rerun postConfigure" << endl << endl; exit (1); } diff --git a/oamapps/resourceMonitor/hardwareMonitor.cpp b/oamapps/resourceMonitor/hardwareMonitor.cpp index 6877836b4..9ec322b2e 100644 --- a/oamapps/resourceMonitor/hardwareMonitor.cpp +++ b/oamapps/resourceMonitor/hardwareMonitor.cpp @@ -34,6 +34,8 @@ #include "loggingid.h" #include "alarmmanager.h" +#include "installdir.h" + using namespace std; using namespace oam; using namespace alarmmanager; @@ -80,11 +82,15 @@ int main (int argc, char** argv) string highCritical; string highFatal; char* p; + + string tmpDir = startup::StartUp::tmpDir(); // loop forever reading the hardware status while (TRUE) { - int returnCode = system("ipmitool sensor list > /tmp/harwareMonitor.txt"); + string logFile = tmpDir + "/harwareMonitor.txt"; + string cmd = "ipmitool sensor list > " + logFile; + int returnCode = system(cmd.c_str()); if (returnCode) { @@ -95,12 +101,12 @@ int main (int argc, char** argv) // parse output file - ifstream File ("/tmp/harwareMonitor.txt"); + ifstream File (logFile); if (!File) { // System error - cout << "Error opening /tmp/harwareMonitor.txt!!!" << endl; + cout << "Error opening " << logFile << endl; exit(-1); } From 87544762723112f9fa55e492e008e3e0f39a56a0 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 25 Sep 2018 13:46:11 -0500 Subject: [PATCH 029/127] MCOL-520 --- oamapps/postConfigure/installer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 3cde4f24e..044d6cb37 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -897,7 +897,7 @@ int main(int argc, char* argv[]) if ( !oam.checkLogStatus(logFile, "System catalog appears to exist") ) { cout << endl << "System Catalog Create Failure" << endl; - cout << "Check latest log file in " << logFile" << endl; + cout << "Check latest log file in " << logFile << endl; cout << " IMPORTANT: Once issue has been resolved, rerun postConfigure" << endl << endl; exit (1); } From 5fa5c81b6724f3d25a7cbab6e44dd4554aaf2a8a Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 25 Sep 2018 14:03:00 -0500 Subject: [PATCH 030/127] MCOL-520 --- exemgr/main.cpp | 7 ++----- oam/etc/Columnstore.xml | 12 ++++++------ tools/configMgt/autoConfigure.cpp | 4 ++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 47f71140c..afecb1129 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1370,14 +1370,11 @@ void cleanTempDir() { config::Config* config = config::Config::makeConfig(); string allowDJS = config->getConfig("HashJoin", "AllowDiskBasedJoin"); - string tmpPrefix = config->getConfig("HashJoin", "TempFilePath"); + string tmpPrefix = startup::StartUp::tmpDir(); if (allowDJS == "N" || allowDJS == "n") return; - - if (tmpPrefix.empty()) - tmpPrefix = "/tmp/infinidb"; - + tmpPrefix += "/"; assert(tmpPrefix != "/"); diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 3d4f4383e..f8cda9d65 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -249,9 +249,9 @@ 90 80 70 - 10 0.0.0.0 @@ -269,7 +269,7 @@ --> /rdwrscratch /columnstore_tmp_files - ${tmpDir}/columnstore_tmp_files + /tmp/columnstore_tmp_files dm @@ -494,8 +494,8 @@ N - $INSTALLDIR${tmpDir} + files are left behind. + /tmp --> Y diff --git a/tools/configMgt/autoConfigure.cpp b/tools/configMgt/autoConfigure.cpp index b30536d8d..5cb7a1a80 100644 --- a/tools/configMgt/autoConfigure.cpp +++ b/tools/configMgt/autoConfigure.cpp @@ -48,6 +48,8 @@ #include "liboamcpp.h" #include "configcpp.h" +#include "installdir.h" + using namespace std; using namespace oam; @@ -2009,13 +2011,11 @@ int main(int argc, char* argv[]) if ( !AllowDiskBasedJoin.empty() ) { - TempFilePath = sysConfigOld->getConfig("HashJoin", "TempFilePath"); TempFileCompression = sysConfigOld->getConfig("HashJoin", "TempFileCompression"); try { sysConfigNew->setConfig("HashJoin", "AllowDiskBasedJoin", AllowDiskBasedJoin); - sysConfigNew->setConfig("HashJoin", "TempFilePath", TempFilePath); sysConfigNew->setConfig("HashJoin", "TempFileCompression", TempFileCompression); } catch (...) From 5b7b3472c3797579c642ea07ab31a5c830b6f2f8 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 25 Sep 2018 14:19:08 -0500 Subject: [PATCH 031/127] MCOL-520 --- oam/install_scripts/post-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 57c5427e6..b2f7a361a 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -176,7 +176,7 @@ fi #get columnstore temp file directory name TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` -tmpDir=$tmpDir+$TempFileDir +tmpDir=${tmpDir}${TempFileDir} #set base columnstore temp file directory $installdir/bin/setConfig -d SystemConfig SystemTempFileDir $tmpDir From 2c38d5ba4fa80f82ed3213cee1d0b4512880f555 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 25 Sep 2018 15:25:16 -0500 Subject: [PATCH 032/127] MCOL-520 --- dbcon/mysql/install_calpont_mysql.sh | 2 +- oam/install_scripts/columnstore | 4 ++-- oam/install_scripts/run.sh | 12 ++++++------ procmon/processmonitor.cpp | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 1080e69bc..e81394cab 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -26,7 +26,7 @@ done df=$installdir/mysql/my.cnf -$installdir/mysql/bin/mysql --defaults-extra-file=$df --force --user=root mysql 2> ${tmpdir}mysql_install.log < ${tmpdir}/mysql_install.log < /dev/null 2>&1 & + exec $InstallDir/bin/run.sh -l ${tmpDir} $InstallDir/bin/ProcMon > /dev/null 2>&1 & return $RETVAL } diff --git a/oam/install_scripts/run.sh b/oam/install_scripts/run.sh index accdd3be2..9cc72ba4b 100755 --- a/oam/install_scripts/run.sh +++ b/oam/install_scripts/run.sh @@ -3,13 +3,14 @@ # $Id$ # -usage="usage: run.sh [-vh] [-s sleep] [-t tries] executable" +usage="usage: run.sh [-vh] [-s sleep] [-t tries] [-l logDir] executable" vflg=0 sopt="5" topt="0" +lopt="/tmp/columnstore_tmp_files" -while getopts "vs:t:h" flag; do +while getopts "vs:t:h:l" flag; do case $flag in v) vflg=1 ;; @@ -17,6 +18,8 @@ while getopts "vs:t:h" flag; do ;; t) topt=$OPTARG ;; + t) lopt=$OPTARG + ;; h) echo $usage exit 0 ;; @@ -45,12 +48,9 @@ if [ $vflg -gt 0 ]; then echo "starting $exename $args with sleep=$sopt and tries=$topt" fi -#get temp directory -tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` - while [ $keep_going -ne 0 ]; do $exename $args - if [ -e ${tmpDir}/StopColumnstore ]; then + if [ -e ${lopt}/StopColumnstore ]; then exit 0 fi if [ $topt -gt 0 -a $retries -ge $topt ]; then diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index a59d785ed..a1f6db8a8 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -2670,16 +2670,16 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, if ( LogFile == "off" ) { - string cmd = "mv " + outFileName + " " + saveoutFileName; + string cmd = "mv " + outFileName + " " + saveoutFileName + " > /dev/null"; system(cmd.c_str()); - cmd = "mv " + errFileName + " " + saveerrFileName; + cmd = "mv " + errFileName + " " + saveerrFileName + " > /dev/null"; system(cmd.c_str()); } else { - string cmd = "mv " + outFileName + " " + saveoutFileName; + string cmd = "mv " + outFileName + " " + saveoutFileName + " > /dev/null"; system(cmd.c_str()); - cmd = "mv " + errFileName + " " + saveerrFileName; + cmd = "mv " + errFileName + " " + saveerrFileName + " > /dev/null"; system(cmd.c_str()); } From f25082bc9a7d9829881f70fe7e004ce1d1a315cd Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 25 Sep 2018 17:20:46 -0500 Subject: [PATCH 033/127] MCOL-520 --- oam/install_scripts/run.sh | 5 +++-- oamapps/mcsadmin/mcsadmin.cpp | 4 ++-- tools/hdfsCheck/hdfsCheck.cpp | 12 +++++++++--- utils/idbdatafile/IDBPolicy.cpp | 4 ++-- writeengine/shared/we_config.cpp | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/oam/install_scripts/run.sh b/oam/install_scripts/run.sh index 9cc72ba4b..2cb05ef32 100755 --- a/oam/install_scripts/run.sh +++ b/oam/install_scripts/run.sh @@ -31,7 +31,7 @@ done shift $((OPTIND - 1)) -exename="$1" +exename="$2" if [ -z "$exename" ]; then echo $usage 1>&2 @@ -49,7 +49,8 @@ if [ $vflg -gt 0 ]; then fi while [ $keep_going -ne 0 ]; do - $exename $args +# $exename $args + $exename if [ -e ${lopt}/StopColumnstore ]; then exit 0 fi diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 0750b1abc..5a9b000e4 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -4926,10 +4926,10 @@ int processCommand(string* arguments) case 39: // getSystemDirectories { - cout << "System Installation and Temporary Logging Directories" << endl << endl; + cout << endl << "System Installation and Temporary Logging Directories" << endl << endl; cout << "System Installation Directory = " << startup::StartUp::installDir() << endl; - cout << "System Temporary Logging Directory = " << tmpDir << endl; + cout << "System Temporary Logging Directory = " << tmpDir << endl << endl; } break; diff --git a/tools/hdfsCheck/hdfsCheck.cpp b/tools/hdfsCheck/hdfsCheck.cpp index e3469668d..7790c9916 100644 --- a/tools/hdfsCheck/hdfsCheck.cpp +++ b/tools/hdfsCheck/hdfsCheck.cpp @@ -28,6 +28,10 @@ using namespace idbdatafile; using namespace std; +#include "installdir.h" + +string tmpDir; + size_t BLK_SIZE = 2048; const size_t MAX_BLK_SIZE = 1048576; size_t MAX_BLOCK = 1024; @@ -189,7 +193,7 @@ bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts ) // build the file name we are going to use ostringstream oss; // embed pid so that this is a new directory path - oss << "/tmp/idbdf-dir-" << getpid() << "-" << m_id; + oss << tmpDir << "/idbdf-dir-" << getpid() << "-" << m_id; string dir = oss.str(); m_fname = dir + "/foobar"; @@ -541,7 +545,7 @@ bool TestRunner::hdfsRdwrExhaustTest() // choose a new filename that is specific to our thread ostringstream oss; // embed pid so that this is a new directory path - oss << "/tmp/hdfsrdwr-" << getpid() << "-" << m_id; + oss << tmpDir << "/hdfsrdwr-" << getpid() << "-" << m_id; string newpath = oss.str(); // open a file with arbitrarily small buffer @@ -755,13 +759,15 @@ int main(int argc, char** argv) logFailure("missing hdfs-plugin argument"); return 1; } + + tmpDir = startup::StartUp::tmpDir(); opts.pluginFile = argv[optind]; try { // init the library with logging enabled - std::string hdfsRdwrScratch = "/tmp/rdwr_scratch"; + std::string hdfsRdwrScratch = tmpDir + "/rdwr_scratch"; IDBPolicy::init( true, true, hdfsRdwrScratch, 2147483648 ); if ( !IDBPolicy::installPlugin(opts.pluginFile) ) diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 4992a1d15..e640eeffe 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -218,8 +218,8 @@ void IDBPolicy::configIDBPolicy() // Directory in which to place file buffer temporary files. string tmpDir = startup::StartUp::tmpDir(); - string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); - hdfsRdwrScratch = tmpDir + hdfsRdwrScratch; + string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); + string hdfsRdwrScratch = tmpDir + scratch; IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); diff --git a/writeengine/shared/we_config.cpp b/writeengine/shared/we_config.cpp index 070e5d918..8b9058057 100644 --- a/writeengine/shared/we_config.cpp +++ b/writeengine/shared/we_config.cpp @@ -266,7 +266,7 @@ void Config::checkReload( ) string TmpFileDir = startup::StartUp::tmpDir(); string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); - hdfsRdwrScratch = TmpFileDir + hdfsRdwrScratch + hdfsRdwrScratch = TmpFileDir + hdfsRdwrScratch; if ( hdfsRdwrScratch.length() == 0 ) { From 07e78c410b106e0bae715a7c5617f1119a0d840e Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 26 Sep 2018 15:05:27 -0500 Subject: [PATCH 034/127] MCOL-520 --- oam/install_scripts/post-install | 16 ++++++---- oam/install_scripts/run.sh | 2 +- oam/install_scripts/syslogSetup.sh | 12 ++++--- utils/idbdatafile/IDBPolicy.cpp | 3 ++ utils/startup/installdir.cpp | 50 +++++++++++++++--------------- 5 files changed, 46 insertions(+), 37 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index b2f7a361a..25452dada 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -172,17 +172,21 @@ tmpDir="/tmp" if [ $user != "root" ]; then tmpDir=$HOME"./tmp" mkdir $tmpDir >/dev/null 2>&1 +else + #get columnstore temp file directory name + TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` + tmpDir=${tmpDir}${TempFileDir} fi -#get columnstore temp file directory name -TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` -tmpDir=${tmpDir}${TempFileDir} - #set base columnstore temp file directory $installdir/bin/setConfig -d SystemConfig SystemTempFileDir $tmpDir -#create columnstore temp file directory -mkdir -p $tmpDir >/dev/null 2>&1 +#get place file buffer temporary files +hdfsRdwrScratch=`$installdir/bin/getConfig SystemConfig hdfsRdwrScratch` +hdfsDir=${tmpDir}${hdfsRdwrScratch} + +#create place file buffer temporary files +mkdir -p $hdfsDir >/dev/null 2>&1 #create mount directories mkdir /mnt/tmp > /dev/null 2>&1 diff --git a/oam/install_scripts/run.sh b/oam/install_scripts/run.sh index 2cb05ef32..ec747bedc 100755 --- a/oam/install_scripts/run.sh +++ b/oam/install_scripts/run.sh @@ -18,7 +18,7 @@ while getopts "vs:t:h:l" flag; do ;; t) topt=$OPTARG ;; - t) lopt=$OPTARG + l) lopt=$OPTARG ;; h) echo $usage exit 0 diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 347889f4a..c741d158a 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -164,12 +164,14 @@ if [ ! -z "$syslog_conf" ] ; then chown $user:$user /home/$user/mariadb/columnstore/etc/* fi - if [ "$syslog_conf" != /etc/rsyslog.d/columnstore.conf ]; then - rm -f ${syslog_conf}.columnstoreSave - cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 - sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 + if [ "$syslog_conf" != /etc/rsyslog.d/columnstore.conf ] || + [ "$syslog_conf" != /etc/rsyslog.d/49-columnstore.conf ]; then + + rm -f ${syslog_conf}.columnstoreSave + cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 + sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 fi - + egrep -qs 'MariaDB Columnstore Database Platform Logging' ${syslog_conf} if [ $? -ne 0 ]; then #set the syslog for ColumnStore logging diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index e640eeffe..4d0679308 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -63,6 +63,7 @@ void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& if ( hdfsRdwrScratch.length() > 0 ) { // TODO-check to make sure this directory has sufficient space, whatever that means. + boost::filesystem::path tmpfilepath( hdfsRdwrScratch ); if (boost::filesystem::exists(tmpfilepath)) @@ -79,6 +80,8 @@ void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& } else { + cout << tmpfilepath << endl; + if (!boost::filesystem::create_directory(tmpfilepath)) { // We failed to create the scratch directory diff --git a/utils/startup/installdir.cpp b/utils/startup/installdir.cpp index 663ac74b3..e53f09132 100644 --- a/utils/startup/installdir.cpp +++ b/utils/startup/installdir.cpp @@ -35,6 +35,9 @@ using namespace boost; using namespace config; +#include + +using namespace std; namespace startup { @@ -97,36 +100,33 @@ const string StartUp::tmpDir() #else - Config* sysConfig = Config::makeConfig(); + //check for non-root user + const char* p = getenv("HOME"); + string homedir = p; - string TempFileDir; - - try + if (homedir == "/root") { - TempFileDir = sysConfig->getConfig("SystemConfig", "TempFileDir"); + Config* sysConfig = Config::makeConfig(); + + string TempFileDir; + + try + { + TempFileDir = sysConfig->getConfig("SystemConfig", "TempFileDir"); + } + catch (...) + {} + + fTmpDirp = new string("/tmp"); + + *fTmpDirp = homedir + TempFileDir; } - catch (...) - {} - - fTmpDirp = new string("/tmp"); - //See if we can figure out the tmp dir in Linux... - //1. env var COLUMNSTORE_INSTALL_DIR - const char* p = 0; - p = getenv("COLUMNSTORE_INSTALL_DIR"); - - if (p && *p) - { - string homedir = "/"; - char* p = getenv("HOME"); - - if (p && *p) - homedir = p; - - *fTmpDirp = homedir + "/.tmp"; + else + { + // non-root user + *fTmpDirp = homedir + "/.tmp"; } - *fTmpDirp = *fTmpDirp + TempFileDir; - #endif return *fTmpDirp; From 92cb0614557d1b4c7f6c7246df17813e88978f8f Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 26 Sep 2018 15:11:05 -0500 Subject: [PATCH 035/127] MCOL-520 --- utils/startup/installdir.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/startup/installdir.cpp b/utils/startup/installdir.cpp index e53f09132..1229f6e58 100644 --- a/utils/startup/installdir.cpp +++ b/utils/startup/installdir.cpp @@ -119,7 +119,7 @@ const string StartUp::tmpDir() fTmpDirp = new string("/tmp"); - *fTmpDirp = homedir + TempFileDir; + *fTmpDirp = *fTmpDirp + TempFileDir; } else { From 4d1729c967c0cd68ae5a296780ebdb88eac7ac8c Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 27 Sep 2018 08:59:03 -0500 Subject: [PATCH 036/127] MCOL-520 --- procmon/processmonitor.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index a1f6db8a8..62136c5c2 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1830,7 +1830,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if ( !rootUser) { - cmd = "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /dev/null"; + cmd = "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /dev/null 2>&1"; system(cmd.c_str()); } @@ -2670,16 +2670,16 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, if ( LogFile == "off" ) { - string cmd = "mv " + outFileName + " " + saveoutFileName + " > /dev/null"; + string cmd = "mv " + outFileName + " " + saveoutFileName + " > /dev/null 2>&1"; system(cmd.c_str()); - cmd = "mv " + errFileName + " " + saveerrFileName + " > /dev/null"; + cmd = "mv " + errFileName + " " + saveerrFileName + " > /dev/null 2>&1"; system(cmd.c_str()); } else { - string cmd = "mv " + outFileName + " " + saveoutFileName + " > /dev/null"; + string cmd = "mv " + outFileName + " " + saveoutFileName + " > /dev/null 2>&1"; system(cmd.c_str()); - cmd = "mv " + errFileName + " " + saveerrFileName + " > /dev/null"; + cmd = "mv " + errFileName + " " + saveerrFileName + " > /dev/null 2>&1"; system(cmd.c_str()); } @@ -3109,7 +3109,7 @@ int ProcessMonitor::updateLog(std::string action, std::string level) //if non-root, change file permissions so we can update it if ( !rootUser) { - string cmd = "chmod 666 " + fileName + " > /dev/null"; + string cmd = "chmod 666 " + fileName + " > /dev/null 2>&1"; system(cmd.c_str()); } @@ -4866,7 +4866,7 @@ std::string ProcessMonitor::sendMsgProcMon1( std::string module, ByteStream msg, string IPAddr = sysConfig->getConfig(msgPort, "IPAddr"); string cmdLine = "ping "; - string cmdOption = " -c 1 -w 5 >> /dev/null"; + string cmdOption = " -c 1 -w 5 >> /dev/null 2>&1"; string cmd = cmdLine + IPAddr + cmdOption; if ( system(cmd.c_str()) != 0 ) @@ -5950,7 +5950,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; + string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null 2>&1"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); @@ -6021,7 +6021,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd = "mount " + startup::StartUp::installDir() + "/data" + oam.itoa(dbrootID) + " > /dev/null"; + string cmd = "mount " + startup::StartUp::installDir() + "/data" + oam.itoa(dbrootID) + " > /dev/null 2>&1"; system(cmd.c_str()); return true; } @@ -6326,7 +6326,7 @@ int ProcessMonitor::checkDataMount() if ( !rootUser) { - cmd = "chown -R " + USER + ":" + USER + " " + dbroot + " > /dev/null"; + cmd = "chown -R " + USER + ":" + USER + " " + dbroot + " > /dev/null 2>&1"; system(cmd.c_str()); } From cd848bfc854cddba5c0ac26bce5ea50396110dc1 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 27 Sep 2018 09:37:55 -0500 Subject: [PATCH 037/127] MCOL-520 --- oam/install_scripts/run.sh | 7 +++---- oam/install_scripts/syslogSetup.sh | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/oam/install_scripts/run.sh b/oam/install_scripts/run.sh index ec747bedc..b4edd77dd 100755 --- a/oam/install_scripts/run.sh +++ b/oam/install_scripts/run.sh @@ -29,9 +29,9 @@ while getopts "vs:t:h:l" flag; do esac done -shift $((OPTIND - 1)) +shift $((OPTIND)) -exename="$2" +exename="$@" if [ -z "$exename" ]; then echo $usage 1>&2 @@ -49,8 +49,7 @@ if [ $vflg -gt 0 ]; then fi while [ $keep_going -ne 0 ]; do -# $exename $args - $exename + $exename $args if [ -e ${lopt}/StopColumnstore ]; then exit 0 fi diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index c741d158a..fce9be413 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -164,8 +164,8 @@ if [ ! -z "$syslog_conf" ] ; then chown $user:$user /home/$user/mariadb/columnstore/etc/* fi - if [ "$syslog_conf" != /etc/rsyslog.d/columnstore.conf ] || - [ "$syslog_conf" != /etc/rsyslog.d/49-columnstore.conf ]; then + if [ "$syslog_conf" == /etc/rsyslog.d/columnstore.conf ] || + [ "$syslog_conf" == /etc/rsyslog.d/49-columnstore.conf ]; then rm -f ${syslog_conf}.columnstoreSave cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 From c86b569c97ab01ee2b107432921aa9b0c239df15 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 27 Sep 2018 15:59:57 -0500 Subject: [PATCH 038/127] MCOL-520 --- dbcon/execplan/calpontsystemcatalog.cpp | 2 +- oam/install_scripts/post-install | 51 +++++++++++--------- oam/install_scripts/pre-uninstall | 30 +++++++----- oam/install_scripts/run.sh | 4 +- oam/install_scripts/syslogSetup.sh | 3 +- oamapps/mcsadmin/mcsadmin.cpp | 62 +++++++++++++------------ oamapps/postConfigure/postConfigure.cpp | 6 +-- utils/startup/installdir.cpp | 4 +- 8 files changed, 90 insertions(+), 72 deletions(-) diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index cf11b6018..7b75c15a9 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -94,7 +94,7 @@ throw runtime_error("CALPONT_INTERNAL_ERROR"); \ #if CSC_DEBUG namespace { - string tmpDir = startup::StartUp::installDir() + "csc.log"; + string tmpDir = startup::StartUp::tmpDir() + "/csc.log"; std::ofstream csclog(tmpDir, std::ios::app); } #define DEBUG csclog diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 25452dada..38e21cb8b 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -170,7 +170,7 @@ rm -f $installdir/data/bulk/tmpjob/* >/dev/null 2>&1 #get temp base directory tmpDir="/tmp" if [ $user != "root" ]; then - tmpDir=$HOME"./tmp" + tmpDir=$HOME"/.tmp" mkdir $tmpDir >/dev/null 2>&1 else #get columnstore temp file directory name @@ -264,12 +264,17 @@ else chown $user:$user $installdir/etc/Columnstore.xml - echo "" - echo "NOTE: For non-root install, you will need to run the following command as root user to" - echo " setup the MariaDB ColumnStore System Logging" - echo "" - echo " $installdir/bin/syslogSetup.sh --installdir=$installdir --user=$user install" - echo "" +cat < /dev/null 2>&1 $installdir/myql/columnstore-Mysql stop > /dev/null 2>&1 -export COLUMNSTORE_INSTALL_DIR=$installdir +if [ $user != "root" ]; then + export COLUMNSTORE_INSTALL_DIR=$installdir + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib +fi -cloud=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation Cloud` +cloud=`$installdir/bin/getConfig Installation Cloud` if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then if test -f /etc/fstab ; then sed -i '/Columnstore\/data/d' /etc/fstab > /dev/null 2>&1 @@ -76,13 +79,13 @@ $installdir/bin/clearShm > /dev/null 2>&1 tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` # delete tmp files -rm -rf $tmpDir +rm -rf $tmpDir/* rm -f $installdir/local/*.columnstore rm -rf $installdir/local/etc/ rm -f $installdir/data/bulk${tmpDir}job/* >/dev/null 2>&1 rm -f $installdir/local/moveDbrootTransactionLog -lockdir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation LockFileDirectory` +lockdir=`$installdir/bin/getConfig Installation LockFileDirectory` rm -f $lockdir/columnstore rm -f $lockdir/mysql-Columnstore @@ -138,13 +141,18 @@ if [ $user = "root" ]; then ldconfig else rm -f /etc/default/columnstore - - echo "" - echo "NOTE: For non-root install, you will need to run the following command as root user to" - echo " uninstall the MariaDB ColumnStore System Logging" - echo "" - echo " $installdir/bin/syslogSetup.sh --installdir=$installdir --user=$user uninstall" - echo "" + +cat </dev/null 2>&1 sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 5a9b000e4..20ec493a5 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -68,6 +68,7 @@ bool rootUser = true; string HOME = "/root"; string SingleServerInstall; string tmpDir; +string installDir; bool repeatStop; @@ -189,7 +190,10 @@ int main(int argc, char* argv[]) string ccHistoryFile = HOME + "/.cc_history"; - string cf = startup::StartUp::installDir() + "/etc/" + ConsoleCmdsFile; + tmpDir = startup::StartUp::tmpDir(); + installDir = startup::StartUp::installDir(); + + string cf = installDir + "/etc/" + ConsoleCmdsFile; fConfig = Config::makeConfig(cf); // setupSignalHandlers(); @@ -236,8 +240,6 @@ int main(int argc, char* argv[]) if (user != 0) rootUser = false; - tmpDir = startup::StartUp::tmpDir(); - // create/open command log file if not created logFile.open(DEFAULT_LOG_FILE.c_str(), ios::app); @@ -2600,7 +2602,7 @@ int processCommand(string* arguments) { string logFile = tmpDir + "/cc-stop.pdsh"; - cmd = "pdsh -a '/" + startup::StartUp::installDir() + "/bin/columnstore stop' > " + logFile + " 2>&1"; + cmd = "pdsh -a '/" + installDir + "/bin/columnstore stop' > " + logFile + " 2>&1"; system(cmd.c_str()); if (oam.checkLogStatus(logFile, "exit") ) @@ -2610,7 +2612,7 @@ int processCommand(string* arguments) } else { - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/status.log"; + cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/status.log"; system(cmd.c_str()); } } @@ -2620,7 +2622,7 @@ int processCommand(string* arguments) if ( gracefulTemp == FORCEFUL ) { - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/status.log"; + cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/status.log"; system(cmd.c_str()); cout << endl << " Successful shutdown of System (stopped local columnstore service) " << endl << endl; } @@ -2628,7 +2630,7 @@ int processCommand(string* arguments) if (Failed.find("Connection refused") != string::npos) { cout << endl << "**** shutdownSystem Error : ProcessManager not Active, stopping columnstore service" << endl; - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/status.log"; + cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/status.log"; system(cmd.c_str()); cout << endl << " Successful stop of local columnstore service " << endl << endl; } @@ -2650,7 +2652,7 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { string logFile = tmpDir + "cc-stop.pdsh"; - cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore stop' > " + logFile + " 2>&1"; + cmd = "pdsh -a '" + installDir + "/bin/columnstore stop' > " + logFile + " 2>&1"; system(cmd.c_str()); if (oam.checkLogStatus(logFile, "exit") ) @@ -2757,7 +2759,7 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { string logFile = tmpDir + "/cc-restart.pdsh"; - cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore restart' > " + logFile + " 2>&1"; + cmd = "pdsh -a '" + installDir + "/bin/columnstore restart' > " + logFile + " 2>&1"; system(cmd.c_str()); if (oam.checkLogStatus(logFile, "exit") ) @@ -2794,7 +2796,7 @@ int processCommand(string* arguments) if ( modulename == localModule ) { - cmd = startup::StartUp::installDir() + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; + cmd = installDir + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; int rtnCode = system(cmd.c_str()); if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0) @@ -2812,7 +2814,7 @@ int processCommand(string* arguments) for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) { //run remote command script - cmd = startup::StartUp::installDir() + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " '" + startup::StartUp::installDir() + "/bin/columnstore restart' 0"; + cmd = installDir + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " '" + installDir + "/bin/columnstore restart' 0"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) < 0) @@ -2821,7 +2823,7 @@ int processCommand(string* arguments) cout << endl << "**** startSystem Failed" << endl; // stop local columnstore service - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; + cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; system(cmd.c_str()); FAILED = true; @@ -2835,7 +2837,7 @@ int processCommand(string* arguments) cout << endl << "**** startSystem Failed" << endl; // stop local columnstore service - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; + cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; system(cmd.c_str()); FAILED = true; @@ -2861,7 +2863,7 @@ int processCommand(string* arguments) //just kick off local server cout << endl << " System being started, please wait..."; cout.flush(); - cmd = startup::StartUp::installDir() + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; + cmd = installDir + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; int rtnCode = system(cmd.c_str()); if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0) @@ -2999,7 +3001,7 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { string logFile = tmpDir + "/cc-restart.pdsh"; - cmd = "pdsh -a '" + startup::StartUp::installDir() + "/bin/columnstore restart' > " + logFile + " 2>&1"; + cmd = "pdsh -a '" + installDir + "/bin/columnstore restart' > " + logFile + " 2>&1"; system(cmd.c_str()); if (oam.checkLogStatus(logFile, "exit") ) @@ -3042,7 +3044,7 @@ int processCommand(string* arguments) for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) { //run remote command script - cmd = startup::StartUp::installDir() + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " '" + startup::StartUp::installDir() + "/bin/columnstore restart' 0"; + cmd = installDir + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " '" + installDir + "/bin/columnstore restart' 0"; int rtnCode = system(cmd.c_str()); @@ -3052,7 +3054,7 @@ int processCommand(string* arguments) cout << endl << "**** restartSystem Failed" << endl; // stop local columnstore service - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; + cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; system(cmd.c_str()); FAILED = true; @@ -3067,7 +3069,7 @@ int processCommand(string* arguments) FAILED = true; // stop local columnstore service - cmd = startup::StartUp::installDir() + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; + cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1"; system(cmd.c_str()); break; @@ -3083,7 +3085,7 @@ int processCommand(string* arguments) break; //RESTART LOCAL HOST - cmd = startup::StartUp::installDir() + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; + cmd = installDir + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; int rtnCode = system(cmd.c_str()); if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0) @@ -3103,7 +3105,7 @@ int processCommand(string* arguments) //just kick off local server cout << " System being restarted, please wait..."; cout.flush(); - cmd = startup::StartUp::installDir() + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; + cmd = installDir + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) @@ -4926,10 +4928,10 @@ int processCommand(string* arguments) case 39: // getSystemDirectories { - cout << endl << "System Installation and Temporary Logging Directories" << endl << endl; + cout << endl << "System Installation and Temporary File Directories" << endl << endl; - cout << "System Installation Directory = " << startup::StartUp::installDir() << endl; - cout << "System Temporary Logging Directory = " << tmpDir << endl << endl; + cout << "System Installation Directory = " << installDir << endl; + cout << "System Temporary File Directory = " << tmpDir << endl << endl; } break; @@ -6148,13 +6150,13 @@ int processCommand(string* arguments) ++it) { string deviceName = *it; - string entry = deviceName + " " + startup::StartUp::installDir() + "/gluster/brick" + oam.itoa(brickID) + " " + deviceType + " defaults 1 2"; + string entry = deviceName + " " + installDir + "/gluster/brick" + oam.itoa(brickID) + " " + deviceType + " defaults 1 2"; //send update pm oam.distributeFstabUpdates(entry, moduleName); } } - string command = startup::StartUp::installDir() + "/bin/remote_command.sh " + (*hostConfigIter).IPAddr + " " + password + " 'mkdir -p " + startup::StartUp::installDir() + "/gluster/brick" + oam.itoa(brickID) + "'"; + string command = installDir + "/bin/remote_command.sh " + (*hostConfigIter).IPAddr + " " + password + " 'mkdir -p " + installDir + "/gluster/brick" + oam.itoa(brickID) + "'"; system(command.c_str()); brickID++; } @@ -7866,7 +7868,7 @@ int processCommand(string* arguments) oam.startModule(devicenetworklist, ackTemp); //reload DBRM with new configuration, needs to be done here after startModule - cmd = startup::StartUp::installDir() + "/bin/dbrmctl reload > /dev/null 2>&1"; + cmd = installDir + "/bin/dbrmctl reload > /dev/null 2>&1"; system(cmd.c_str()); sleep(15); @@ -8182,7 +8184,7 @@ int ProcessSupportCommand(int CommandID, std::string arguments[]) //run remote command script HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - cmd = startup::StartUp::installDir() + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " reboot " ; + cmd = installDir + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " reboot " ; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) @@ -8301,7 +8303,7 @@ int ProcessSupportCommand(int CommandID, std::string arguments[]) HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); string ipAddr = (*pt1).IPAddr; //run remote command script - cmd = startup::StartUp::installDir() + "/bin/remote_command.sh " + ipAddr + " " + password + " reboot " ; + cmd = installDir + "/bin/remote_command.sh " + ipAddr + " " + password + " reboot " ; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) @@ -9427,7 +9429,7 @@ void printModuleDisk(ModuleDisk moduledisk) cout << "Mount Point Total Blocks Used Blocks Usage %" << endl; cout << "----------------------------- ------------ ------------ -------" << endl; - string etcdir = startup::StartUp::installDir() + "/etc"; + string etcdir = installDir + "/etc"; for ( unsigned int i = 0 ; i < moduledisk.diskusage.size(); i++) { @@ -9468,7 +9470,7 @@ void printModuleDisk(ModuleDisk moduledisk) void printModuleResources(TopProcessCpuUsers topprocesscpuusers, ModuleCpu modulecpu, TopProcessMemoryUsers topprocessmemoryusers, ModuleMemory modulememory, ModuleDisk moduledisk) { Oam oam; - string etcdir = startup::StartUp::installDir() + "/etc"; + string etcdir = installDir + "/etc"; cout << endl << "Module '" + topprocesscpuusers.ModuleName + "' Resource Usage" << endl << endl; diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 6ffef9012..bf967d5f9 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3621,7 +3621,7 @@ int main(int argc, char* argv[]) if ( remote_installer_debug == "1" ) { - logfile = tmpDir; + logfile = tmpDir + "/"; logfile += remoteModuleName + "_" + EEPackageType + "_install.log"; debug_logfile = " > " + logfile; } @@ -3928,7 +3928,7 @@ int main(int argc, char* argv[]) if (hdfs && !nonDistribute ) { - string postConfigurePsdhLog = tmpDir + "postConfigure.pdsh.log"; + string postConfigurePsdhLog = tmpDir + "/postConfigure.pdsh.log"; cout << endl << "----- Starting MariaDB ColumnStore Service on all Modules -----" << endl << endl; string cmd = "pdsh -a '" + installDir + "/bin/columnstore restart' > " + postConfigurePsdhLog + " 2>&1"; @@ -4022,7 +4022,7 @@ int main(int argc, char* argv[]) } } - string dbbuilderLog = tmpDir + "dbbuilder.log"; + string dbbuilderLog = tmpDir + "/dbbuilder.log"; if (hdfs) cmd = "bash -c '. " + installDir + "/bin/" + DataFileEnvFile + ";" + installDir + "/bin/dbbuilder 7 > " + dbbuilderLog; diff --git a/utils/startup/installdir.cpp b/utils/startup/installdir.cpp index 1229f6e58..f17740ca2 100644 --- a/utils/startup/installdir.cpp +++ b/utils/startup/installdir.cpp @@ -124,7 +124,9 @@ const string StartUp::tmpDir() else { // non-root user - *fTmpDirp = homedir + "/.tmp"; + fTmpDirp = new string(homedir); + + *fTmpDirp = *fTmpDirp + "/.tmp"; } #endif From 3393bd1855ddeed1bc828cb27aae556a023aff4f Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 28 Sep 2018 11:00:50 -0500 Subject: [PATCH 039/127] MCOL-520 --- oam/install_scripts/module_installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 0d9008fd4..9627acff7 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -144,7 +144,7 @@ if [ $module = "um" ] || ( [ $module = "pm" ] && [ $PMwithUM = "y" ] ) || [ $Ser fi echo "Run post-mysql-install" - $COLUMNSTORE_INSTALL_DIR/bin/post-mysql-install --installdir=$COLUMNSTORE_INSTALL_DIR > ${tmpDir}/post-mysql-install.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/post-mysql-install --installdir=$COLUMNSTORE_INSTALL_DIR --tmpdir=${tmpDir} > ${tmpDir}/post-mysql-install.log 2>&1 if [ $? -ne 0 ]; then echo "ERROR: post-mysql-install failed: check ${tmpDir}/post-mysql-install.log" exit 1 From 2f0fb500dfe1a499f4610a4b72891b2dd80338fc Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 28 Sep 2018 14:39:00 -0500 Subject: [PATCH 040/127] MCOL-520 --- oam/install_scripts/pre-uninstall | 11 ++++++----- oamapps/postConfigure/postConfigure.cpp | 16 ++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 57336f854..bfaac6603 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -64,12 +64,12 @@ else rm -f /etc/profile.d/columnstoreAlias.sh fi - #remove log file directories if [ $user == "root" ]; then rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1 rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1 fi + rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1\ # delete Mariab Columnstore shared memory segments @@ -77,12 +77,13 @@ $installdir/bin/clearShm > /dev/null 2>&1 #get temp directory tmpDir=`$installdir/bin/getConfig SystemConfig SystemTempFileDir` - -# delete tmp files -rm -rf $tmpDir/* +if [[ $tmpDir = *"tmp" ]]; then + # delete tmp files + rm -rf $tmpDir/* +fi rm -f $installdir/local/*.columnstore rm -rf $installdir/local/etc/ -rm -f $installdir/data/bulk${tmpDir}job/* >/dev/null 2>&1 +rm -f $installdir/data/bulk/tmp/job/* >/dev/null 2>&1 rm -f $installdir/local/moveDbrootTransactionLog lockdir=`$installdir/bin/getConfig Installation LockFileDirectory` diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index bf967d5f9..cc1ef6f8f 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1806,8 +1806,8 @@ int main(int argc, char* argv[]) // // Module Configuration // - cout << endl; - cout << "===== Setup the Module Configuration =====" << endl << endl; + cout << endl << endl; + cout << "===== Setup the Module Configuration =====" << endl; if (amazonInstall) { @@ -3469,7 +3469,7 @@ int main(int argc, char* argv[]) ( (IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM) && pmwithum ) ) { //run the mysql / mysqld setup scripts - cout << endl << "===== Running the MariaDB ColumnStore MariaDB Server setup scripts =====" << endl << endl; + cout << endl << "===== Running the MariaDB ColumnStore MariaDB Server setup scripts =====" << endl; checkMysqlPort(mysqlPort, sysConfig); @@ -3485,8 +3485,8 @@ int main(int argc, char* argv[]) { cout << endl; cout << "Next step is to enter the password to access the other Servers." << endl; - cout << "This is either your password or you can default to using a ssh key" << endl; - cout << "If using a password, the password needs to be the same on all Servers." << endl << endl; + cout << "This is either user password or you can default to using a ssh key" << endl; + cout << "If using a user password, the password needs to be the same on all Servers." << endl << endl; if ( noPrompting ) { cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl; @@ -3573,14 +3573,14 @@ int main(int argc, char* argv[]) if ( EEPackageType == "rpm" ) { cout << "Performing a MariaDB ColumnStore System install using RPM packages" << endl; - cout << "located in the " + HOME + " directory." << endl << endl; + cout << "located in the " + HOME + " directory." << endl; } else { if ( EEPackageType == "binary" ) { cout << "Performing a MariaDB ColumnStore System install using a Binary package" << endl; - cout << "located in the " + HOME + " directory." << endl << endl; + cout << "located in the " + HOME + " directory." << endl; } else { @@ -4851,7 +4851,7 @@ bool storageSetup(bool amazonInstall) return true; } - cout << "===== Setup Storage Configuration =====" << endl << endl; + cout << "===== Setup Storage Configuration =====" << endl; string storageType; From d435335cdc92b077e8b5c0918da8e626f1545164 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 28 Sep 2018 14:58:36 -0500 Subject: [PATCH 041/127] MCOL-520 --- oam/install_scripts/slave-rep-columnstore.sh | 2 -- oamapps/postConfigure/postConfigure.cpp | 8 +++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/oam/install_scripts/slave-rep-columnstore.sh b/oam/install_scripts/slave-rep-columnstore.sh index 8468d903e..a181c3ead 100644 --- a/oam/install_scripts/slave-rep-columnstore.sh +++ b/oam/install_scripts/slave-rep-columnstore.sh @@ -38,8 +38,6 @@ for arg in "$@"; do elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then tmpdir="$(echo $arg | awk -F= '{print $2}')" fi - - fi done test -f $installdir/post/functions && . $installdir/post/functions diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index cc1ef6f8f..1c864c833 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1705,7 +1705,13 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - + + + + + + + string numBlocksPct; try From 05bcd58e7603a329ba9fe3c96849c5a3d4953c11 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 1 Oct 2018 12:51:38 -0500 Subject: [PATCH 042/127] MCOL-520 --- oam/install_scripts/binary_installer.sh | 2 +- oam/install_scripts/slave-rep-columnstore.sh | 2 +- procmon/processmonitor.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/install_scripts/binary_installer.sh b/oam/install_scripts/binary_installer.sh index 59a35dd8b..6b8b2f642 100644 --- a/oam/install_scripts/binary_installer.sh +++ b/oam/install_scripts/binary_installer.sh @@ -133,7 +133,7 @@ send_user "\n" # send_user "Install MariaDB Columnstore Package on Module " send_user " \n" -send "ssh -v $USERNAME@$SERVER 'tar -C $PREFIX --exclude db -zxvf $CALPONTPKG'\n" +send "ssh -v $USERNAME@$SERVER 'tar -C $PREFIX --exclude db -zxf $CALPONTPKG'\n" set timeout 360 expect { "word: " { send "$PASSWORD\n" diff --git a/oam/install_scripts/slave-rep-columnstore.sh b/oam/install_scripts/slave-rep-columnstore.sh index a181c3ead..bb4c5b6af 100644 --- a/oam/install_scripts/slave-rep-columnstore.sh +++ b/oam/install_scripts/slave-rep-columnstore.sh @@ -50,7 +50,7 @@ password="Calpont1" # # Run stop slave command # -echo "Run stop slave command" >>${tmpdir}slave-rep-status.log +echo "Run stop slave command" >>${tmpdir}/slave-rep-status.log cat >${tmpdir}/idb_slave-rep.sql < " + tmpLogDir + "/mariadb-command-line.sh.log 2>&1"; + string cmd = startup::StartUp::installDir() + "/bin/mariadb-command-line.sh --installdir=" + startup::StartUp::installDir() + " --command='" + command + "' --port=" + MySQLPort + " --tmpdir=" + tmpLogDir + " > " + tmpLogDir + "/mariadb-command-line.sh.log 2>&1"; log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); From 8d3ce9757f18e5bbf69683e882ed1ff9764cb259 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 1 Oct 2018 14:39:17 -0500 Subject: [PATCH 043/127] MCOL-520 - initial changes for /dev/shm --- oam/install_scripts/post-install | 6 ++++++ oamapps/postConfigure/postConfigure.cpp | 2 +- procmon/main.cpp | 15 ++++++++++----- utils/rwlock/rwlock.cpp | 17 ++++++++++++++++- versioning/BRM/brmshmimpl.cpp | 21 ++++++++++++++++++--- versioning/BRM/mastersegmenttable.cpp | 19 +++++++++++++++++-- 6 files changed, 68 insertions(+), 12 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 38e21cb8b..98a7e4a02 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -191,6 +191,12 @@ mkdir -p $hdfsDir >/dev/null 2>&1 #create mount directories mkdir /mnt/tmp > /dev/null 2>&1 +#create shared memory if non-root +if [ $user != "root" ]; then + mkdir -p ${installdir}/dev/shm + chmod 666 ${installdir}/dev/shm +fi + # remove mysql archive log test -d $installdir/mysql/db || mkdir -p $installdir/mysql/db rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1 diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1c864c833..31239341f 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -852,7 +852,7 @@ int main(int argc, char* argv[]) if (temp == "1") { singleServerInstall = temp; - cout << endl << "Performing the Single Server Install." << endl; + cout << endl << "Performing the Single Server Install." << endl << endl; if ( reuseConfig == "n" ) { diff --git a/procmon/main.cpp b/procmon/main.cpp index 6047f2dfa..688424f1e 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -1886,6 +1886,11 @@ static void statusControlThread() // //Allocate Shared Memory for storing Process Status Data // + + string shmLocation = "/dev/shm/"; + if ( !rootUser) + shmLocation = startup::StartUp::installDir() + "/dev/shm/"; + PROCSTATshmsize = MAX_PROCESS * sizeof(shmProcessStatus); bool memInit = true; #if 0 @@ -1914,7 +1919,7 @@ static void statusControlThread() bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif @@ -1992,7 +1997,7 @@ static void statusControlThread() bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif @@ -2111,7 +2116,7 @@ static void statusControlThread() bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif @@ -2209,7 +2214,7 @@ static void statusControlThread() bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif @@ -2299,7 +2304,7 @@ static void statusControlThread() bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif diff --git a/utils/rwlock/rwlock.cpp b/utils/rwlock/rwlock.cpp index 8d37b5124..4ffec938f 100644 --- a/utils/rwlock/rwlock.cpp +++ b/utils/rwlock/rwlock.cpp @@ -52,6 +52,8 @@ using namespace boost::posix_time; #include "shmkeys.h" +#include "installdir.h" + namespace { using namespace rwlock; @@ -141,13 +143,26 @@ RWLockShmImpl::RWLockShmImpl(int key, bool excl) string keyName = BRM::ShmKeys::keyToName(key); fKeyString = keyName; + bool rootUser = true; + + //check if root-user + int user; + user = getuid(); + + if (user != 0) + rootUser = false; + + string shmLocation = "/dev/shm/"; + if ( !rootUser) + shmLocation = startup::StartUp::installDir() + "/dev/shm/"; + try { #if BOOST_VERSION < 104500 bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index f5985fc8f..38584760e 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -36,6 +36,8 @@ namespace bi = boost::interprocess; #include "brmshmimpl.h" #include "brmtypes.h" +#include "installdir.h" + namespace BRM { @@ -44,6 +46,19 @@ BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : { string keyName = ShmKeys::keyToName(fKey); + bool rootUser = true; + + //check if root-user + int user; + user = getuid(); + + if (user != 0) + rootUser = false; + + string shmLocation = "/dev/shm/"; + if ( !rootUser) + shmLocation = startup::StartUp::installDir() + "/dev/shm/"; + if (fSize == 0) { unsigned tries = 0; @@ -89,7 +104,7 @@ again: bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = "shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif @@ -147,7 +162,7 @@ int BRMShmImpl::grow(unsigned newKey, off_t newSize) bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif @@ -194,7 +209,7 @@ int BRMShmImpl::clear(unsigned newKey, off_t newSize) bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif diff --git a/versioning/BRM/mastersegmenttable.cpp b/versioning/BRM/mastersegmenttable.cpp index 4d5816f41..0abf81c0c 100644 --- a/versioning/BRM/mastersegmenttable.cpp +++ b/versioning/BRM/mastersegmenttable.cpp @@ -41,6 +41,8 @@ using namespace rwlock; #include "mastersegmenttable.h" #undef MASTERSEGMENTTABLE_DLLEXPORT +#include "installdir.h" + namespace { using namespace BRM; @@ -73,13 +75,26 @@ MasterSegmentTableImpl::MasterSegmentTableImpl(int key, int size) { string keyName = ShmKeys::keyToName(key); + bool rootUser = true; + + //check if root-user + int user; + user = getuid(); + + if (user != 0) + rootUser = false; + + string shmLocation = "/dev/shm/"; + if ( !rootUser) + shmLocation = startup::StartUp::installDir() + "/dev/shm/"; + try { #if BOOST_VERSION < 104500 bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif @@ -96,7 +111,7 @@ MasterSegmentTableImpl::MasterSegmentTableImpl(int key, int size) bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); #ifdef __linux__ { - string pname = "/dev/shm/" + keyName; + string pname = shmLocation + keyName; chmod(pname.c_str(), 0666); } #endif From 662604553538795f9a03e1167c7b44376349a56a Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 2 Oct 2018 16:33:06 -0500 Subject: [PATCH 044/127] WIP. First cut of excising OAM and root access from the dev process. This passes most tests; need to make sure that the tests that fail have nothing to do with this change. DMLProc is doing something that results in a sudo password prompt. Will obliterate that next. --- config.h.cmake | 3 +++ configureEngine.cmake | 3 +++ exemgr/main.cpp | 10 ++++++++++ oam/oamcpp/liboamcpp.cpp | 1 + oam/oamcpp/oamcache.cpp | 4 +++- 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/config.h.cmake b/config.h.cmake index e320eb82c..74d707b11 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -2,6 +2,9 @@ #ifndef TEST_CONFIG_H #define TEST_CONFIG_H +/* Define to 1 to let the system come up without using OAM */ +#cmakedefine SKIP_OAM_INIT 1 + /* Define to 1 if you have the `alarm' function. */ #cmakedefine HAVE_ALARM 1 diff --git a/configureEngine.cmake b/configureEngine.cmake index 0a299c146..b06746792 100644 --- a/configureEngine.cmake +++ b/configureEngine.cmake @@ -17,6 +17,9 @@ INCLUDE (CheckSymbolExists) INCLUDE (CheckCXXSymbolExists) INCLUDE (CheckTypeSize) +IF($ENV{SKIP_OAM_INIT}) + SET(SKIP_OAM_INIT 1) +ENDIF() CHECK_INCLUDE_FILE_CXX (alloca.h HAVE_ALLOCA_H) CHECK_INCLUDE_FILE_CXX (arpa/inet.h HAVE_ARPA_INET_H) CHECK_INCLUDE_FILE_CXX (dlfcn.h HAVE_DLFCN_H) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 8742188a0..4e3b42347 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -63,6 +63,7 @@ using namespace std; #include using namespace boost; +#include "config.h" #include "configcpp.h" using namespace config; #include "messagequeue.h" @@ -100,6 +101,10 @@ using namespace querytele; #include "threadpool.h" #include "crashtrace.h" +#if defined(SKIP_OAM_INIT) +#include "dbrm.h" +#endif + namespace { @@ -1601,6 +1606,11 @@ int main(int argc, char* argv[]) { } } +#if defined(SKIP_OAM_INIT) + BRM::DBRM *dbrm = new BRM::DBRM(); + dbrm->setSystemQueryReady(true); + delete dbrm; +#endif threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); exeMgrThreadPool.setName("ExeMgrServer"); diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index d53dd66de..bba892b0d 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -56,6 +56,7 @@ #ifdef _MSC_VER #include "idbregistry.h" #endif +#include "config.h" #include "installdir.h" #include "dbrm.h" #include "sessionmanager.h" diff --git a/oam/oamcpp/oamcache.cpp b/oam/oamcpp/oamcache.cpp index 084f88157..fcda0aa4c 100644 --- a/oam/oamcpp/oamcache.cpp +++ b/oam/oamcpp/oamcache.cpp @@ -34,6 +34,7 @@ using namespace boost; #include "exceptclasses.h" #include "configcpp.h" #include "installdir.h" +#include "config.h" namespace { @@ -83,7 +84,7 @@ void OamCache::checkReload() for (uint32_t i = 0; i < dbroots.size(); i++) { oam.getDbrootPmConfig(dbroots[i], temp); - //cout << " dbroot " << dbroots[i] << " -> PM " << temp << endl; + cout << " dbroot " << dbroots[i] << " -> PM " << temp << endl; (*dbRootPMMap)[dbroots[i]] = temp; } @@ -180,6 +181,7 @@ void OamCache::checkReload() } } #else + pmToConnectionMap[*it] = i++; moduleIds.push_back(*it); #endif it++; From e47e784c53705463696916f5c8dae1c014732f77 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 3 Oct 2018 12:13:23 -0500 Subject: [PATCH 045/127] Moved the SKIP_OAM_INIT check for cleanliness, added 'config.h' to other places that need it. --- configureEngine.cmake | 7 +++---- writeengine/redistribute/we_redistributecontrolthread.cpp | 1 + writeengine/splitter/we_sdhandler.cpp | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/configureEngine.cmake b/configureEngine.cmake index b06746792..a3ac9d3c1 100644 --- a/configureEngine.cmake +++ b/configureEngine.cmake @@ -17,9 +17,6 @@ INCLUDE (CheckSymbolExists) INCLUDE (CheckCXXSymbolExists) INCLUDE (CheckTypeSize) -IF($ENV{SKIP_OAM_INIT}) - SET(SKIP_OAM_INIT 1) -ENDIF() CHECK_INCLUDE_FILE_CXX (alloca.h HAVE_ALLOCA_H) CHECK_INCLUDE_FILE_CXX (arpa/inet.h HAVE_ARPA_INET_H) CHECK_INCLUDE_FILE_CXX (dlfcn.h HAVE_DLFCN_H) @@ -719,7 +716,9 @@ IF (NOT INLINE) SET (inline "") ENDIF() - +IF($ENV{SKIP_OAM_INIT}) + SET(SKIP_OAM_INIT 1) +ENDIF() EXECUTE_PROCESS( COMMAND rm -f conftest.data conftest.file conftest.sym diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index ab2897834..70b5d1e2d 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -37,6 +37,7 @@ using namespace std; #include "boost/filesystem/operations.hpp" using namespace boost; +#include "config.h" #include "installdir.h" #include "configcpp.h" diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 61fe45239..647a90e89 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -43,6 +43,7 @@ using namespace std; #include using namespace boost; +#include "config.h" #include "configcpp.h" using namespace config; From faaee9141af019363cbe207e4cdbe01e01493d0f Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 3 Oct 2018 13:03:02 -0500 Subject: [PATCH 046/127] Commented a debugging printout. --- oam/oamcpp/oamcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/oamcpp/oamcache.cpp b/oam/oamcpp/oamcache.cpp index fcda0aa4c..d8035bbf3 100644 --- a/oam/oamcpp/oamcache.cpp +++ b/oam/oamcpp/oamcache.cpp @@ -84,7 +84,7 @@ void OamCache::checkReload() for (uint32_t i = 0; i < dbroots.size(); i++) { oam.getDbrootPmConfig(dbroots[i], temp); - cout << " dbroot " << dbroots[i] << " -> PM " << temp << endl; + //cout << " dbroot " << dbroots[i] << " -> PM " << temp << endl; (*dbRootPMMap)[dbroots[i]] = temp; } From 7aff68969ea3919df5a5be32a81f27fa4a0d263c Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 3 Oct 2018 14:09:48 -0500 Subject: [PATCH 047/127] MCOL-520 - back out nonroot shm changes --- oam/install_scripts/post-install | 6 ------ procmon/main.cpp | 2 -- utils/rwlock/rwlock.cpp | 2 -- versioning/BRM/brmshmimpl.cpp | 2 -- versioning/BRM/mastersegmenttable.cpp | 2 -- 5 files changed, 14 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 98a7e4a02..38e21cb8b 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -191,12 +191,6 @@ mkdir -p $hdfsDir >/dev/null 2>&1 #create mount directories mkdir /mnt/tmp > /dev/null 2>&1 -#create shared memory if non-root -if [ $user != "root" ]; then - mkdir -p ${installdir}/dev/shm - chmod 666 ${installdir}/dev/shm -fi - # remove mysql archive log test -d $installdir/mysql/db || mkdir -p $installdir/mysql/db rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1 diff --git a/procmon/main.cpp b/procmon/main.cpp index 688424f1e..b70bb1552 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -1888,8 +1888,6 @@ static void statusControlThread() // string shmLocation = "/dev/shm/"; - if ( !rootUser) - shmLocation = startup::StartUp::installDir() + "/dev/shm/"; PROCSTATshmsize = MAX_PROCESS * sizeof(shmProcessStatus); bool memInit = true; diff --git a/utils/rwlock/rwlock.cpp b/utils/rwlock/rwlock.cpp index 4ffec938f..bba7f2505 100644 --- a/utils/rwlock/rwlock.cpp +++ b/utils/rwlock/rwlock.cpp @@ -153,8 +153,6 @@ RWLockShmImpl::RWLockShmImpl(int key, bool excl) rootUser = false; string shmLocation = "/dev/shm/"; - if ( !rootUser) - shmLocation = startup::StartUp::installDir() + "/dev/shm/"; try { diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index 38584760e..6b86d2d37 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -56,8 +56,6 @@ BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : rootUser = false; string shmLocation = "/dev/shm/"; - if ( !rootUser) - shmLocation = startup::StartUp::installDir() + "/dev/shm/"; if (fSize == 0) { diff --git a/versioning/BRM/mastersegmenttable.cpp b/versioning/BRM/mastersegmenttable.cpp index 0abf81c0c..34fca9451 100644 --- a/versioning/BRM/mastersegmenttable.cpp +++ b/versioning/BRM/mastersegmenttable.cpp @@ -85,8 +85,6 @@ MasterSegmentTableImpl::MasterSegmentTableImpl(int key, int size) rootUser = false; string shmLocation = "/dev/shm/"; - if ( !rootUser) - shmLocation = startup::StartUp::installDir() + "/dev/shm/"; try { From bc19fb2f2d4d42717cb88ec89cd026937d867b61 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 4 Oct 2018 08:47:14 -0500 Subject: [PATCH 048/127] MCOL-520. Removed the chmod 666 calls on our shared mem objects. --- procmon/main.cpp | 40 +++++----------------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/procmon/main.cpp b/procmon/main.cpp index b70bb1552..6c0161176 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -138,7 +138,7 @@ int main(int argc, char** argv) string systemLang = "C"; setlocale(LC_ALL, systemLang.c_str()); - + //get tmp log directory tmpLogDir = startup::StartUp::tmpDir(); @@ -345,7 +345,7 @@ int main(int argc, char** argv) //Set the alarm // aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET); // sleep (1); - + string cmd = startup::StartUp::installDir() + "/bin/infinidb stop > /dev/null 2>&1"; system(cmd.c_str()); @@ -1886,9 +1886,9 @@ static void statusControlThread() // //Allocate Shared Memory for storing Process Status Data // - + string shmLocation = "/dev/shm/"; - + PROCSTATshmsize = MAX_PROCESS * sizeof(shmProcessStatus); bool memInit = true; #if 0 @@ -1915,12 +1915,6 @@ static void statusControlThread() { #if BOOST_VERSION < 104500 bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif #else bi::permissions perms; perms.set_unrestricted(); @@ -1993,12 +1987,6 @@ static void statusControlThread() try { bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif shm.truncate(SYSTEMSTATshmsize); fSysStatShmobj.swap(shm); } @@ -2112,12 +2100,6 @@ static void statusControlThread() try { bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif shm.truncate(NICSTATshmsize); fNICStatShmobj.swap(shm); } @@ -2210,12 +2192,6 @@ static void statusControlThread() try { bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif shm.truncate(EXTDEVICESTATshmsize); fExtStatShmobj.swap(shm); } @@ -2300,12 +2276,6 @@ static void statusControlThread() try { bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif shm.truncate(DBROOTSTATshmsize); fDbrootShmobj.swap(shm); } @@ -2709,7 +2679,7 @@ void processStatusMSG(messageqcpp::IOSocket* cfIos) memcpy(fShmSystemStatus[0].StateChangeDate, oam.getCurrentTime().c_str(), DATESIZE); log.writeLog(__LINE__, "statusControl: REQUEST RECEIVED: Set System State = " + oamState[state], LOG_TYPE_DEBUG); } - + BRM::DBRM dbrm; dbrm.setSystemQueryReady(true); } From 1b1026bbe4bde1d6df9b1ab66b63104ec8e6afb5 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 4 Oct 2018 10:30:30 -0500 Subject: [PATCH 049/127] MCOL-520. Fixed shm usage in procmon. Untested. --- procmon/main.cpp | 173 ++++++++++++----------------------------------- 1 file changed, 43 insertions(+), 130 deletions(-) diff --git a/procmon/main.cpp b/procmon/main.cpp index 6c0161176..9c2bdfff5 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -68,6 +68,41 @@ extern bool gOAMParentModuleFlag; pthread_mutex_t STATUS_LOCK; +bool getshm(const string &name, int size, bi::shared_memory_object &target) { + MonitorLog log; + bool created = false; + try + { + bi::shared_memory_object shm(bi::create_only, name.c_str(), bi::read_write); + created = true; + shm.truncate(size); + target.swap(shm); + } + catch (bi::interprocess_exception& biex) + { + if (biex.get_error_code() == bi::already_exists_error) { + try { + bi::shared_memory_object shm(bi::open_only, name.c_str(), bi::read_write); + target.swap(shm); + } + catch (exception &e) { + ostringstream os; + os << "ProcMon failed to attach to the " << name << " shared mem segment, got " << e.what(); + log.writeLog(__LINE__, os.str(), LOG_TYPE_CRITICAL); + exit(1); + } + } + else { + ostringstream os; + os << "ProcMon failed to create the 'proc stat' shared mem segment, got " << biex.what(); + log.writeLog(__LINE__, os.str(), LOG_TYPE_CRITICAL); + exit(1); + } + } + return created; +} + + /****************************************************************************************** * @brief main * @@ -1910,29 +1945,7 @@ static void statusControlThread() fShmProcessStatus = static_cast(shmat(shmid, NULL, 0)); #endif string keyName = BRM::ShmKeys::keyToName(fShmKeys.PROCESSSTATUS_SYSVKEY); - - try - { -#if BOOST_VERSION < 104500 - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#else - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); -#endif - shm.truncate(PROCSTATshmsize); - fProcStatShmobj.swap(shm); - } - catch (bi::interprocess_exception& biex) - { - memInit = false; - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - fProcStatShmobj.swap(shm); - } - catch (...) - { - throw; - } + memInit = getshm(keyName, PROCSTATshmsize, fProcStatShmobj); bi::mapped_region region(fProcStatShmobj, bi::read_write); fProcStatMapreg.swap(region); @@ -1983,23 +1996,7 @@ static void statusControlThread() fShmSystemStatus = static_cast(shmat(shmid, NULL, 0)); #endif keyName = BRM::ShmKeys::keyToName(fShmKeys.SYSTEMSTATUS_SYSVKEY); - - try - { - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); - shm.truncate(SYSTEMSTATshmsize); - fSysStatShmobj.swap(shm); - } - catch (bi::interprocess_exception& biex) - { - memInit = false; - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - fSysStatShmobj.swap(shm); - } - catch (...) - { - throw; - } + memInit = getshm(keyName, SYSTEMSTATshmsize, fSysStatShmobj); bi::mapped_region region2(fSysStatShmobj, bi::read_write); fSysStatMapreg.swap(region2); @@ -2076,43 +2073,9 @@ static void statusControlThread() // boost::interprocess::shared_memory_object fNICStatShmobj; static const int NICSTATshmsize = (MAX_MODULE * MAX_NIC) * sizeof(shmDeviceStatus); - memInit = true; -#if 0 - shmid = shmget(fShmKeys.NICSTATUS_SYSVKEY, NICSTATshmsize, IPC_EXCL | IPC_CREAT | 0666); - if (shmid == -1) - { - // table already exist - memInit = false; - shmid = shmget(fShmKeys.NICSTATUS_SYSVKEY, NICSTATshmsize, 0666); - - if (shmid == -1) - { - log.writeLog(__LINE__, "*****NICStatusTable shmget failed.", LOG_TYPE_ERROR); - exit(1); - } - } - - fShmNICStatus = static_cast(shmat(shmid, NULL, 0)); -#endif keyName = BRM::ShmKeys::keyToName(fShmKeys.NICSTATUS_SYSVKEY); - - try - { - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); - shm.truncate(NICSTATshmsize); - fNICStatShmobj.swap(shm); - } - catch (bi::interprocess_exception& biex) - { - memInit = false; - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - fNICStatShmobj.swap(shm); - } - catch (...) - { - throw; - } + memInit = getshm(keyName, NICSTATshmsize, fNICStatShmobj); bi::mapped_region fNICStatMapreg(fNICStatShmobj, bi::read_write); fShmNICStatus = static_cast(fNICStatMapreg.get_address()); @@ -2168,43 +2131,8 @@ static void statusControlThread() boost::interprocess::shared_memory_object fExtStatShmobj; static const int EXTDEVICESTATshmsize = MAX_EXT_DEVICE * sizeof(shmDeviceStatus); - memInit = true; -#if 0 - shmid = shmget(fShmKeys.SWITCHSTATUS_SYSVKEY, EXTDEVICESTATshmsize, IPC_EXCL | IPC_CREAT | 0666); - - if (shmid == -1) - { - // table already exist - memInit = false; - shmid = shmget(fShmKeys.SWITCHSTATUS_SYSVKEY, EXTDEVICESTATshmsize, 0666); - - if (shmid == -1) - { - log.writeLog(__LINE__, "*****ExtDeviceStatusTable shmget failed.", LOG_TYPE_ERROR); - exit(1); - } - } - - fShmExtDeviceStatus = static_cast(shmat(shmid, NULL, 0)); -#endif keyName = BRM::ShmKeys::keyToName(fShmKeys.SWITCHSTATUS_SYSVKEY); - - try - { - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); - shm.truncate(EXTDEVICESTATshmsize); - fExtStatShmobj.swap(shm); - } - catch (bi::interprocess_exception& biex) - { - memInit = false; - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - fExtStatShmobj.swap(shm); - } - catch (...) - { - throw; - } + memInit = getshm(keyName, EXTDEVICESTATshmsize, fExtStatShmobj); bi::mapped_region fExtStatMapreg(fExtStatShmobj, bi::read_write); fShmExtDeviceStatus = static_cast(fExtStatMapreg.get_address()); @@ -2270,25 +2198,8 @@ static void statusControlThread() boost::interprocess::shared_memory_object fDbrootShmobj; static const int DBROOTSTATshmsize = MAX_DBROOT * sizeof(shmDeviceStatus); - memInit = true; keyName = BRM::ShmKeys::keyToName(fShmKeys.DBROOTSTATUS_SYSVKEY); - - try - { - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); - shm.truncate(DBROOTSTATshmsize); - fDbrootShmobj.swap(shm); - } - catch (bi::interprocess_exception& biex) - { - memInit = false; - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - fDbrootShmobj.swap(shm); - } - catch (...) - { - throw; - } + memInit = getshm(keyName, DBROOTSTATshmsize, fDbrootShmobj); bi::mapped_region fdDbrootStatMapreg(fDbrootShmobj, bi::read_write); fShmDbrootStatus = static_cast(fdDbrootStatMapreg.get_address()); @@ -2376,6 +2287,7 @@ static void statusControlThread() log.writeLog(__LINE__, "statusControlThread Thread reading " + portName + " port", LOG_TYPE_DEBUG); } + fIos = NULL; try { //log.writeLog(__LINE__, "***before accept", LOG_TYPE_DEBUG); @@ -2402,7 +2314,8 @@ static void statusControlThread() } catch (...) { - delete fIos; + if (fIos) + delete fIos; } if ( runStandby ) From f514ed6775063e1b8da057bd8ab197babc3acbda Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 4 Oct 2018 11:11:35 -0500 Subject: [PATCH 050/127] MCOL-520. Got rid of the chmods on shmsegs & added better error msgs. Untested. --- versioning/BRM/brmshmimpl.cpp | 49 ++++++++++++++--------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index 6b86d2d37..60bb66e31 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -45,16 +45,6 @@ BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : fKey(key), fSize(size), fReadOnly(readOnly) { string keyName = ShmKeys::keyToName(fKey); - - bool rootUser = true; - - //check if root-user - int user; - user = getuid(); - - if (user != 0) - rootUser = false; - string shmLocation = "/dev/shm/"; if (fSize == 0) @@ -98,43 +88,42 @@ again: try { -#if BOOST_VERSION < 104500 bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = "shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif -#else - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); -#endif idbassert(fSize > 0); shm.truncate(fSize); fShmobj.swap(shm); } - catch (bi::interprocess_exception&) + catch (bi::interprocess_exception &b) { - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); + if (b.get_error_code() != bi::already_exists_error) { + ostringstream o; + o << "BRM caught an exception creating a shared memory segment: " << b.what(); + log(o.str()); + throw; + } + bi::shared_memory_object *shm = NULL; + try { + shm = new bi::shared_memory_object(bi::open_only, keyName.c_str(), bi::read_write); + } + catch (exception &e) { + ostringstream o; + o << "BRM caught an exception attaching to a shared memory segment: " << b.what(); + log(o.str()); + throw; + } off_t curSize = 0; #ifdef _MSC_VER bi::offset_t tmp = 0; shm.get_size(tmp); curSize = static_cast(tmp); #else - shm.get_size(curSize); + shm->get_size(curSize); #endif idbassert(curSize > 0); idbassert(curSize >= fSize); - fShmobj.swap(shm); + fShmobj.swap(*shm); fSize = curSize; } - catch (...) - { - throw; - } if (fReadOnly) { From a35009c8ee6e8fdd49ce13ada925f4108ca6d447 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 4 Oct 2018 11:27:08 -0500 Subject: [PATCH 051/127] MCOL-520. Removed chmod stuff from MST and did a little cleaning. --- versioning/BRM/brmshmimpl.cpp | 27 +------------- versioning/BRM/mastersegmenttable.cpp | 51 +++++++++------------------ 2 files changed, 17 insertions(+), 61 deletions(-) diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index 60bb66e31..504450c99 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -45,7 +45,6 @@ BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : fKey(key), fSize(size), fReadOnly(readOnly) { string keyName = ShmKeys::keyToName(fKey); - string shmLocation = "/dev/shm/"; if (fSize == 0) { @@ -107,7 +106,7 @@ again: } catch (exception &e) { ostringstream o; - o << "BRM caught an exception attaching to a shared memory segment: " << b.what(); + o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << b.what(); log(o.str()); throw; } @@ -145,19 +144,7 @@ int BRMShmImpl::grow(unsigned newKey, off_t newSize) string oldName = fShmobj.get_name(); string keyName = ShmKeys::keyToName(newKey); -#if BOOST_VERSION < 104500 bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif -#else - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); -#endif shm.truncate(newSize); bi::mapped_region region(shm, bi::read_write); @@ -192,19 +179,7 @@ int BRMShmImpl::clear(unsigned newKey, off_t newSize) string oldName = fShmobj.get_name(); string keyName = ShmKeys::keyToName(newKey); -#if BOOST_VERSION < 104500 bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif -#else - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); -#endif shm.truncate(newSize); bi::mapped_region region(shm, bi::read_write); diff --git a/versioning/BRM/mastersegmenttable.cpp b/versioning/BRM/mastersegmenttable.cpp index 34fca9451..9153c112d 100644 --- a/versioning/BRM/mastersegmenttable.cpp +++ b/versioning/BRM/mastersegmenttable.cpp @@ -75,51 +75,32 @@ MasterSegmentTableImpl::MasterSegmentTableImpl(int key, int size) { string keyName = ShmKeys::keyToName(key); - bool rootUser = true; - - //check if root-user - int user; - user = getuid(); - - if (user != 0) - rootUser = false; - - string shmLocation = "/dev/shm/"; - try { -#if BOOST_VERSION < 104500 bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif -#else - bi::permissions perms; - perms.set_unrestricted(); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); -#endif shm.truncate(size); fShmobj.swap(shm); } catch (bi::interprocess_exception& biex) { - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); + if (biex.get_error_code() == bi::already_exists_error) { + try { + bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); + fShmobj.swap(shm); + } + catch (exception &e) { + ostringstream o; + o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << e.what(); + throw; + } + } + else { + ostringstream o; + o << "BRM caught an exception creating a shared memory segment (" << keyName << "): " << biex.what(); + log(o.str()); + throw; } -#endif - fShmobj.swap(shm); } - catch (...) - { - throw; - } - bi::mapped_region region(fShmobj, bi::read_write); fMapreg.swap(region); } From 4215a47b5c92072de975bead4854659e8d232988 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 4 Oct 2018 14:25:01 -0500 Subject: [PATCH 052/127] MCOL-520. More untested changes. Went down the list in shmkeys.h, added sane error msgs on perms errors accessing rest of the shmsegs there. Realized we probably do need 666 to the shmsegs. Also deleted a little dead code that was copy-pasted everywhere. --- dbcon/execplan/sessionmonitor.h | 4 ---- dmlproc/dmlproc.cpp | 12 ------------ procmon/main.cpp | 4 +++- tools/clearShm/main.cpp | 2 -- utils/compress/version1.cpp | 4 ---- utils/rwlock/rwlock.cpp | 24 ------------------------ versioning/BRM/brmshmimpl.cpp | 20 +++++++++++--------- versioning/BRM/mastersegmenttable.cpp | 4 +++- versioning/BRM/shmkeys.cpp | 1 - versioning/BRM/shmkeys.h | 1 - 10 files changed, 17 insertions(+), 59 deletions(-) diff --git a/dbcon/execplan/sessionmonitor.h b/dbcon/execplan/sessionmonitor.h index 42a9ec899..cf31b63b3 100644 --- a/dbcon/execplan/sessionmonitor.h +++ b/dbcon/execplan/sessionmonitor.h @@ -32,10 +32,6 @@ #include #include -#include -#include -#include - #include "calpontsystemcatalog.h" #include "sessionmanager.h" #include "shmkeys.h" diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index dec752213..5fa99be39 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -584,18 +584,6 @@ int main(int argc, char* argv[]) if (temp > 0) serverQueueSize = temp; - - bool rootUser = true; -#ifndef _MSC_VER - //check if root-user - int user; - user = getuid(); - - if (user != 0) - rootUser = false; - -#endif - //read and cleanup port before trying to use try { diff --git a/procmon/main.cpp b/procmon/main.cpp index 9c2bdfff5..90ec33238 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -73,7 +73,9 @@ bool getshm(const string &name, int size, bi::shared_memory_object &target) { bool created = false; try { - bi::shared_memory_object shm(bi::create_only, name.c_str(), bi::read_write); + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, name.c_str(), bi::read_write, perms); created = true; shm.truncate(size); target.swap(shm); diff --git a/tools/clearShm/main.cpp b/tools/clearShm/main.cpp index 02cc0a0cf..e883bc0f6 100644 --- a/tools/clearShm/main.cpp +++ b/tools/clearShm/main.cpp @@ -220,7 +220,6 @@ int main(int argc, char** argv) shmDoit(BrmKeys.PROCESSSTATUS_SYSVKEY, "PROC_STAT "); shmDoit(BrmKeys.SYSTEMSTATUS_SYSVKEY, "SYS_STAT "); shmDoit(BrmKeys.SWITCHSTATUS_SYSVKEY, "SW_STAT "); - shmDoit(BrmKeys.STORAGESTATUS_SYSVKEY, "STORE_STAT "); shmDoit(BrmKeys.NICSTATUS_SYSVKEY, "NIC_STAT "); shmDoit(BrmKeys.DBROOTSTATUS_SYSVKEY, "DBROOT_STAT"); } @@ -239,7 +238,6 @@ int main(int argc, char** argv) semDoit(BrmKeys.PROCESSSTATUS_SYSVKEY, "PROC_STAT "); semDoit(BrmKeys.SYSTEMSTATUS_SYSVKEY, "SYS_STAT "); semDoit(BrmKeys.SWITCHSTATUS_SYSVKEY, "SW_STAT "); - semDoit(BrmKeys.STORAGESTATUS_SYSVKEY, "STORE_STAT "); semDoit(BrmKeys.NICSTATUS_SYSVKEY, "NIC_STAT "); shmDoit(BrmKeys.DBROOTSTATUS_SYSVKEY, "DBROOT_STAT"); } diff --git a/utils/compress/version1.cpp b/utils/compress/version1.cpp index 59d080022..945e2617a 100644 --- a/utils/compress/version1.cpp +++ b/utils/compress/version1.cpp @@ -188,13 +188,9 @@ void initCtlShm() } catch (bi::interprocess_exception&) { -#if BOOST_VERSION < 104500 - bi::shared_memory_object shm(bi::create_only, DecomShmName.c_str(), bi::read_write); -#else bi::permissions perms; perms.set_unrestricted(); bi::shared_memory_object shm(bi::create_only, DecomShmName.c_str(), bi::read_write, perms); -#endif shm.truncate(sizeof(CtlShmImage)); bi::mapped_region region(shm, bi::read_write); tmpptr = new (region.get_address()) CtlShmImage; diff --git a/utils/rwlock/rwlock.cpp b/utils/rwlock/rwlock.cpp index bba7f2505..abe298849 100644 --- a/utils/rwlock/rwlock.cpp +++ b/utils/rwlock/rwlock.cpp @@ -143,32 +143,11 @@ RWLockShmImpl::RWLockShmImpl(int key, bool excl) string keyName = BRM::ShmKeys::keyToName(key); fKeyString = keyName; - bool rootUser = true; - - //check if root-user - int user; - user = getuid(); - - if (user != 0) - rootUser = false; - - string shmLocation = "/dev/shm/"; - try { -#if BOOST_VERSION < 104500 - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); -#ifdef __linux__ - { - string pname = shmLocation + keyName; - chmod(pname.c_str(), 0666); - } -#endif -#else bi::permissions perms; perms.set_unrestricted(); bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); -#endif shm.truncate(sizeof(struct State)); fStateShm.swap(shm); bi::mapped_region region(fStateShm, bi::read_write); @@ -190,10 +169,7 @@ RWLockShmImpl::RWLockShmImpl(int key, bool excl) catch (bi::interprocess_exception&) { if (excl) - { - //don't think we can get here anymore... throw not_excl(); - } bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); fStateShm.swap(shm); diff --git a/versioning/BRM/brmshmimpl.cpp b/versioning/BRM/brmshmimpl.cpp index 504450c99..3bf834188 100644 --- a/versioning/BRM/brmshmimpl.cpp +++ b/versioning/BRM/brmshmimpl.cpp @@ -64,12 +64,7 @@ again: #endif if (curSize == 0) throw -#if BOOST_VERSION < 104500 - bi::interprocess_exception(); - -#else bi::interprocess_exception("shm size is zero"); -#endif } catch (bi::interprocess_exception&) { @@ -87,7 +82,9 @@ again: try { - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); idbassert(fSize > 0); shm.truncate(fSize); fShmobj.swap(shm); @@ -113,7 +110,7 @@ again: off_t curSize = 0; #ifdef _MSC_VER bi::offset_t tmp = 0; - shm.get_size(tmp); + shm->get_size(tmp); curSize = static_cast(tmp); #else shm->get_size(curSize); @@ -121,6 +118,7 @@ again: idbassert(curSize > 0); idbassert(curSize >= fSize); fShmobj.swap(*shm); + delete shm; fSize = curSize; } @@ -144,7 +142,9 @@ int BRMShmImpl::grow(unsigned newKey, off_t newSize) string oldName = fShmobj.get_name(); string keyName = ShmKeys::keyToName(newKey); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); shm.truncate(newSize); bi::mapped_region region(shm, bi::read_write); @@ -179,7 +179,9 @@ int BRMShmImpl::clear(unsigned newKey, off_t newSize) string oldName = fShmobj.get_name(); string keyName = ShmKeys::keyToName(newKey); - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); shm.truncate(newSize); bi::mapped_region region(shm, bi::read_write); diff --git a/versioning/BRM/mastersegmenttable.cpp b/versioning/BRM/mastersegmenttable.cpp index 9153c112d..a2f228d58 100644 --- a/versioning/BRM/mastersegmenttable.cpp +++ b/versioning/BRM/mastersegmenttable.cpp @@ -77,7 +77,9 @@ MasterSegmentTableImpl::MasterSegmentTableImpl(int key, int size) try { - bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write); + bi::permissions perms; + perms.set_unrestricted(); + bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms); shm.truncate(size); fShmobj.swap(shm); } diff --git a/versioning/BRM/shmkeys.cpp b/versioning/BRM/shmkeys.cpp index fcdf7c999..6abe149ac 100644 --- a/versioning/BRM/shmkeys.cpp +++ b/versioning/BRM/shmkeys.cpp @@ -55,7 +55,6 @@ ShmKeys::ShmKeys() PROCESSSTATUS_SYSVKEY = 0xfd000000 | BRM_UID; SYSTEMSTATUS_SYSVKEY = 0xfc000000 | BRM_UID; SWITCHSTATUS_SYSVKEY = 0xfb000000 | BRM_UID; - STORAGESTATUS_SYSVKEY = 0xfa000000 | BRM_UID; NICSTATUS_SYSVKEY = 0xf9000000 | BRM_UID; DBROOTSTATUS_SYSVKEY = 0xf8000000 | BRM_UID; DECOMSVRMUTEX_SYSVKEY = 0xf7000000 | BRM_UID; diff --git a/versioning/BRM/shmkeys.h b/versioning/BRM/shmkeys.h index bfb2cba0f..62dd67866 100644 --- a/versioning/BRM/shmkeys.h +++ b/versioning/BRM/shmkeys.h @@ -61,7 +61,6 @@ public: uint32_t PROCESSSTATUS_SYSVKEY; uint32_t SYSTEMSTATUS_SYSVKEY; uint32_t SWITCHSTATUS_SYSVKEY; - uint32_t STORAGESTATUS_SYSVKEY; uint32_t NICSTATUS_SYSVKEY; uint32_t DBROOTSTATUS_SYSVKEY; uint32_t DECOMSVRMUTEX_SYSVKEY; From f54cc0803ad3686f2587bd002b9e036f5c78c1cd Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 4 Oct 2018 16:17:28 -0500 Subject: [PATCH 053/127] MCOL-520. Made the error msgs users will hit more informative. --- procmon/main.cpp | 3 ++- utils/rwlock/rwlock.cpp | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/procmon/main.cpp b/procmon/main.cpp index 90ec33238..fce730c35 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -96,7 +96,8 @@ bool getshm(const string &name, int size, bi::shared_memory_object &target) { } else { ostringstream os; - os << "ProcMon failed to create the 'proc stat' shared mem segment, got " << biex.what(); + os << "ProcMon failed to create the '" << name << "' shared mem segment, got " << biex.what() << "."; + os << " Check the permissions on /dev/shm; should be 1777"; log.writeLog(__LINE__, os.str(), LOG_TYPE_CRITICAL); exit(1); } diff --git a/utils/rwlock/rwlock.cpp b/utils/rwlock/rwlock.cpp index abe298849..d042b905e 100644 --- a/utils/rwlock/rwlock.cpp +++ b/utils/rwlock/rwlock.cpp @@ -166,13 +166,25 @@ RWLockShmImpl::RWLockShmImpl(int key, bool excl) new (&fState->sems[RWLock::READERS]) bi::interprocess_semaphore(0); new (&fState->sems[RWLock::WRITERS]) bi::interprocess_semaphore(0); } - catch (bi::interprocess_exception&) + catch (bi::interprocess_exception &e) { - if (excl) + if (e.get_error_code() == bi::security_error) { + cerr << "RWLock: Failed to create the lock. Check perms on /dev/shm; should be 1777" << endl; + throw; + } + if (e.get_error_code() == bi::already_exists_error && excl) throw not_excl(); + if (e.get_error_code() != bi::already_exists_error) + throw; - bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); - fStateShm.swap(shm); + try { + bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write); + fStateShm.swap(shm); + } + catch (exception &e) { + cerr << "RWLock failed to attach to the " << keyName << " shared mem segment, got " << e.what() << endl; + throw; + } bi::mapped_region region(fStateShm, bi::read_write); fRegion.swap(region); fState = static_cast(fRegion.get_address()); From d196b06ae6323c65e538603514d19f999f97de21 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 5 Oct 2018 13:07:24 -0500 Subject: [PATCH 054/127] MCOL-520 - removed /etc/defaults/columnstore reference --- oam/install_scripts/post-install | 3 --- 1 file changed, 3 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 38e21cb8b..640c34efd 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -255,11 +255,8 @@ if [ $user = "root" ]; then cat $tmpDir/syslog_install.log fi else - sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstore.def sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstoreLogRotate - cp $installdir/bin/columnstore.def /etc/default/columnstore - sed -i -e s@prefix=/home/guest@prefix=$prefix@g $installdir/bin/* chown $user:$user $installdir/etc/Columnstore.xml From 29046b20da8ef509c2683f851c217fa1ed85865b Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 5 Oct 2018 13:42:15 -0500 Subject: [PATCH 055/127] MCOL-520. Added a test for /dev/shm access in post-install. --- oam/install_scripts/post-install | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 38e21cb8b..11727b39a 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -66,6 +66,12 @@ fi cd $installdir/lib || exit 1 +touch /dev/shm/columnstore-test && rm /dev/shm/columnstore-test +if [ $? -ne 0 ] ; then + echo "User $user will need R/W access to /dev/shm." + exit 1 +fi + #remove libudfsdk.so.1.0.0 file, if it exist # mcol-875 rm -f $installdir/lib/libudfsdk.so rm -f $installdir/lib/libudfsdk.so.1 From c798d464e063dca9e2c0ca2b4dcee1681af22259 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 8 Oct 2018 13:20:00 -0500 Subject: [PATCH 056/127] MCOL-520 --- oam/install_scripts/columnstore | 1 + oam/install_scripts/post-install | 1 + oamapps/postConfigure/postConfigure.cpp | 2 +- procmon/main.cpp | 3 +++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index 38b6bc15c..c7adef015 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -60,6 +60,7 @@ lockdir=`$InstallDir/bin/getConfig Installation LockFileDirectory` #get temp directory tmpDir=`$InstallDir/bin/getConfig SystemConfig SystemTempFileDir` +mkdir $tmpDir >/dev/null 2>&1 checkInstallSetup() { InitialInstallFlag=`$InstallDir/bin/getConfig -c $InstallDir/etc/Columnstore.xml Installation InitialInstallFlag` diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 3e2aec4dc..01422dc8e 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -182,6 +182,7 @@ else #get columnstore temp file directory name TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` tmpDir=${tmpDir}${TempFileDir} + mkdir $tmpDir >/dev/null 2>&1 fi #set base columnstore temp file directory diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 31239341f..b2268b36f 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -866,7 +866,7 @@ int main(int argc, char* argv[]) try { - sysConfig->setConfig(InstallSection, "SingleServerInstall", "n"); + sysConfig->setConfig(InstallSection, "SingleServerInstall", "y"); sysConfig->setConfig(InstallSection, "ServerTypeInstall", "2"); } catch (...) diff --git a/procmon/main.cpp b/procmon/main.cpp index fce730c35..e26d00d70 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -179,6 +179,9 @@ int main(int argc, char** argv) //get tmp log directory tmpLogDir = startup::StartUp::tmpDir(); + + string cmd = "mkdir -p tmpLogDir"; + system(cmd.c_str()); // create message thread pthread_t MessageThread; From f48b92e46f00f09abb46f606e2e49d117ca994ca Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 8 Oct 2018 13:46:49 -0500 Subject: [PATCH 057/127] MCOL-520 --- procmon/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmon/main.cpp b/procmon/main.cpp index e26d00d70..dac166bce 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -180,7 +180,7 @@ int main(int argc, char** argv) //get tmp log directory tmpLogDir = startup::StartUp::tmpDir(); - string cmd = "mkdir -p tmpLogDir"; + string cmd = "mkdir -p " + tmpLogDir; system(cmd.c_str()); // create message thread From d9aab31afdc6ec82276543116e19c91298e9e1db Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 8 Oct 2018 15:32:10 -0500 Subject: [PATCH 058/127] MCOL-520. Fixed some stuff our older compilers don't like. --- oam/oamcpp/liboamcpp.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 73bf5c9d2..c87066c39 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -150,7 +150,7 @@ Oam::Oam() } catch (...) {} // defaulted to false } - + //get user string USER = "root"; char* p = getenv("USER"); @@ -162,7 +162,7 @@ Oam::Oam() if ( USER != "root") userDir = "home/" + USER; - + tmpdir = startup::StartUp::tmpDir(); } @@ -7871,7 +7871,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) //get pid cmd = "cat " + InstallDir + "/mysql/db/*.pid > " + pidtmp; system(cmd.c_str()); - ifstream oldFile (pidtmp); + ifstream oldFile(pidtmp.c_str()); //fail if file size 0 oldFile.seekg(0, std::ios::end); @@ -7930,7 +7930,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) //get pid cmd = "cat " + InstallDir + "/mysql/db/*.pid > " + pidtmp; system(cmd.c_str()); - ifstream oldFile (pidtmp); + ifstream oldFile(pidtmp.c_str()); char line[400]; string pid; @@ -7957,7 +7957,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) //check if pid has changed cmd = "cat " + InstallDir + "/mysql/db/*.pid > " + pidtmp; system(cmd.c_str()); - ifstream oldFile (pidtmp); + ifstream oldFile(pidtmp.c_str()); char line[400]; string pid; @@ -10886,17 +10886,17 @@ bool Oam::checkSystemRunning() struct stat st; string lockFileDir = "/var/subsys/lock"; - + try { Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); lockFileDir = sysConfig->getConfig("Installation", "LockFileDirectory"); } - catch (...) + catch (...) {} // defaulted to false string lockFile = lockFileDir + "/columnstore"; - + if (stat(lockFile.c_str(), &st) == 0) { return true; From 6a353b40b59f87e9b7b8244893e6e554586fdae8 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 10 Oct 2018 14:58:18 -0500 Subject: [PATCH 059/127] MCOL-520, missed a line to log an error in the new shm error handling. --- versioning/BRM/mastersegmenttable.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/versioning/BRM/mastersegmenttable.cpp b/versioning/BRM/mastersegmenttable.cpp index a2f228d58..047c647f8 100644 --- a/versioning/BRM/mastersegmenttable.cpp +++ b/versioning/BRM/mastersegmenttable.cpp @@ -93,6 +93,7 @@ MasterSegmentTableImpl::MasterSegmentTableImpl(int key, int size) catch (exception &e) { ostringstream o; o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << e.what(); + log(o.str()); throw; } } From f3ce8458423329756edbb66b81276e15fdef4d3a Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 10 Oct 2018 14:58:50 -0500 Subject: [PATCH 060/127] MCOL-520. Fixed several instances where we were using a too-new contructor to ifstream. --- dbcon/mysql/ha_calpont_impl.cpp | 6 +++--- procmgr/processmanager.cpp | 30 +++++++++++++++--------------- procmon/processmonitor.cpp | 24 ++++++++++++------------ 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 42eb9adb7..2b854e3a6 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -3058,7 +3058,7 @@ int ha_calpont_impl_rnd_init(TABLE* table) // plan serialization string tmpDir = aTmpDir + "/li1-plan.hex"; - ifstream ifs(tmpDir); + ifstream ifs(tmpDir.c_str()); ByteStream bs1; ifs >> bs1; ifs.close(); @@ -5121,7 +5121,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE ci->warningMsg = msg; } - // If the previous query has error and + // If the previous query has error and // this is not a subquery run by the server(MCOL-1601) // re-establish the connection if (ci->queryState != 0) @@ -5221,7 +5221,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE return 0; string query; - // Set the query text only once if the server executes + // Set the query text only once if the server executes // subqueries separately. if(ci->queryState) query.assign(""); diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 45df060fb..424da5605 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -674,8 +674,8 @@ void processMSG(messageqcpp::IOSocket* cfIos) processManager.recycleProcess(target, true); //distribute config file - processManager.distributeConfigFile("system"); - + processManager.distributeConfigFile("system"); + //set query system state ready processManager.setQuerySystemState(true); @@ -917,7 +917,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) //check for SIMPLEX Processes on mate might need to be started processManager.checkSimplexModule(moduleName); - + processManager.setSystemState(oam::ACTIVE); //set query system state ready @@ -1016,7 +1016,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) { processManager.stopModule(moduleName, graceful, manualFlag); log.writeLog(__LINE__, "stop Module Completed on " + moduleName, LOG_TYPE_INFO); - + status = processManager.enableModule(moduleName, oam::MAN_OFFLINE); log.writeLog(__LINE__, "Enable Module Completed on " + moduleName, LOG_TYPE_INFO); } @@ -1783,7 +1783,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) pthread_join(startsystemthread, NULL); status = startsystemthreadStatus; } - + // setup MySQL Replication after FORCE restart command if ( (status == API_SUCCESS) && (graceful == oam::FORCEFUL) ) @@ -3762,7 +3762,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule) stopProcessType("WriteEngineServer"); stopProcessType("ExeMgr"); - + stopProcessType("PrimProc"); stopProcessType("DBRMControllerNode"); @@ -3774,13 +3774,13 @@ void ProcessManager::recycleProcess(string module, bool enableModule) stopProcessType("mysqld"); // restartProcessType("mysqld"); - + startProcessType("DBRMControllerNode"); startProcessType("DBRMWorkerNode"); startProcessType("PrimProc"); sleep(5); - + startProcessType("WriteEngineServer"); sleep(3); @@ -3840,7 +3840,7 @@ int ProcessManager::enableModule(string target, int state, bool failover) if ( newStandbyModule == target) setStandbyModule(newStandbyModule); - + log.writeLog(__LINE__, "enableModule request for " + target + " completed", LOG_TYPE_DEBUG); return API_SUCCESS; @@ -4152,7 +4152,7 @@ void ProcessManager::setSystemState(uint16_t state) if( state == oam::ACTIVE ) { //set query system states ready processManager.setQuerySystemState(true); - + //clear alarms if set aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_AUTO, CLEAR); aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_MANUAL, CLEAR); @@ -4694,7 +4694,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski if ( systemprocessstatus.processstatus[i].ProcessOpState != oam::ACTIVE ) continue; - + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { string procModuleType = systemprocessstatus.processstatus[i].Module.substr(0, MAX_MODULE_TYPE_SIZE); @@ -7722,7 +7722,7 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist) // wait some more sleep(2); } - + if ( rtn = oam::ACTIVE ) //set query system state not ready processManager.setQuerySystemState(true); @@ -8265,7 +8265,7 @@ void ProcessManager::checkSimplexModule(std::string moduleName) if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc") { setPMProcIPs((*pt).DeviceName); - + log.writeLog(__LINE__, "Set Primary UM Module = " + (*pt).DeviceName, LOG_TYPE_DEBUG); oam.setSystemConfig("PrimaryUMModuleName", (*pt).DeviceName); @@ -8284,7 +8284,7 @@ void ProcessManager::checkSimplexModule(std::string moduleName) log.writeLog(__LINE__, "checkSimplexModule: mate process started: " + (*pt).DeviceName + "/" + systemprocessconfig.processconfig[j].ProcessName, LOG_TYPE_DEBUG); status = processManager.startProcess((*pt).DeviceName, - "DMLProc", + "DMLProc", FORCEFUL); if ( status == API_SUCCESS ) { log.writeLog(__LINE__, "checkSimplexModule: mate process started: " + (*pt).DeviceName + "/DMLProc", LOG_TYPE_DEBUG); @@ -11114,7 +11114,7 @@ int ProcessManager::mountDBRoot(std::string dbrootID) system(cmd.c_str()); } - ifstream in(tmpMount); + ifstream in(tmpMount.c_str()); in.seekg(0, std::ios::end); int size = in.tellg(); diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index a406b3f1e..a60c503b5 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1057,7 +1057,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO msg >> manualFlag; int requestStatus = oam::API_SUCCESS; log.writeLog(__LINE__, "MSG RECEIVED: Start All process request..."); - + //start the mysqld daemon try @@ -1548,7 +1548,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO system(cmd.c_str()); string tmpFile = tmpLogDir + "/mysqldstart"; - ifstream file (tmpFile); + ifstream file (tmpFile.c_str()); if (!file) { @@ -1829,7 +1829,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO } return_status = API_SUCCESS; - ifstream in(tmpMount); + ifstream in(tmpMount.c_str()); in.seekg(0, std::ios::end); int size = in.tellg(); @@ -4973,7 +4973,7 @@ void ProcessMonitor::checkModuleFailover( std::string processName) oam::DeviceNetworkConfig devicenetworkconfig; oam::DeviceNetworkList devicenetworklist; - + devicenetworkconfig.DeviceName = config.moduleName(); devicenetworklist.push_back(devicenetworkconfig); @@ -5348,7 +5348,7 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master // in log - show-master-status.log string masterLog = tmpLogDir + "/show-master-status.log"; - ifstream file (masterLog); + ifstream file (masterLog.c_str()); if (!file) { @@ -5512,7 +5512,7 @@ int ProcessMonitor::runDisableRep() if ( MySQLPort.empty() ) MySQLPort = "3306"; - + string logFile = tmpLogDir + "/disable-rep-columnstore.log"; string cmd = startup::StartUp::installDir() + "/bin/disable-rep-columnstore.sh --installdir=" + startup::StartUp::installDir() + " --tmpdir=" + tmpLogDir + " > " + logFile + " 2>&1"; @@ -6298,7 +6298,7 @@ int ProcessMonitor::checkDataMount() cmd = "export LC_ALL=C;mount " + dbroot + " > " + mountLog + " 2>&1"; system(cmd.c_str()); - ifstream in(mountLog); + ifstream in(mountLog.c_str()); in.seekg(0, std::ios::end); int size = in.tellg(); @@ -6506,7 +6506,7 @@ int ProcessMonitor::glusterAssign(std::string dbrootID) { log.writeLog(__LINE__, "glusterAssign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); - ifstream in(tmpLog); + ifstream in(tmpLog.c_str()); in.seekg(0, std::ios::end); int size = in.tellg(); @@ -6540,9 +6540,9 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) std::string errmsg = ""; log.writeLog(__LINE__, "glusterUnassign called: " + dbrootID, LOG_TYPE_DEBUG); - + string tmpLog = tmpLogDir + "/glusterUnassign.log"; - + command = "umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpLog + " 2>&1"; int ret = system(command.c_str()); @@ -6551,7 +6551,7 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) { log.writeLog(__LINE__, "glusterUnassign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); - ifstream in(tmpLog); + ifstream in(tmpLog.c_str()); in.seekg(0, std::ios::end); int size = in.tellg(); @@ -6560,7 +6560,7 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) if (!oam.checkLogStatus(tmpLog, "not mounted")) { log.writeLog(__LINE__, "glusterUnassign failed.", LOG_TYPE_ERROR); - + string cmd = "mv -f " + tmpLog + " " + tmpLog + "failed"; system(cmd.c_str()); return oam::API_FAILURE; From e1865d8216d03e948cbbc18384aca9598412aa80 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 15 Oct 2018 16:49:50 -0500 Subject: [PATCH 061/127] Tentative change to get past an obstacle in the distributed install. --- oam/install_scripts/binary_installer.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/binary_installer.sh b/oam/install_scripts/binary_installer.sh index 6b8b2f642..0fc104152 100644 --- a/oam/install_scripts/binary_installer.sh +++ b/oam/install_scripts/binary_installer.sh @@ -67,7 +67,7 @@ send_user "\n" send_user "Stop ColumnStore service " -send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/columnstore stop'\n" +send "ssh -v $USERNAME@$SERVER 'export COLUMNSTORE_INSTALL_DIR=$INSTALLDIR; $INSTALLDIR/bin/columnstore stop'\n" set timeout 60 # check return expect { @@ -214,7 +214,7 @@ send_user "\n" send_user "Start ColumnStore service " send_user " \n" -send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/columnstore restart'\n" +send "ssh -v $USERNAME@$SERVER 'export COLUMNSTORE_INSTALL_DIR=$INSTALLDIR; $INSTALLDIR/bin/columnstore restart'\n" set timeout 120 # check return expect { From 904d21ccd8d5e8aae6a8e23d0ff82a9697b63f00 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 22 Oct 2018 10:49:13 -0500 Subject: [PATCH 062/127] MCOL-520 - fix centos 7 compile errors --- oamapps/postConfigure/postConfigure.cpp | 4 ++-- oamapps/serverMonitor/cpuMonitor.cpp | 4 ++-- oamapps/serverMonitor/memoryMonitor.cpp | 2 +- oamapps/serverMonitor/msgProcessor.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index b2268b36f..a2f9a0df3 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1345,7 +1345,7 @@ int main(int argc, char* argv[]) string cmd = "aws --version > " + amazonLog + " 2>&1"; int rtnCode = system(cmd.c_str()); - ifstream in(amazonLog); + ifstream in(amazonLog.c_str()); in.seekg(0, std::ios::end); int size = in.tellg(); @@ -5122,7 +5122,7 @@ bool storageSetup(bool amazonInstall) cmd = "which hadoop > " + hadoopLog + " 2>&1"; system(cmd.c_str()); - ifstream in(hadoopLog); + ifstream in(hadoopLog.c_str()); in.seekg(0, std::ios::end); int size = in.tellg(); diff --git a/oamapps/serverMonitor/cpuMonitor.cpp b/oamapps/serverMonitor/cpuMonitor.cpp index 58c5bb24f..49d01ec3f 100644 --- a/oamapps/serverMonitor/cpuMonitor.cpp +++ b/oamapps/serverMonitor/cpuMonitor.cpp @@ -526,7 +526,7 @@ void ServerMonitor::getCPUdata() string cmd = "top -b -n1 | head -12 | awk '{print $9,$12}' | tail -5 > " + tmpProcessCpu; system(cmd.c_str()); - ifstream oldFile1 (tmpProcessCpu); + ifstream oldFile1 (tmpProcessCpu.c_str()); // read top 5 users int i = 0; @@ -559,7 +559,7 @@ void ServerMonitor::getCPUdata() cmd = "top -b -n 6 -d 1 | awk '{print $5}' | grep %id > " + tmpsystemCpu; system(cmd.c_str()); - ifstream oldFile (tmpsystemCpu); + ifstream oldFile (tmpsystemCpu.c_str()); float systemIdle = 0; // skip first line in file, and average the next 5 entries which contains idle times diff --git a/oamapps/serverMonitor/memoryMonitor.cpp b/oamapps/serverMonitor/memoryMonitor.cpp index ec3d2fcb2..b72d17e64 100644 --- a/oamapps/serverMonitor/memoryMonitor.cpp +++ b/oamapps/serverMonitor/memoryMonitor.cpp @@ -507,7 +507,7 @@ void ServerMonitor::outputProcMemory(bool log) string cmd = "ps -e -orss=1,args= | sort -b -k1,1n |tail -n 5 | awk '{print $1,$2}' > " + tmpprocessMem; system(cmd.c_str()); - ifstream oldFile (tmpprocessMem); + ifstream oldFile (tmpprocessMem.c_str()); string process; long long memory; diff --git a/oamapps/serverMonitor/msgProcessor.cpp b/oamapps/serverMonitor/msgProcessor.cpp index c4a7fe53d..48bfaa2f0 100644 --- a/oamapps/serverMonitor/msgProcessor.cpp +++ b/oamapps/serverMonitor/msgProcessor.cpp @@ -259,7 +259,7 @@ void msgProcessor() string cmd = "cat /proc/meminfo | grep Cached -m 1 | awk '{print $2}' > " + tmpcached; system(cmd.c_str()); - ifstream oldFile (tmpcached); + ifstream oldFile (tmpcached.c_str()); string strCache; long long cache; From ef4fab84087ed993d22dc5a2b359b591152ea1e9 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 22 Oct 2018 13:23:00 -0500 Subject: [PATCH 063/127] MCOL-520 - remove use of /usr/default/columnstore --- dbcon/mysql/mysql-Columnstore | 27 ++++++++++++++++++++++++--- oam/install_scripts/columnstore | 26 +++++++++++++++++++++----- oam/install_scripts/startupTests.sh | 24 +++++++++++++++++++++--- oam/post/functions | 25 +++++++++++++++++++++++-- oam/post/test-001.sh | 25 +++++++++++++++++++++++-- oam/post/test-002.sh | 25 +++++++++++++++++++++++-- oam/post/test-003.sh | 25 +++++++++++++++++++++++-- oam/post/test-004.sh | 26 ++++++++++++++++++++++++-- 8 files changed, 182 insertions(+), 21 deletions(-) diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore index 9f130d5dc..596cdb025 100755 --- a/dbcon/mysql/mysql-Columnstore +++ b/dbcon/mysql/mysql-Columnstore @@ -43,12 +43,33 @@ # If you change base dir, you must also change datadir. These may get # overwritten by settings in the MySQL configuration files. -if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - test -f /etc/default/columnstore && . /etc/default/columnstore +prefix=/usr/local + +USER=`whoami 2>/dev/null` + +if [ $USER != "root" ]; then + prefix=$HOME +fi + +if [ $USER != "root" ]; then + if [ -f $prefix/.bash_profile ]; then + profileFile=$prefix/.bash_profile + elif [ -f $prefix/.profile ]; then + profileFile=$prefix/.profile + else + profileFile=$prefix/.bashrc + fi + + . .$profileFile +fi + +# Source function library. +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions fi if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore + COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore fi export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index c7adef015..409497e32 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -17,15 +17,31 @@ # Short-Description: Start/stop MariaDB Columnstore DW DBMS ### END INIT INFO +prefix=/usr/local + +USER=`whoami 2>/dev/null` + +if [ $USER != "root" ]; then + prefix=$HOME +fi + +if [ $USER != "root" ]; then + if [ -f $prefix/.bash_profile ]; then + profileFile=$prefix/.bash_profile + elif [ -f $prefix/.profile ]; then + profileFile=$prefix/.profile + else + profileFile=$prefix/.bashrc + fi + + . .$profileFile +fi + # Source function library. if [ -f /etc/init.d/functions ]; then . /etc/init.d/functions fi -if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - test -f /etc/default/columnstore && . /etc/default/columnstore -fi - if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore fi @@ -40,7 +56,7 @@ if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then fi #hadoop -plugin=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig DataFilePlugin` +plugin=`$InstallDir/bin/getConfig SystemConfig DataFilePlugin` if [ -n "$plugin" ]; then setenv=`$InstallDir/bin/getConfig SystemConfig DataFileEnvFile` . $InstallDir/bin/$setenv >/dev/null 2>&1 diff --git a/oam/install_scripts/startupTests.sh b/oam/install_scripts/startupTests.sh index 00615518b..ac290df43 100755 --- a/oam/install_scripts/startupTests.sh +++ b/oam/install_scripts/startupTests.sh @@ -5,8 +5,27 @@ # startupTests - perform sanity testing on system DB at system startup time # called by Process-Monitor -if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - test -f /etc/default/columnstore && . /etc/default/columnstore +USER=`whoami 2>/dev/null` + +if [ $USER != "root" ]; then + prefix=$HOME +fi + +if [ $USER != "root" ]; then + if [ -f $prefix/.bash_profile ]; then + profileFile=$prefix/.bash_profile + elif [ -f $prefix/.profile ]; then + profileFile=$prefix/.profile + else + profileFile=$prefix/.bashrc + fi + + . .$profileFile +fi + +# Source function library. +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions fi if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then @@ -14,7 +33,6 @@ if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then fi export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR - test -f $COLUMNSTORE_INSTALL_DIR/post/functions && . $COLUMNSTORE_INSTALL_DIR/post/functions for testScript in $COLUMNSTORE_INSTALL_DIR/post/*.sh; do diff --git a/oam/post/functions b/oam/post/functions index 477b0d77a..e5cb8c3a4 100755 --- a/oam/post/functions +++ b/oam/post/functions @@ -2,8 +2,29 @@ # $Id: functions 2937 2012-05-30 18:17:09Z rdempsey $ # -if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - test -f /etc/default/columnstore && . /etc/default/columnstore +prefix=/usr/local + +USER=`whoami 2>/dev/null` + +if [ $USER != "root" ]; then + prefix=$HOME +fi + +if [ $USER != "root" ]; then + if [ -f $prefix/.bash_profile ]; then + profileFile=$prefix/.bash_profile + elif [ -f $prefix/.profile ]; then + profileFile=$prefix/.profile + else + profileFile=$prefix/.bashrc + fi + + . .$profileFile +fi + +# Source function library. +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions fi if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then diff --git a/oam/post/test-001.sh b/oam/post/test-001.sh index 030705ed2..47e18487f 100755 --- a/oam/post/test-001.sh +++ b/oam/post/test-001.sh @@ -2,8 +2,29 @@ # # $Id: test-001.sh 3704 2013-08-07 03:33:20Z bwilkinson $ -if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - test -f /etc/default/columnstore && . /etc/default/columnstore +prefix=/usr/local + +USER=`whoami 2>/dev/null` + +if [ $USER != "root" ]; then + prefix=$HOME +fi + +if [ $USER != "root" ]; then + if [ -f $prefix/.bash_profile ]; then + profileFile=$prefix/.bash_profile + elif [ -f $prefix/.profile ]; then + profileFile=$prefix/.profile + else + profileFile=$prefix/.bashrc + fi + + . .$profileFile +fi + +# Source function library. +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions fi if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then diff --git a/oam/post/test-002.sh b/oam/post/test-002.sh index ed0b6589f..42f52d3a8 100755 --- a/oam/post/test-002.sh +++ b/oam/post/test-002.sh @@ -2,8 +2,29 @@ # # $Id: test-002.sh 2937 2012-05-30 18:17:09Z rdempsey $ -if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - test -f /etc/default/columnstore && . /etc/default/columnstore +prefix=/usr/local + +USER=`whoami 2>/dev/null` + +if [ $USER != "root" ]; then + prefix=$HOME +fi + +if [ $USER != "root" ]; then + if [ -f $prefix/.bash_profile ]; then + profileFile=$prefix/.bash_profile + elif [ -f $prefix/.profile ]; then + profileFile=$prefix/.profile + else + profileFile=$prefix/.bashrc + fi + + . .$profileFile +fi + +# Source function library. +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions fi if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then diff --git a/oam/post/test-003.sh b/oam/post/test-003.sh index a01f3e933..c3dc2467c 100755 --- a/oam/post/test-003.sh +++ b/oam/post/test-003.sh @@ -2,8 +2,29 @@ # # $Id: test-003.sh 2937 2012-05-30 18:17:09Z rdempsey $ -if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - test -f /etc/default/columnstore && . /etc/default/columnstore +prefix=/usr/local + +USER=`whoami 2>/dev/null` + +if [ $USER != "root" ]; then + prefix=$HOME +fi + +if [ $USER != "root" ]; then + if [ -f $prefix/.bash_profile ]; then + profileFile=$prefix/.bash_profile + elif [ -f $prefix/.profile ]; then + profileFile=$prefix/.profile + else + profileFile=$prefix/.bashrc + fi + + . .$profileFile +fi + +# Source function library. +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions fi if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then diff --git a/oam/post/test-004.sh b/oam/post/test-004.sh index dcabcbfbb..6f6e009cb 100644 --- a/oam/post/test-004.sh +++ b/oam/post/test-004.sh @@ -6,14 +6,36 @@ # Validates that FilesPerColumnPartition setting is not set lower than existing extents. # -if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then - test -f /etc/default/columnstore && . /etc/default/columnstore +prefix=/usr/local + +USER=`whoami 2>/dev/null` + +if [ $USER != "root" ]; then + prefix=$HOME +fi + +if [ $USER != "root" ]; then + if [ -f $prefix/.bash_profile ]; then + profileFile=$prefix/.bash_profile + elif [ -f $prefix/.profile ]; then + profileFile=$prefix/.profile + else + profileFile=$prefix/.bashrc + fi + + . .$profileFile +fi + +# Source function library. +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions fi if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore fi +export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR test -f $COLUMNSTORE_INSTALL_DIR/post/functions && . $COLUMNSTORE_INSTALL_DIR/post/functions From 9cae0ef4ffd3f2af9cdae990b09c403cf88b1471 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 08:49:53 -0500 Subject: [PATCH 064/127] MCOL-520 --- dbcon/mysql/mysql-Columnstore | 2 +- oam/install_scripts/columnstore | 2 +- oam/install_scripts/startupTests.sh | 2 +- oam/install_scripts/syslogSetup.sh | 10 ++++++++++ oam/post/functions | 2 +- oam/post/test-001.sh | 2 +- oam/post/test-002.sh | 2 +- oam/post/test-003.sh | 2 +- oam/post/test-004.sh | 4 ++-- 9 files changed, 19 insertions(+), 9 deletions(-) diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore index 596cdb025..370b277d0 100755 --- a/dbcon/mysql/mysql-Columnstore +++ b/dbcon/mysql/mysql-Columnstore @@ -60,7 +60,7 @@ if [ $USER != "root" ]; then profileFile=$prefix/.bashrc fi - . .$profileFile + . $profileFile fi # Source function library. diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index 409497e32..130511a1c 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -34,7 +34,7 @@ if [ $USER != "root" ]; then profileFile=$prefix/.bashrc fi - . .$profileFile + . $profileFile fi # Source function library. diff --git a/oam/install_scripts/startupTests.sh b/oam/install_scripts/startupTests.sh index ac290df43..2c0decc4d 100755 --- a/oam/install_scripts/startupTests.sh +++ b/oam/install_scripts/startupTests.sh @@ -20,7 +20,7 @@ if [ $USER != "root" ]; then profileFile=$prefix/.bashrc fi - . .$profileFile + . $profileFile fi # Source function library. diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 0fd9f5998..e2d690fb4 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -156,7 +156,17 @@ fi } +makeDir() { + test -d /var/log/mariadb/columnstore || mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 + chmod 777 -R /var/log/mariadb/columnstore + chown $user:$user -R /var/log/mariadb +} + install() { +makeDir checkSyslog if [ ! -z "$syslog_conf" ] ; then $installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1 diff --git a/oam/post/functions b/oam/post/functions index e5cb8c3a4..4e9975eb4 100755 --- a/oam/post/functions +++ b/oam/post/functions @@ -19,7 +19,7 @@ if [ $USER != "root" ]; then profileFile=$prefix/.bashrc fi - . .$profileFile + . $profileFile fi # Source function library. diff --git a/oam/post/test-001.sh b/oam/post/test-001.sh index 47e18487f..2f0d0fcfb 100755 --- a/oam/post/test-001.sh +++ b/oam/post/test-001.sh @@ -19,7 +19,7 @@ if [ $USER != "root" ]; then profileFile=$prefix/.bashrc fi - . .$profileFile + . $profileFile fi # Source function library. diff --git a/oam/post/test-002.sh b/oam/post/test-002.sh index 42f52d3a8..7e4665672 100755 --- a/oam/post/test-002.sh +++ b/oam/post/test-002.sh @@ -19,7 +19,7 @@ if [ $USER != "root" ]; then profileFile=$prefix/.bashrc fi - . .$profileFile + . $profileFile fi # Source function library. diff --git a/oam/post/test-003.sh b/oam/post/test-003.sh index c3dc2467c..7d733ef78 100755 --- a/oam/post/test-003.sh +++ b/oam/post/test-003.sh @@ -19,7 +19,7 @@ if [ $USER != "root" ]; then profileFile=$prefix/.bashrc fi - . .$profileFile + . $profileFile fi # Source function library. diff --git a/oam/post/test-004.sh b/oam/post/test-004.sh index 6f6e009cb..3da9255ac 100644 --- a/oam/post/test-004.sh +++ b/oam/post/test-004.sh @@ -23,7 +23,7 @@ if [ $USER != "root" ]; then profileFile=$prefix/.bashrc fi - . .$profileFile + . $profileFile fi # Source function library. @@ -35,7 +35,7 @@ if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore fi -export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR +export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DI export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR test -f $COLUMNSTORE_INSTALL_DIR/post/functions && . $COLUMNSTORE_INSTALL_DIR/post/functions From e9b9cffa4579cdaaad363363b6b4c53ab0f058b3 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 09:31:51 -0500 Subject: [PATCH 065/127] MCOL-520 --- oam/install_scripts/post-install | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 01422dc8e..ba12ab186 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -322,12 +322,18 @@ if [ -z "$hadoop" ]; then cat < Date: Tue, 23 Oct 2018 10:02:45 -0500 Subject: [PATCH 066/127] MCOL-520 --- oamapps/mcsadmin/mcsadmin.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index a255f5e4b..07ec8f49c 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -9190,6 +9190,22 @@ void printSystemStatus() if ( MySQLRep == "y" ) cout << "MariaDB ColumnStore Replication Feature is enabled" << endl << endl; + + //display Distributed Install feature + if ( SingleServerInstall == "n" ) + { + string DistributedInstall; + + try + { + oam.getSystemConfig("DistributedInstall", DistributedInstall); + if ( DistributedInstall == "y" ) + cout << "MariaDB ColumnStore set for Distributed Install" << endl << endl; + else + cout << "MariaDB ColumnStore set for Non-Distributed Install" << endl << endl; + } + catch (...) {} + } } catch (exception& e) { From b96f8abcbbebe92f5befd8f666cee324326ba3ba Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 10:06:15 -0500 Subject: [PATCH 067/127] MCOL-520 --- oamapps/mcsadmin/mcsadmin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 07ec8f49c..43d2f2353 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -2685,7 +2685,7 @@ int processCommand(string* arguments) if (!oam.checkSystemRunning()) { - cout << "startSystem command, 'columnstore' service is down, sending command to" << endl; + cout << endl << "startSystem command, 'columnstore' service is down, sending command to" << endl; cout << "start the 'columnstore' service on all modules" << endl << endl; SystemModuleTypeConfig systemmoduletypeconfig; From 03f8154adaca1414e0a594be2fce5b62f85d89b2 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 12:21:10 -0500 Subject: [PATCH 068/127] MCOL-520 --- oamapps/mcsadmin/mcsadmin.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 43d2f2353..6089abc69 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -9189,7 +9189,7 @@ void printSystemStatus() catch (...) {} if ( MySQLRep == "y" ) - cout << "MariaDB ColumnStore Replication Feature is enabled" << endl << endl; + cout << "MariaDB ColumnStore Replication Feature is enabled" << endl; //display Distributed Install feature if ( SingleServerInstall == "n" ) @@ -9199,12 +9199,15 @@ void printSystemStatus() try { oam.getSystemConfig("DistributedInstall", DistributedInstall); + if ( DistributedInstall == "y" ) - cout << "MariaDB ColumnStore set for Distributed Install" << endl << endl; + cout << "MariaDB ColumnStore set for Distributed Install" << endl; else - cout << "MariaDB ColumnStore set for Non-Distributed Install" << endl << endl; + cout << "MariaDB ColumnStore set for Non-Distributed Install" << endl; } catch (...) {} + + cout << endl; } } catch (exception& e) From 6c3c219a5e2ee80f12a922fc86ef1aa62edc88f1 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 13:08:27 -0500 Subject: [PATCH 069/127] MCOL-520 --- oam/install_scripts/columnstore | 2 -- 1 file changed, 2 deletions(-) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index 130511a1c..bba8ca0fa 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -129,8 +129,6 @@ stop() { touch ${tmpDir}/StopColumnstore pkill ProcMon pkill ProcMgr - sleep 1 - $InstallDir/bin/clearShm RETVAL=$? rm -f $lockdir/columnstore fuser -k 8604/tcp > /dev/null 2>&1 From 9bce56beb398d1e300bc1393c184fbe3a39d6de5 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 14:40:17 -0500 Subject: [PATCH 070/127] MCOL-520 --- oam/oamcpp/liboamcpp.cpp | 22 +++++++++++++--------- oamapps/postConfigure/postConfigure.cpp | 25 +++++++++++++++---------- procmgr/processmanager.cpp | 2 +- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index cf8a15cde..ba40f9279 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -8798,14 +8798,18 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string group = getgid(); string glustercmd = "gluster "; - - errmsg = ""; + + string SUDO = ""; + if ( user != "root" ) + SUDO = "sudo "; + + string errmsg = ""; switch ( command ) { case (oam::GLUSTER_STATUS): { - string command = glustercmd + "volume status"; + string command = SUDO + glustercmd + "volume status"; char buffer[128]; string result = ""; @@ -9010,7 +9014,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string } sleep(5); - command = glustercmd + "peer status " + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = SUDO + glustercmd + "peer status " + " >> " + tmpdir + "/glusterCommands.log 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9032,7 +9036,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string { int newDbrootID = db + 1; - command = glustercmd + "volume create dbroot" + itoa(newDbrootID) + " transport tcp replica " + itoa(dataRedundancyCopies) + " "; + command = SUDO + glustercmd + "volume create dbroot" + itoa(newDbrootID) + " transport tcp replica " + itoa(dataRedundancyCopies) + " "; vector::iterator dbrootPmIter = dbrootPms[db].begin(); @@ -9074,7 +9078,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string } } - command = glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = SUDO + glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> " + tmpdir + "/glusterCommands.log 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9121,7 +9125,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string int status; writeLog("glusterctl: GLUSTER_DELETE: dbroot = " + dbrootID, LOG_TYPE_DEBUG ); - command = glustercmd + "--mode=script volume stop dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = SUDO + glustercmd + "--mode=script volume stop dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; status = system(command.c_str()); @@ -9134,7 +9138,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string // give time for transaction to finish after stopping sleep(10); - command = glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = SUDO + glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; status = system(command.c_str()); @@ -9154,7 +9158,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string string command = ""; int status; - command = glustercmd + "peer probe " + ipAddress + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = SUDO + glustercmd + "peer probe " + ipAddress + " >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << "gluster peer probe " + ipAddress << endl; status = system(command.c_str()); diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index a2f9a0df3..06e6bad36 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -198,13 +198,14 @@ string DataFileEnvFile; string installDir; string tmpDir; string HOME = "/root"; - +string SUDO = ""; extern string pwprompt; string mysqlpw = oam::UnassignedName; extern const char* pcommand; extern string prompt; + /* create thread argument struct for thr_func() */ typedef struct _thread_data_t { @@ -274,9 +275,13 @@ int main(int argc, char* argv[]) user = getuid(); usergroup = getgid(); - if (user != 0) + SUDO = "" + if (user != 0) { + rootUser = false; - + SUDO = "sudo "; + } + char* p = getenv("USER"); if (p && *p) @@ -5104,7 +5109,7 @@ bool storageSetup(bool amazonInstall) //check if gluster is installed int rtnCode = 1; - string cmd = "gluster --version > " + tmpDir + "/gluster.log 2>&1"; + string cmd = SUDO + "gluster --version > " + tmpDir + "/gluster.log 2>&1"; rtnCode = system(cmd.c_str()); if (rtnCode == 0) @@ -5515,7 +5520,7 @@ bool storageSetup(bool amazonInstall) // if gluster if ( storageType == "3" ) { - string command = "stat /var/run/glusterd.pid > /dev/null 2>&1"; + string command = SUDO + "stat /var/run/glusterd.pid > /dev/null 2>&1"; int status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -6704,7 +6709,7 @@ bool glusterSetup(string password) string glusterCommandsLog = tmpDir + "/glusterCommands.log"; - command = "gluster peer status " + glusterCommandsLog + "2>&1"; + command = SUDO + "gluster peer status " + glusterCommandsLog + "2>&1"; status = system(command.c_str()); @@ -6729,7 +6734,7 @@ bool glusterSetup(string password) { int dbrootID = db + 1; - command = "gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; + command = SUDO + "gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; vector::iterator dbrootPmIter = dbrootPms[db].begin(); @@ -6788,7 +6793,7 @@ bool glusterSetup(string password) { int user = getuid(); int group = getgid(); - command = "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> " + glusterCommandsLog + " 2>&1"; + command = SUDO + "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> " + glusterCommandsLog + " 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -6797,7 +6802,7 @@ bool glusterSetup(string password) exit(1); } - command = "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> " + glusterCommandsLog + " 2>&1"; + command = SUDO + "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> " + glusterCommandsLog + " 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -6806,7 +6811,7 @@ bool glusterSetup(string password) exit(1); } - command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> " + glusterCommandsLog + " 2>&1"; + command = SUDO + "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> " + glusterCommandsLog + " 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index b8385964f..11ed49fe3 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -1334,7 +1334,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) //now stop local module processManager.stopModule(config.moduleName(), graceful, manualFlag ); - //run save.brm script + //run save brm script processManager.saveBRM(false); log.writeLog(__LINE__, "Stop System Completed Success", LOG_TYPE_INFO); From 43c21d0ff93613125439ef4706c2769888632d9d Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 14:53:40 -0500 Subject: [PATCH 071/127] MCOL-520 --- oam/oamcpp/liboamcpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index ba40f9279..3633dde9f 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -8800,7 +8800,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string string glustercmd = "gluster "; string SUDO = ""; - if ( user != "root" ) + if ( user != 0 ) SUDO = "sudo "; string errmsg = ""; From 5bffb0a04766e9f4453b875a82c342a929340d2e Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 14:56:02 -0500 Subject: [PATCH 072/127] MCOL-520 --- oam/oamcpp/liboamcpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 3633dde9f..98297d555 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -8803,7 +8803,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string if ( user != 0 ) SUDO = "sudo "; - string errmsg = ""; + errmsg = ""; switch ( command ) { From c79672f7179332a990301817fc7f264e9306569d Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 14:57:16 -0500 Subject: [PATCH 073/127] MCOL-520 --- oamapps/postConfigure/postConfigure.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 06e6bad36..4ae8ea70c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -275,9 +275,9 @@ int main(int argc, char* argv[]) user = getuid(); usergroup = getgid(); - SUDO = "" - if (user != 0) { - + SUDO = ""; + if (user != 0) + { rootUser = false; SUDO = "sudo "; } From 47b62092ab892f075312bb29676199f511e33a16 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 23 Oct 2018 14:58:23 -0500 Subject: [PATCH 074/127] MCOL-520 --- oamapps/postConfigure/postConfigure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 4ae8ea70c..2bdaf365e 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -275,7 +275,7 @@ int main(int argc, char* argv[]) user = getuid(); usergroup = getgid(); - SUDO = ""; + string SUDO = ""; if (user != 0) { rootUser = false; From bca342d262b63321cc4a08a91ca408a56fc4cf6c Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 24 Oct 2018 09:25:08 -0500 Subject: [PATCH 075/127] MCOL-520 gluster changes --- oam/oamcpp/liboamcpp.cpp | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 98297d555..64a22aa7b 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -8796,20 +8796,20 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string int user, group; user = getuid(); group = getgid(); - - string glustercmd = "gluster "; string SUDO = ""; if ( user != 0 ) SUDO = "sudo "; - + + string glustercmd = SUDO + "gluster "; + errmsg = ""; switch ( command ) { case (oam::GLUSTER_STATUS): { - string command = SUDO + glustercmd + "volume status"; + string command = glustercmd + "volume status"; char buffer[128]; string result = ""; @@ -8995,12 +8995,14 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string for (int pm = (pmID - 1); pm < numberPMs; pm++) { cout << "gluster peer probe " + DataRedundancyConfigs[pm].pmIpAddr << endl; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { cout << "ERROR: peer probe command failed." << endl; command = InstallDir + "/bin/remote_command.sh " + DataRedundancyConfigs[pm].pmIpAddr + " " + password + " 'stat /var/run/glusterd.pid > /dev/null 2>&1'"; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9015,6 +9017,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string sleep(5); command = SUDO + glustercmd + "peer status " + " >> " + tmpdir + "/glusterCommands.log 2>&1"; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9049,6 +9052,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string command += "force >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << "Gluster create and start volume dbroot" << itoa(newDbrootID) << "..."; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9059,7 +9063,8 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string if (user != 0) { - command = "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-uid " + itoa(user) + " >> " + tmpdir + "/glusterCommands.log 2>&1";; + command = SUDO + "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-uid " + itoa(user) + " >> " + tmpdir + "/glusterCommands.log 2>&1";; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9068,7 +9073,8 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string exceptionControl("GLUSTER_ADD", API_FAILURE); } - command = "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-gid " + itoa(group) + " >> " + tmpdir + "/glusterCommands.log 2>&1";; + command = SUDO + "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-gid " + itoa(group) + " >> " + tmpdir + "/glusterCommands.log 2>&1";; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9079,6 +9085,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string } command = SUDO + glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> " + tmpdir + "/glusterCommands.log 2>&1"; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9125,8 +9132,9 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string int status; writeLog("glusterctl: GLUSTER_DELETE: dbroot = " + dbrootID, LOG_TYPE_DEBUG ); - command = SUDO + glustercmd + "--mode=script volume stop dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = glustercmd + "--mode=script volume stop dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9140,6 +9148,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string command = SUDO + glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9161,12 +9170,14 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string command = SUDO + glustercmd + "peer probe " + ipAddress + " >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << "gluster peer probe " + ipAddress << endl; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { cout << "ERROR: peer probe command failed." << endl; command = InstallDir + "/bin/remote_command.sh " + ipAddress + " " + password + " 'stat /var/run/glusterd.pid > /dev/null 2>&1'"; +cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) From 1456303ca8c239d9d1badb534d82a360de83eb0e Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 24 Oct 2018 09:41:52 -0500 Subject: [PATCH 076/127] MCOL-520 gluster changes --- oam/oamcpp/liboamcpp.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 64a22aa7b..65f619235 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -9016,7 +9016,7 @@ cout << command << endl; } sleep(5); - command = SUDO + glustercmd + "peer status " + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = glustercmd + "peer status " + " >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << command << endl; status = system(command.c_str()); @@ -9039,7 +9039,7 @@ cout << command << endl; { int newDbrootID = db + 1; - command = SUDO + glustercmd + "volume create dbroot" + itoa(newDbrootID) + " transport tcp replica " + itoa(dataRedundancyCopies) + " "; + command = glustercmd + "volume create dbroot" + itoa(newDbrootID) + " transport tcp replica " + itoa(dataRedundancyCopies) + " "; vector::iterator dbrootPmIter = dbrootPms[db].begin(); @@ -9084,7 +9084,7 @@ cout << command << endl; } } - command = SUDO + glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << command << endl; status = system(command.c_str()); @@ -9146,7 +9146,7 @@ cout << command << endl; // give time for transaction to finish after stopping sleep(10); - command = SUDO + glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << command << endl; status = system(command.c_str()); @@ -9167,7 +9167,7 @@ cout << command << endl; string command = ""; int status; - command = SUDO + glustercmd + "peer probe " + ipAddress + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = glustercmd + "peer probe " + ipAddress + " >> " + tmpdir + "/glusterCommands.log 2>&1"; cout << "gluster peer probe " + ipAddress << endl; cout << command << endl; From 29d83043d3648c9a3a0cb55dd0aa6c78ca0899c4 Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 24 Oct 2018 11:15:09 -0500 Subject: [PATCH 077/127] MCOL-520 gluster changes --- oamapps/postConfigure/postConfigure.cpp | 28 ++++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 2bdaf365e..c00cee54b 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -6710,9 +6710,9 @@ bool glusterSetup(string password) string glusterCommandsLog = tmpDir + "/glusterCommands.log"; command = SUDO + "gluster peer status " + glusterCommandsLog + "2>&1"; - +cout << "pc " << command << endl; status = system(command.c_str()); - +cout << status << endl; if (WEXITSTATUS(status) != 0 ) { cout << "ERROR: peer status command failed." << endl; @@ -6747,7 +6747,9 @@ bool glusterSetup(string password) command += "force >> " + glusterCommandsLog + " 2>&1"; cout << "Gluster create and start volume dbroot" << oam.itoa(dbrootID) << "..."; - status = system(command.c_str()); +cout << "pc " << command << endl; + status = system(command.c_str()); +cout << status << endl; if (WEXITSTATUS(status) != 0 ) { @@ -6763,7 +6765,9 @@ bool glusterSetup(string password) exit(1); } - status = system(command.c_str()); +cout << "pc " << command << endl; + status = system(command.c_str()); +cout << status << endl; if (WEXITSTATUS(status) != 0 ) { @@ -6781,7 +6785,9 @@ bool glusterSetup(string password) if (rootUser) { command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> " + glusterCommandsLog + " 2>&1"; - status = system(command.c_str()); +cout << "pc " << command << endl; + status = system(command.c_str()); +cout << status << endl; if (WEXITSTATUS(status) != 0 ) { @@ -6794,7 +6800,9 @@ bool glusterSetup(string password) int user = getuid(); int group = getgid(); command = SUDO + "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> " + glusterCommandsLog + " 2>&1"; - status = system(command.c_str()); +cout << "pc " << command << endl; + status = system(command.c_str()); +cout << status << endl; if (WEXITSTATUS(status) != 0 ) { @@ -6803,7 +6811,9 @@ bool glusterSetup(string password) } command = SUDO + "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> " + glusterCommandsLog + " 2>&1"; - status = system(command.c_str()); +cout << "pc " << command << endl; + status = system(command.c_str()); +cout << status << endl; if (WEXITSTATUS(status) != 0 ) { @@ -6812,7 +6822,9 @@ bool glusterSetup(string password) } command = SUDO + "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> " + glusterCommandsLog + " 2>&1"; - status = system(command.c_str()); +cout << "pc " << command << endl; + status = system(command.c_str()); +cout << status << endl; if (WEXITSTATUS(status) != 0 ) { From 2e2ef0360247ef10d8ae7aef9ee80f62c6ed2032 Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 24 Oct 2018 14:17:16 -0500 Subject: [PATCH 078/127] MCOL-520 gluster changes --- oam/oamcpp/liboamcpp.cpp | 38 +++---- oamapps/postConfigure/postConfigure.cpp | 131 +++++++++++++++++------- 2 files changed, 110 insertions(+), 59 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 65f619235..f3f6ae22e 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -8796,12 +8796,13 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string int user, group; user = getuid(); group = getgid(); - - string SUDO = ""; - if ( user != 0 ) - SUDO = "sudo "; - string glustercmd = SUDO + "gluster "; + string glustercmd = "gluster "; + + if (user != 0) + { + glustercmd = "sudo " + glustercmd; + } errmsg = ""; @@ -8995,14 +8996,12 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string for (int pm = (pmID - 1); pm < numberPMs; pm++) { cout << "gluster peer probe " + DataRedundancyConfigs[pm].pmIpAddr << endl; -cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { cout << "ERROR: peer probe command failed." << endl; command = InstallDir + "/bin/remote_command.sh " + DataRedundancyConfigs[pm].pmIpAddr + " " + password + " 'stat /var/run/glusterd.pid > /dev/null 2>&1'"; -cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9016,8 +9015,7 @@ cout << command << endl; } sleep(5); - command = glustercmd + "peer status " + " >> " + tmpdir + "/glusterCommands.log 2>&1"; -cout << command << endl; + command = glustercmd + "peer status " + " >> /tmp/glusterCommands.txt 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9050,9 +9048,8 @@ cout << command << endl; pmnextbrick[pm]++; } - command += "force >> " + tmpdir + "/glusterCommands.log 2>&1"; + command += "force >> /tmp/glusterCommands.txt 2>&1"; cout << "Gluster create and start volume dbroot" << itoa(newDbrootID) << "..."; -cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9063,8 +9060,7 @@ cout << command << endl; if (user != 0) { - command = SUDO + "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-uid " + itoa(user) + " >> " + tmpdir + "/glusterCommands.log 2>&1";; -cout << command << endl; + command = "sudo gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-uid " + itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9073,8 +9069,7 @@ cout << command << endl; exceptionControl("GLUSTER_ADD", API_FAILURE); } - command = SUDO + "gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-gid " + itoa(group) + " >> " + tmpdir + "/glusterCommands.log 2>&1";; -cout << command << endl; + command = "sudo gluster volume set dbroot" + itoa(newDbrootID) + " storage.owner-gid " + itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9084,8 +9079,7 @@ cout << command << endl; } } - command = glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> " + tmpdir + "/glusterCommands.log 2>&1"; -cout << command << endl; + command = glustercmd + "volume start dbroot" + itoa(newDbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9132,9 +9126,8 @@ cout << command << endl; int status; writeLog("glusterctl: GLUSTER_DELETE: dbroot = " + dbrootID, LOG_TYPE_DEBUG ); - command = glustercmd + "--mode=script volume stop dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = glustercmd + "--mode=script volume stop dbroot" + dbrootID + " >> /tmp/glusterCommands.txt 2>&1"; -cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9146,9 +9139,8 @@ cout << command << endl; // give time for transaction to finish after stopping sleep(10); - command = glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = glustercmd + " --mode=script volume delete dbroot" + dbrootID + " >> /tmp/glusterCommands.txt 2>&1"; -cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) @@ -9167,17 +9159,15 @@ cout << command << endl; string command = ""; int status; - command = glustercmd + "peer probe " + ipAddress + " >> " + tmpdir + "/glusterCommands.log 2>&1"; + command = glustercmd + "peer probe " + ipAddress + " >> /tmp/glusterCommands.txt 2>&1"; cout << "gluster peer probe " + ipAddress << endl; -cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { cout << "ERROR: peer probe command failed." << endl; command = InstallDir + "/bin/remote_command.sh " + ipAddress + " " + password + " 'stat /var/run/glusterd.pid > /dev/null 2>&1'"; -cout << command << endl; status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index c00cee54b..183a0b3fa 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -5109,8 +5109,10 @@ bool storageSetup(bool amazonInstall) //check if gluster is installed int rtnCode = 1; - string cmd = SUDO + "gluster --version > " + tmpDir + "/gluster.log 2>&1"; - rtnCode = system(cmd.c_str()); + if (rootUser) + rtnCode = system("gluster --version > /tmp/gluster.log 2>&1"); + else + rtnCode = system("sudo gluster --version > /tmp/gluster.log 2>&1"); if (rtnCode == 0) { @@ -5124,7 +5126,7 @@ bool storageSetup(bool amazonInstall) //check if hadoop is installed string hadoopLog = tmpDir + "/hadoop.log"; - cmd = "which hadoop > " + hadoopLog + " 2>&1"; + string cmd = "which hadoop > " + hadoopLog + " 2>&1"; system(cmd.c_str()); ifstream in(hadoopLog.c_str()); @@ -6253,7 +6255,6 @@ std::string launchInstance(ModuleIP moduleip) bool glusterSetup(string password) { - Oam oam; int dataRedundancyCopies = 0; int dataRedundancyNetwork = 0; @@ -6693,7 +6694,67 @@ bool glusterSetup(string password) cout << "ERROR: failed to make directory(" << DataRedundancyConfigs[pm].pmIpAddr << "): '" << command << "'" << endl; exit(1); } - } + + /* + if (dataRedundancyStorage == 2) + { + //walk data storage locations and modify fstab to reflect the storage locations entered by user + vector::iterator storageSetupIter=DataRedundancyConfigs[pm].storageLocations.begin(); + for (; storageSetupIter < DataRedundancyConfigs[pm].storageLocations.end(); storageSetupIter++ ) + { + if (rootUser) + { + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + " 'echo " + (*storageSetupIter).storageLocation + " " + + installDir + "/gluster/brick" + oam.itoa(brick) + " " + + (*storageSetupIter).storageFilesytemType + " defaults 1 2 >> /etc/fstab'"; + } + else + { + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + " 'sudo bash -c `sudo echo " + (*storageSetupIter).storageLocation + " " + + installDir + "/gluster/brick" + oam.itoa(brick) + " " + + (*storageSetupIter).storageFilesytemType + " defaults 1 2 >> /etc/fstab`'"; + } + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + if (rootUser) + { + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + " 'mount " + installDir + "/gluster/brick" + oam.itoa(brick) + "'"; + } + else + { + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + " 'sudo bash -c `sudo mount " + installDir + "/gluster/brick" + oam.itoa(brick) + "`'"; + } + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + if (!rootUser) + { + int user; + user = getuid(); + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + "'sudo bash -c `sudo chown -R " + oam.itoa(user) + ":" + oam.itoa(user) + " " + installDir + "/gluster/brick" + oam.itoa(brick) + "`'"; + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR(" << status <<"): command failed: " << command << endl; + } + } + + } + } + */ + } string errmsg1; string errmsg2; @@ -6706,13 +6767,18 @@ bool glusterSetup(string password) } sleep(5); - - string glusterCommandsLog = tmpDir + "/glusterCommands.log"; - command = SUDO + "gluster peer status " + glusterCommandsLog + "2>&1"; -cout << "pc " << command << endl; + if (rootUser) + { + command = "gluster peer status >> /tmp/glusterCommands.txt 2>&1"; + } + else + { + command = "sudo gluster peer status >> /tmp/glusterCommands.txt 2>&1"; + } + status = system(command.c_str()); -cout << status << endl; + if (WEXITSTATUS(status) != 0 ) { cout << "ERROR: peer status command failed." << endl; @@ -6734,7 +6800,14 @@ cout << status << endl; { int dbrootID = db + 1; - command = SUDO + "gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; + if (rootUser) + { + command = "gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; + } + else + { + command = "sudo gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; + } vector::iterator dbrootPmIter = dbrootPms[db].begin(); @@ -6745,15 +6818,13 @@ cout << status << endl; pmnextbrick[pm]++; } - command += "force >> " + glusterCommandsLog + " 2>&1"; + command += "force >> /tmp/glusterCommands.txt 2>&1"; cout << "Gluster create and start volume dbroot" << oam.itoa(dbrootID) << "..."; -cout << "pc " << command << endl; - status = system(command.c_str()); -cout << status << endl; + status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { - if (oam.checkLogStatus(glusterCommandsLog, "dbroot" + oam.itoa(dbrootID) + " already exists" )) + if (oam.checkLogStatus("/tmp/glusterCommands.txt", "dbroot" + oam.itoa(dbrootID) + " already exists" )) { string errmsg1; string errmsg2; @@ -6765,9 +6836,7 @@ cout << status << endl; exit(1); } -cout << "pc " << command << endl; - status = system(command.c_str()); -cout << status << endl; + status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { @@ -6784,10 +6853,8 @@ cout << status << endl; if (rootUser) { - command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> " + glusterCommandsLog + " 2>&1"; -cout << "pc " << command << endl; - status = system(command.c_str()); -cout << status << endl; + command = "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; + status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { @@ -6799,10 +6866,8 @@ cout << status << endl; { int user = getuid(); int group = getgid(); - command = SUDO + "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> " + glusterCommandsLog + " 2>&1"; -cout << "pc " << command << endl; - status = system(command.c_str()); -cout << status << endl; + command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; + status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { @@ -6810,10 +6875,8 @@ cout << status << endl; exit(1); } - command = SUDO + "gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> " + glusterCommandsLog + " 2>&1"; -cout << "pc " << command << endl; - status = system(command.c_str()); -cout << status << endl; + command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; + status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { @@ -6821,10 +6884,8 @@ cout << status << endl; exit(1); } - command = SUDO + "gluster volume start dbroot" + oam.itoa(dbrootID) + " >> " + glusterCommandsLog + " 2>&1"; -cout << "pc " << command << endl; - status = system(command.c_str()); -cout << status << endl; + command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; + status = system(command.c_str()); if (WEXITSTATUS(status) != 0 ) { From f65c2ecc912f28fa08d45fe15aae0b5640aa5396 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 25 Oct 2018 13:56:50 -0500 Subject: [PATCH 079/127] MCOL-520 amazon changes --- oam/cloud/MCSInstanceCmds.sh | 2 +- oam/cloud/MCSVolumeCmds.sh | 4 ++-- oamapps/postConfigure/quick_installer_amazon.sh | 10 ++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/oam/cloud/MCSInstanceCmds.sh b/oam/cloud/MCSInstanceCmds.sh index 4691f5d11..20930f87c 100755 --- a/oam/cloud/MCSInstanceCmds.sh +++ b/oam/cloud/MCSInstanceCmds.sh @@ -11,7 +11,7 @@ fi export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR #get temp directory -tmpDir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir` +tmpdir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir` #check command if [ "$1" = "" ]; then diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh index 6228c93e9..f6fb06a55 100755 --- a/oam/cloud/MCSVolumeCmds.sh +++ b/oam/cloud/MCSVolumeCmds.sh @@ -11,7 +11,7 @@ fi export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR #get temp directory -tmpDir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir` +tmpdir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir` #check command if [ "$1" = "" ]; then @@ -277,7 +277,7 @@ deletevolume() { createTag() { #create tag - $AWSCLI create-tags --resources $resourceName --tags Key=$tagName,Value=$tagValue --region $Region > ${tmpdir}/createTag_$volumeName 2>&1 + $AWSCLI create-tags --resources $resourceName --tags Key=$tagName,Value=$tagValue --region $Region > ${tmpdir}/createTag_$resourceName 2>&1 return } diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh index 4a1df903c..a844e0ee8 100755 --- a/oamapps/postConfigure/quick_installer_amazon.sh +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -16,8 +16,6 @@ for arg in "$@"; do elif [ `expr -- "$arg" : '--system-name='` -eq 14 ]; then systemName="`echo $arg | awk -F= '{print $2}'`" systemName="-sn "$systemName - elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then - nonDistrubutedInstall=" " elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then echo "Usage ./quick_installer_amazon.sh [OPTION]" echo "" @@ -63,9 +61,9 @@ if [[ $HOME = "/root" ]]; then echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umCount = "" ]]; then - /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount $systemName + /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount $systemName -d else - /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount -um-count $umCount $systemName + /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount -um-count $umCount $systemName -d fi else echo "${bold}Run post-install script${normal}" @@ -74,8 +72,8 @@ else echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umCount = "" ]]; then - . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName -d else - . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName + . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName -d fi fi From 3693e729779ad43c1f8aba6b01a0a4b4afadf51f Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 25 Oct 2018 14:30:03 -0500 Subject: [PATCH 080/127] MCOL-520 amazon changes --- oamapps/postConfigure/postConfigure.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 183a0b3fa..f7c1bc88d 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4862,7 +4862,7 @@ bool storageSetup(bool amazonInstall) return true; } - cout << "===== Setup Storage Configuration =====" << endl; + cout << "===== Setup Storage Configuration =====" << endl << endl; string storageType; @@ -5109,10 +5109,7 @@ bool storageSetup(bool amazonInstall) //check if gluster is installed int rtnCode = 1; - if (rootUser) - rtnCode = system("gluster --version > /tmp/gluster.log 2>&1"); - else - rtnCode = system("sudo gluster --version > /tmp/gluster.log 2>&1"); + rtnCode = system("gluster --version > /tmp/gluster.log 2>&1"); if (rtnCode == 0) { From d08fcfec306f87df0dcf280d3e3d679eaec9fe93 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 25 Oct 2018 15:40:38 -0500 Subject: [PATCH 081/127] MCOL-520 amazon changes --- oamapps/postConfigure/postConfigure.cpp | 2 +- procmgr/processmanager.cpp | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index f7c1bc88d..9735225b1 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1818,7 +1818,7 @@ int main(int argc, char* argv[]) // Module Configuration // cout << endl << endl; - cout << "===== Setup the Module Configuration =====" << endl; + cout << "===== Setup the Module Configuration =====" << endl << endl; if (amazonInstall) { diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 11ed49fe3..c67da955e 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -5134,10 +5134,11 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str if (!oam.checkLogStatus(loginTmp, "README")) { //check for RSA KEY ISSUE and fix - if (oam.checkLogStatus(loginTmp, "Offending")) + if (oam.checkLogStatus(loginTmp, "Host key verification failed")) { - log.writeLog(__LINE__, "addModule - login failed, Offending key issue, try fixing: " + moduleName, LOG_TYPE_DEBUG); - oam.fixRSAkey(loginTmp); + log.writeLog(__LINE__, "addModule - login failed, Host key verification failed, try fixing: " + moduleName, LOG_TYPE_DEBUG); + cmd = "rm -f " + homedir + "/.ssh/known_hosts"; + system(cmd.c_str()); } log.writeLog(__LINE__, "addModule - login failed, retry login test: " + moduleName, LOG_TYPE_DEBUG); From 1b968682c49ad5e07671e70bb06ef929a4c676d5 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 25 Oct 2018 16:14:20 -0500 Subject: [PATCH 082/127] MCOL-520 amazon changes --- oamapps/postConfigure/quick_installer_amazon.sh | 7 +++++-- oamapps/postConfigure/quick_installer_multi_server.sh | 7 +++++-- oamapps/postConfigure/quick_installer_single_server.sh | 5 ++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh index a844e0ee8..95fb4978a 100755 --- a/oamapps/postConfigure/quick_installer_amazon.sh +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -69,11 +69,14 @@ else echo "${bold}Run post-install script${normal}" echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + + tmpDir=`$HOME/mariadb/columnstore/bin/getConfig SystemConfig SystemTempFileDir` + echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umCount = "" ]]; then - . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName -d + . $tmpDir;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName -d else - . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName -d + . $tmpDir;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName -d fi fi diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh index b9786f5ef..2a3042550 100755 --- a/oamapps/postConfigure/quick_installer_multi_server.sh +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -75,11 +75,14 @@ else echo "${bold}Run post-install script${normal}" echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + + tmpDir=`$HOME/mariadb/columnstore/bin/getConfig SystemConfig SystemTempFileDir` + echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umIpAddrs = "" ]]; then - . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName + . $tmpDir;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName else - . /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName + . $tmpDir;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName fi fi diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh index 432b395c4..aae015dba 100755 --- a/oamapps/postConfigure/quick_installer_single_server.sh +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -28,7 +28,10 @@ else echo "Run post-install script" echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore + + tmpDir=`$HOME/mariadb/columnstore/bin/getConfig SystemConfig SystemTempFileDir` + echo "Run postConfigure script" echo "" - . /etc/profile.d/columnstoreEnv.sh; $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs + . $tmpDir; $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs fi From f4aef75028a42eab9112ac973d5ae1f1d7e74ba4 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 25 Oct 2018 16:28:03 -0500 Subject: [PATCH 083/127] MCOL-520 amazon changes --- procmgr/processmanager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index c67da955e..17e4eb7ce 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -5129,9 +5129,9 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string loginTmp = tmpLogDir + "/login_test.log"; string cmd = installDir + "/bin/remote_command.sh " + IPAddr + " " + password + " 'ls' 1 > " + loginTmp; - system(cmd.c_str()); + int rtnCode = system(cmd.c_str()); - if (!oam.checkLogStatus(loginTmp, "README")) + if (WEXITSTATUS(rtnCode) != "0") { //check for RSA KEY ISSUE and fix if (oam.checkLogStatus(loginTmp, "Host key verification failed")) From d6d1a11d0b161fcde98f8654c191e2f01ab799b2 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 25 Oct 2018 16:43:58 -0500 Subject: [PATCH 084/127] MCOL-520 amazon changes --- oam/cloud/MCSVolumeCmds.sh | 2 +- procmgr/processmanager.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh index f6fb06a55..9311000c1 100755 --- a/oam/cloud/MCSVolumeCmds.sh +++ b/oam/cloud/MCSVolumeCmds.sh @@ -132,7 +132,7 @@ checkInfostatus() { return fi #check if attaching - cat ${tmpdir}volumeInfo_$volumeName | grep attaching > ${tmpdir}/volumeStatus_$volumeName + cat ${tmpdir}/volumeInfo_$volumeName | grep attaching > ${tmpdir}/volumeStatus_$volumeName if [ `cat ${tmpdir}/volumeStatus_$volumeName | wc -c` -ne 0 ]; then STATUS="attaching" RETVAL=0 diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 17e4eb7ce..8591f1f37 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -5131,7 +5131,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string cmd = installDir + "/bin/remote_command.sh " + IPAddr + " " + password + " 'ls' 1 > " + loginTmp; int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != "0") + if (WEXITSTATUS(rtnCode) != 0) { //check for RSA KEY ISSUE and fix if (oam.checkLogStatus(loginTmp, "Host key verification failed")) From e6855c3221eafcffe7c26f4a991d5fa99df1ee0c Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 26 Oct 2018 09:10:45 -0500 Subject: [PATCH 085/127] MCOL-520 amazon changes --- oamapps/postConfigure/postConfigure.cpp | 184 +++++++++--------- .../postConfigure/quick_installer_amazon.sh | 10 +- .../quick_installer_multi_server.sh | 9 +- .../quick_installer_single_server.sh | 6 +- 4 files changed, 112 insertions(+), 97 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9735225b1..63ba2f9a2 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -805,12 +805,16 @@ int main(int argc, char* argv[]) { cout << "===== Quick Install Multi-Server Configuration =====" << endl << endl; + cout << "Multi-Server install defaulting to using local storage" << endl; + singleServerInstall = "2"; } else if (amazon_quick_install) { cout << "===== Quick Install Amazon Configuration =====" << endl << endl; + cout << "Amazon AMI EC2 install defaulting to using local storage" << endl; + singleServerInstall = "2"; } else @@ -856,95 +860,14 @@ int main(int argc, char* argv[]) if (temp == "1") { - singleServerInstall = temp; - cout << endl << "Performing the Single Server Install." << endl << endl; - - if ( reuseConfig == "n" ) - { - //setup to Columnstore.xml file for single server - singleServerConfigSetup(sysConfig); - } - - //module ProcessConfig.xml to setup all apps on the pm - if ( !updateProcessConfig() ) - cout << "Update ProcessConfig.xml error" << endl; - - try - { - sysConfig->setConfig(InstallSection, "SingleServerInstall", "y"); - sysConfig->setConfig(InstallSection, "ServerTypeInstall", "2"); - } - catch (...) - { - cout << "ERROR: Problem setting SingleServerInstall from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - if ( !writeConfig(sysConfig) ) - { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - setSystemName(); - cout << endl; - - system(cmd.c_str()); - - // setup storage - if (!storageSetup(false)) - { - cout << "ERROR: Problem setting up storage" << endl; - exit(1); - } - - if (hdfs || !rootUser) - if ( !updateBash() ) - cout << "updateBash error" << endl; - - // setup storage - if (!singleServerDBrootSetup()) - { - cout << "ERROR: Problem setting up DBRoot IDs" << endl; - exit(1); - } - - //set system DBRoot count and check 'files per parition' with number of dbroots - try - { - sysConfig->setConfig(SystemSection, "DBRootCount", oam.itoa(DBRootCount)); - } - catch (...) - { - cout << "ERROR: Problem setting DBRoot Count in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - //check if dbrm data resides in older directory path and inform user if it does - dbrmDirCheck(); - - if (startOfflinePrompt) - offLineAppCheck(); - - checkMysqlPort(mysqlPort, sysConfig); - - if ( !writeConfig(sysConfig) ) - { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; - - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; - system(cmd.c_str()); - exit(0); + singleServerInstall = "1"; + break; } else { if (temp == "2") { - singleServerInstall = temp; + singleServerInstall = "2"; break; } } @@ -953,14 +876,99 @@ int main(int argc, char* argv[]) if ( noPrompting ) exit(1); - - continue; } - - break; } } + // perform single server install + if (singleServerInstall == "1") + { + cout << endl << "Performing the Single Server Install." << endl << endl; + + if ( reuseConfig == "n" ) + { + //setup to Columnstore.xml file for single server + singleServerConfigSetup(sysConfig); + } + + //module ProcessConfig.xml to setup all apps on the pm + if ( !updateProcessConfig() ) + cout << "Update ProcessConfig.xml error" << endl; + + try + { + sysConfig->setConfig(InstallSection, "SingleServerInstall", "y"); + sysConfig->setConfig(InstallSection, "ServerTypeInstall", "2"); + } + catch (...) + { + cout << "ERROR: Problem setting SingleServerInstall from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + if ( !writeConfig(sysConfig) ) + { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + setSystemName(); + cout << endl; + + system(cmd.c_str()); + + // setup storage + if (!storageSetup(false)) + { + cout << "ERROR: Problem setting up storage" << endl; + exit(1); + } + + if (hdfs || !rootUser) + if ( !updateBash() ) + cout << "updateBash error" << endl; + + // setup storage + if (!singleServerDBrootSetup()) + { + cout << "ERROR: Problem setting up DBRoot IDs" << endl; + exit(1); + } + + //set system DBRoot count and check 'files per parition' with number of dbroots + try + { + sysConfig->setConfig(SystemSection, "DBRootCount", oam.itoa(DBRootCount)); + } + catch (...) + { + cout << "ERROR: Problem setting DBRoot Count in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + //check if dbrm data resides in older directory path and inform user if it does + dbrmDirCheck(); + + if (startOfflinePrompt) + offLineAppCheck(); + + checkMysqlPort(mysqlPort, sysConfig); + + if ( !writeConfig(sysConfig) ) + { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; + + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; + system(cmd.c_str()); + exit(0); + } + + // perform multi-node install + try { sysConfig->setConfig(InstallSection, "SingleServerInstall", "n"); diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh index 95fb4978a..f1f287cbe 100755 --- a/oamapps/postConfigure/quick_installer_amazon.sh +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -55,6 +55,7 @@ else fi if [[ $HOME = "/root" ]]; then + echo "" echo "${bold}Run post-install script${normal}" echo "" /usr/local/mariadb/columnstore/bin/post-install @@ -70,13 +71,14 @@ else echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore - tmpDir=`$HOME/mariadb/columnstore/bin/getConfig SystemConfig SystemTempFileDir` - + export COLUMNSTORE_INSTALL_DIR=$HOME/mariadb/columnstore + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/mariadb/columnstore/lib:$HOME/mariadb/columnstore/mysql/lib + echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umCount = "" ]]; then - . $tmpDir;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName -d + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName -d else - . $tmpDir;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName -d + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName -d fi fi diff --git a/oamapps/postConfigure/quick_installer_multi_server.sh b/oamapps/postConfigure/quick_installer_multi_server.sh index 2a3042550..d47bdaacd 100755 --- a/oamapps/postConfigure/quick_installer_multi_server.sh +++ b/oamapps/postConfigure/quick_installer_multi_server.sh @@ -61,6 +61,7 @@ else fi if [[ $HOME = "/root" ]]; then + echo "" echo "${bold}Run post-install script${normal}" echo "" /usr/local/mariadb/columnstore/bin/post-install @@ -72,17 +73,19 @@ if [[ $HOME = "/root" ]]; then /usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName fi else + echo "" echo "${bold}Run post-install script${normal}" echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore - tmpDir=`$HOME/mariadb/columnstore/bin/getConfig SystemConfig SystemTempFileDir` + export COLUMNSTORE_INSTALL_DIR=$HOME/mariadb/columnstore + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/mariadb/columnstore/lib:$HOME/mariadb/columnstore/mysql/lib echo "${bold}Run postConfigure script${normal}" echo "" if [[ $umIpAddrs = "" ]]; then - . $tmpDir;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName else - . $tmpDir;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName fi fi diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh index aae015dba..5a88abbef 100755 --- a/oamapps/postConfigure/quick_installer_single_server.sh +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -18,6 +18,7 @@ done if [ $HOME == "/root" ]; then + echo "" echo "Run post-install script" echo "" /usr/local/mariadb/columnstore/bin/post-install @@ -29,9 +30,10 @@ else echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore - tmpDir=`$HOME/mariadb/columnstore/bin/getConfig SystemConfig SystemTempFileDir` + export COLUMNSTORE_INSTALL_DIR=$HOME/mariadb/columnstore + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/mariadb/columnstore/lib:$HOME/mariadb/columnstore/mysql/lib echo "Run postConfigure script" echo "" - . $tmpDir; $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs + $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs fi From 42a328652be6ef16bfa658d886679e3f7c0f84d6 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 26 Oct 2018 14:21:57 -0500 Subject: [PATCH 086/127] MCOL-520 amazon changes --- oamapps/postConfigure/quick_installer_amazon.sh | 1 + oamapps/postConfigure/quick_installer_single_server.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/oamapps/postConfigure/quick_installer_amazon.sh b/oamapps/postConfigure/quick_installer_amazon.sh index f1f287cbe..67f0f2ce1 100755 --- a/oamapps/postConfigure/quick_installer_amazon.sh +++ b/oamapps/postConfigure/quick_installer_amazon.sh @@ -67,6 +67,7 @@ if [[ $HOME = "/root" ]]; then /usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount -um-count $umCount $systemName -d fi else + echo "" echo "${bold}Run post-install script${normal}" echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh index 5a88abbef..1b747adc3 100755 --- a/oamapps/postConfigure/quick_installer_single_server.sh +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -26,6 +26,7 @@ if [ $HOME == "/root" ]; then echo "" /usr/local/mariadb/columnstore/bin/postConfigure -qs else + echo "" echo "Run post-install script" echo "" $HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore From a82b431ae674c6b34663b0c36c1b1138f5301381 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 26 Oct 2018 14:42:44 -0500 Subject: [PATCH 087/127] MCOL-520 - amazon changes --- oam/install_scripts/module_installer.sh | 4 ++-- oamapps/postConfigure/postConfigure.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 9627acff7..9e3a1a0e5 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -172,8 +172,8 @@ fi touch $RCFILE if [ $user != "root" ]; then - echo "uncomment runuser in rc.local" - sed -i -e 's/#MCS//g' /etc/rc.d/rc.local >/dev/null 2>&1 + echo "uncomment runuser in rc.local, amazon AMI" + sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 fi echo "!!!Module Installation Successfully Completed!!!" diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 63ba2f9a2..30b91eb38 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1477,7 +1477,7 @@ int main(int argc, char* argv[]) // setup to start on reboot, for non-root amazon installs if ( !rootUser ) { - system("sed -i -e 's/#MCS//g' /etc/rc.d/rc.local >/dev/null 2>&1"); + system("sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); } if ( !writeConfig(sysConfig) ) From 648a1e74368f98affd34409e84aa6fb8cbbe12de Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 26 Oct 2018 18:57:09 -0500 Subject: [PATCH 088/127] MCOL-520 amazon ebs changes --- oam/oamcpp/liboamcpp.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index f3f6ae22e..aca414bf9 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -6505,8 +6505,11 @@ void Oam::addUMdisk(const int moduleID, std::string& volumeName, std::string& de int user; user = getuid(); - cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1"; - + if ( user == 0 ) + cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1"; + else + cmd = "sudo mkfs.ext2 -F " + device + " > /dev/null 2>&1"; + system(cmd.c_str()); //detach volume @@ -6752,7 +6755,10 @@ void Oam::addDbroot(const int dbrootNumber, DBRootConfigList& dbrootlist, string int user; user = getuid(); - cmd = "mkfs.ext2 -F " + amazonDeviceName + " > " + tmpdir + "/format.log 2>&1"; + if (user == 0 ) + cmd = "mkfs.ext2 -F " + amazonDeviceName + " > " + tmpdir + "/format.log 2>&1"; + else + cmd = "sudo mkfs.ext2 -F " + amazonDeviceName + " > " + tmpdir + "/format.log 2>&1"; writeLog("addDbroot format cmd: " + cmd, LOG_TYPE_DEBUG ); From cc158c9c968a75fcc9c5802bb3892ba34d0006ed Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 29 Oct 2018 14:19:04 -0500 Subject: [PATCH 089/127] MCOL-520 - amazon added sudo for chmod --- oam/oamcpp/liboamcpp.cpp | 5 +++-- procmon/processmonitor.cpp | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index aca414bf9..9b7ee2fab 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -9473,7 +9473,7 @@ std::string Oam::getMySQLPassword() /****************************************************************************************** * @brief updateFstab * -* purpose: check and get mysql user password +* purpose: Update Fstabs for Amazon EBS setup * ******************************************************************************************/ std::string Oam::updateFstab(std::string device, std::string dbrootID) @@ -9499,7 +9499,8 @@ std::string Oam::updateFstab(std::string device, std::string dbrootID) if (WEXITSTATUS(status) != 0 ) { - //update /etc/fstab with mount + //chmod before update, used on amazon ami EBS. not other systems + system("sudo chmod 666 /etc/fstab"); //update local fstab cmd = "echo " + entry + " >> /etc/fstab"; diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 2f61ae690..482b5c2bb 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1867,6 +1867,9 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if (WEXITSTATUS(status) != 0 ) { + //chmod before update, used on amazon ami EBS. not other systems + system("sudo chmod 666 /etc/fstab"); + cmd = "echo " + entry + " >> /etc/fstab"; system(cmd.c_str()); From d4005120a69e52cb0e16206ddd16a6dd4dd9464c Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 08:55:42 -0500 Subject: [PATCH 090/127] MCOL-520 - chown sudo amazon --- oamapps/postConfigure/postConfigure.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 30b91eb38..4cfc275a4 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4654,9 +4654,15 @@ bool createDbrootDirs(string DBRootStorageType) string cmd = "mount " + installDir + "/data1 > " + tmpDir + "/mount.txt 2>&1"; system(cmd.c_str()); + //if Amazon, Non-Root, and External EBS, use sudo + string SUDO = ""; + if ( amazonInstall && !rootUser && + ( DBRootStorageType == "external" ) ) + SUDO = "sudo "; + if ( !rootUser) { - cmd = "chown -R " + USER + ":" + USER + " " + installDir + "/data1 > /dev/null"; + cmd = SUDO + "chown -R " + USER + ":" + USER + " " + installDir + "/data1 > /dev/null"; system(cmd.c_str()); } From d13e63ecaaba67f07f59fc8ae9006b1ac566af0f Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 09:18:39 -0500 Subject: [PATCH 091/127] MCOL-520 - chown sudo amazon --- oamapps/postConfigure/postConfigure.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 4cfc275a4..26cb8b5c4 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4654,16 +4654,17 @@ bool createDbrootDirs(string DBRootStorageType) string cmd = "mount " + installDir + "/data1 > " + tmpDir + "/mount.txt 2>&1"; system(cmd.c_str()); - //if Amazon, Non-Root, and External EBS, use sudo - string SUDO = ""; - if ( amazonInstall && !rootUser && - ( DBRootStorageType == "external" ) ) - SUDO = "sudo "; - if ( !rootUser) { + //if Amazon, Non-Root, and External EBS, use sudo + string SUDO = ""; + if ( amazonInstall ) + SUDO = "sudo "; + cmd = SUDO + "chown -R " + USER + ":" + USER + " " + installDir + "/data1 > /dev/null"; +cout << cmd << endl; system(cmd.c_str()); +cout << cmd << endl; } // create system file directories From a9920193f86f71d345d6ae4a5c5a907a93401b07 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 10:10:00 -0500 Subject: [PATCH 092/127] MCOL-520 - chown sudo amazon --- oamapps/postConfigure/postConfigure.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 26cb8b5c4..b723bd846 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -118,7 +118,7 @@ bool makeModuleFile(string moduleName, string parentOAMModuleName); bool updateProcessConfig(); bool uncommentCalpontXml( string entry); bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall); -bool createDbrootDirs(string DBRootStorageType); +bool createDbrootDirs(string DBRootStorageType, bool amazonInstall); bool pkgCheck(std::string columnstorePackage); bool storageSetup(bool amazonInstall); void setSystemName(); @@ -3469,7 +3469,7 @@ int main(int argc, char* argv[]) } //create directories on dbroot1 - if ( !createDbrootDirs(DBRootStorageType) ) + if ( !createDbrootDirs(DBRootStorageType, amazonInstall) ) { cout << "createDbrootDirs error" << endl; exit(1); @@ -4643,7 +4643,7 @@ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall) /* * createDbrootDirs */ -bool createDbrootDirs(string DBRootStorageType) +bool createDbrootDirs(string DBRootStorageType, bool amazonInstall) { int rtnCode; string cmd; From 9a169e2395966949db0a7c7b5f12b0a1901c6b34 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 10:25:34 -0500 Subject: [PATCH 093/127] MCOL-520 - chown sudo amazon --- oamapps/postConfigure/postConfigure.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index b723bd846..622350ded 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4662,9 +4662,7 @@ bool createDbrootDirs(string DBRootStorageType, bool amazonInstall) SUDO = "sudo "; cmd = SUDO + "chown -R " + USER + ":" + USER + " " + installDir + "/data1 > /dev/null"; -cout << cmd << endl; system(cmd.c_str()); -cout << cmd << endl; } // create system file directories From 8847a2fade6ca249b214f7ac2d9b777ce7f95536 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 10:38:13 -0500 Subject: [PATCH 094/127] MCOL-520 - mkfs testing amazon --- oam/oamcpp/liboamcpp.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 9b7ee2fab..40d7afa7a 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -6506,10 +6506,10 @@ void Oam::addUMdisk(const int moduleID, std::string& volumeName, std::string& de user = getuid(); if ( user == 0 ) - cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1"; + cmd = "mkfs.ext2 -F " + device + " > " + tmpdir + "/format.log 2>&1"; else - cmd = "sudo mkfs.ext2 -F " + device + " > /dev/null 2>&1"; - + cmd = "sudo mkfs.ext2 -F " + device + " > " + tmpdir + "/format.log 2>&1"; +cout << "1 " << cmd << endl; system(cmd.c_str()); //detach volume @@ -6762,6 +6762,7 @@ void Oam::addDbroot(const int dbrootNumber, DBRootConfigList& dbrootlist, string writeLog("addDbroot format cmd: " + cmd, LOG_TYPE_DEBUG ); +cout << "1 " << cmd << endl; system(cmd.c_str()); //detach From fe4de755ea7a8ca777601f92f5819fc4b0bad5d9 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 13:39:06 -0500 Subject: [PATCH 095/127] MCOL-520 - mkfs testing amazon --- oam/oamcpp/liboamcpp.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 40d7afa7a..921488a0c 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -6509,7 +6509,7 @@ void Oam::addUMdisk(const int moduleID, std::string& volumeName, std::string& de cmd = "mkfs.ext2 -F " + device + " > " + tmpdir + "/format.log 2>&1"; else cmd = "sudo mkfs.ext2 -F " + device + " > " + tmpdir + "/format.log 2>&1"; -cout << "1 " << cmd << endl; + system(cmd.c_str()); //detach volume @@ -6762,7 +6762,6 @@ void Oam::addDbroot(const int dbrootNumber, DBRootConfigList& dbrootlist, string writeLog("addDbroot format cmd: " + cmd, LOG_TYPE_DEBUG ); -cout << "1 " << cmd << endl; system(cmd.c_str()); //detach From c6e6ea78e347d4163a7407ce7e9cbc0660868adb Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 13:45:45 -0500 Subject: [PATCH 096/127] MCOL-520 - sudo testing amazon --- oam/install_scripts/module_installer.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 9e3a1a0e5..2c1150197 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -66,7 +66,12 @@ if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then if [ $module = "pm" ]; then if test -f $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab ; then echo "Setup fstab on Module" + SUDO="" + if [ $user != "root" ]; then + SUDO="sudo " + fi touch /etc/fstab + $SUDO chmod 666 /etc/fstab rm -f /etc/fstab.columnstoreSave cp /etc/fstab /etc/fstab.columnstoreSave cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab From 32d9b20cf363f7a108050e529bd6b6b0ffa54fab Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 15:27:39 -0500 Subject: [PATCH 097/127] MCOL-520 - sudo testing amazon --- oam/install_scripts/module_installer.sh | 2 +- procmon/main.cpp | 8 ++++++-- procmon/processmonitor.cpp | 18 +++++++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 2c1150197..ba810b15c 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -178,7 +178,7 @@ touch $RCFILE if [ $user != "root" ]; then echo "uncomment runuser in rc.local, amazon AMI" - sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 + sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 fi echo "!!!Module Installation Successfully Completed!!!" diff --git a/procmon/main.cpp b/procmon/main.cpp index 6d7406ee1..02ad7abbb 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -62,6 +62,7 @@ string USER = "root"; string PMwithUM = "n"; bool startProcMon = false; string tmpLogDir; +string SUDO = ""; //extern std::string gOAMParentModuleName; extern bool gOAMParentModuleFlag; @@ -164,9 +165,12 @@ int main(int argc, char** argv) int user; user = getuid(); - if (user != 0) + if (user != 0) + { rootUser = false; - + SUDO = "sudo "; + } + char* p = getenv("USER"); if (p && *p) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 482b5c2bb..95b2fd503 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -54,6 +54,7 @@ extern bool HDFS; extern string PMwithUM; extern bool startProcMon; extern string tmpLogDir; +extern string SUDO; //std::string gOAMParentModuleName; bool gOAMParentModuleFlag; @@ -1822,9 +1823,9 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string cmd = "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpMount + "2>&1"; system(cmd.c_str()); - if ( !rootUser) + if ( !rootUser ) { - cmd = "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /dev/null 2>&1"; + cmd = SUDO + "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /dev/null 2>&1"; system(cmd.c_str()); } @@ -5916,7 +5917,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + cmd = SUDO + "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); @@ -5949,7 +5950,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + cmd = SUDO + "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); return true; @@ -6016,13 +6017,16 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - - log.writeLog(__LINE__, "amazonVolumeCheck function , volume to attached: " + volumeName, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "amazonVolumeCheck function , volume to attached: " + volumeName, LOG_TYPE_DEBUG); string cmd = "mount " + startup::StartUp::installDir() + "/data" + oam.itoa(dbrootID) + " > /dev/null"; system(cmd.c_str()); log.writeLog(__LINE__, "amazonVolumeCheck function , volume to mounted: " + volumeName, LOG_TYPE_DEBUG); + + cmd = SUDO + "chown mysql:mysql -R " + startup::StartUp::installDir() + "/data" + oam.itoa(dbrootID); + system(cmd.c_str()); + return true; } else @@ -6326,7 +6330,7 @@ int ProcessMonitor::checkDataMount() if ( !rootUser) { - cmd = "chown -R " + USER + ":" + USER + " " + dbroot + " > /dev/null 2>&1"; + cmd = SUDO + "chown -R " + USER + ":" + USER + " " + dbroot + " > /dev/null 2>&1"; system(cmd.c_str()); } From 3e4b869a258f489041c16b7132207cf0b000654c Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 16:25:11 -0500 Subject: [PATCH 098/127] MCOL-520 - sudo testing amazon --- oamapps/postConfigure/postConfigure.cpp | 2 +- procmon/processmonitor.cpp | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 622350ded..943005c1d 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1477,7 +1477,7 @@ int main(int argc, char* argv[]) // setup to start on reboot, for non-root amazon installs if ( !rootUser ) { - system("sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); + system("sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); } if ( !writeConfig(sysConfig) ) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 95b2fd503..410fd747f 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1035,7 +1035,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO cacheutils::dropPrimProcFdCache(); flushInodeCache(); - string cmd = "umount " + startup::StartUp::installDir() + "/data* -l > /dev/null 2>&1"; + string cmd = SUDO + "umount " + startup::StartUp::installDir() + "/data* -l > /dev/null 2>&1"; system(cmd.c_str()); sleep(1); @@ -1752,7 +1752,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string tmpUmount = tmpLogDir + "/umount.log"; for ( ; retry < 5 ; retry++) { - string cmd = "export LC_ALL=C;umount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpUmount + " 2>&1"; + string cmd = SUDO + "export LC_ALL=C;umount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpUmount + " 2>&1"; system(cmd.c_str()); @@ -1820,7 +1820,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if (DataRedundancyConfig == "n") { string tmpMount = tmpLogDir + "/mount.log"; - string cmd = "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpMount + "2>&1"; + string cmd = SUDO + "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpMount + "2>&1"; system(cmd.c_str()); if ( !rootUser ) @@ -5912,7 +5912,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) { string cmd; string mountLog = tmpLogDir + "/um_mount.log"; - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > " + mountLog; + cmd = SUDO + "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,user > " + mountLog; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); @@ -5946,7 +5946,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null 2>&1"; + string cmd = SUDO + "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,user > /dev/null 2>&1"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); @@ -6019,7 +6019,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) { log.writeLog(__LINE__, "amazonVolumeCheck function , volume to attached: " + volumeName, LOG_TYPE_DEBUG); - string cmd = "mount " + startup::StartUp::installDir() + "/data" + oam.itoa(dbrootID) + " > /dev/null"; + string cmd = SUDO + "mount " + startup::StartUp::installDir() + "/data" + oam.itoa(dbrootID) + " > /dev/null"; system(cmd.c_str()); log.writeLog(__LINE__, "amazonVolumeCheck function , volume to mounted: " + volumeName, LOG_TYPE_DEBUG); @@ -6106,7 +6106,7 @@ void ProcessMonitor::unmountExtraDBroots() { if ( DataRedundancyConfig == "n" ) { - string cmd = "umount " + startup::StartUp::installDir() + "/data" + oam.itoa(id) + " > /dev/null 2>&1"; + string cmd = SUDO + "umount " + startup::StartUp::installDir() + "/data" + oam.itoa(id) + " > /dev/null 2>&1"; system(cmd.c_str()); } else @@ -6288,7 +6288,7 @@ int ProcessMonitor::checkDataMount() if ( DataRedundancyConfig == "n" ) { //remove any local check flag for starters - string cmd = "umount " + dbroot + " > " + tmpLogDir + "/umount.log 2>&1"; + string cmd = SUDO + "umount " + dbroot + " > " + tmpLogDir + "/umount.log 2>&1"; system(cmd.c_str()); unlink(fileName.c_str()); @@ -6302,7 +6302,7 @@ int ProcessMonitor::checkDataMount() { // not mounted, mount string mountLog = tmpLogDir + "/mount.log"; - cmd = "export LC_ALL=C;mount " + dbroot + " > " + mountLog + " 2>&1"; + cmd = SUDO + "export LC_ALL=C;mount " + dbroot + " > " + mountLog + " 2>&1"; system(cmd.c_str()); ifstream in(mountLog.c_str()); @@ -6505,7 +6505,7 @@ int ProcessMonitor::glusterAssign(std::string dbrootID) } string tmpLog = tmpLogDir + "/glusterAssign.log"; - command = "mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpLog + " 2>&1"; + command = SUDO + "mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpLog + " 2>&1"; int ret = system(command.c_str()); @@ -6550,7 +6550,7 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) string tmpLog = tmpLogDir + "/glusterUnassign.log"; - command = "umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpLog + " 2>&1"; + command = SUDO + "umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpLog + " 2>&1"; int ret = system(command.c_str()); From 6c3b6cd4703446c667de7368381c2c69f8ebf232 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 30 Oct 2018 16:49:42 -0500 Subject: [PATCH 099/127] MCOL-520 - fix um device name --- oam/oamcpp/liboamcpp.cpp | 1 - oamapps/postConfigure/postConfigure.cpp | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 921488a0c..14b601cfb 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -5658,7 +5658,6 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s cout << endl << "ERROR: umountDBRoot api failure" << endl; exceptionControl("manualMovePmDbroot", API_FAILURE); } - } //check for amazon moving required diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 943005c1d..0d239af2a 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -2837,8 +2837,8 @@ int main(int argc, char* argv[]) callFree(pcommand); } - //get device name based on DBRoot ID - deviceName = "/dev/sdf"; + //get amazon device name for UM + deviceName = "/dev/xvdf"; } else { From 84c62d4fd130651443d2d3bad0dbeef7429a87aa Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 1 Nov 2018 08:59:02 -0500 Subject: [PATCH 100/127] MCOL-520 - sudo for mount amazon --- procmon/processmonitor.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 410fd747f..c9099b910 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1752,7 +1752,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string tmpUmount = tmpLogDir + "/umount.log"; for ( ; retry < 5 ; retry++) { - string cmd = SUDO + "export LC_ALL=C;umount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpUmount + " 2>&1"; + string cmd = "export LC_ALL=C;" + SUDO + "umount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpUmount + " 2>&1"; system(cmd.c_str()); @@ -1820,7 +1820,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if (DataRedundancyConfig == "n") { string tmpMount = tmpLogDir + "/mount.log"; - string cmd = SUDO + "export LC_ALL=C;mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpMount + "2>&1"; + string cmd = SUDO + "export LC_ALL=C;" + SUDO + "mount " + startup::StartUp::installDir() + "/data" + dbrootID + " > " + tmpMount + "2>&1"; system(cmd.c_str()); if ( !rootUser ) @@ -6302,7 +6302,7 @@ int ProcessMonitor::checkDataMount() { // not mounted, mount string mountLog = tmpLogDir + "/mount.log"; - cmd = SUDO + "export LC_ALL=C;mount " + dbroot + " > " + mountLog + " 2>&1"; + cmd = "export LC_ALL=C;" + SUDO + "mount " + dbroot + " > " + mountLog + " 2>&1"; system(cmd.c_str()); ifstream in(mountLog.c_str()); From efc69d86f454010108ecce069ad6f6a29283b663 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 1 Nov 2018 09:41:10 -0500 Subject: [PATCH 101/127] MCOL-520 - sudo for mount amazon --- procmon/processmonitor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index c9099b910..242df76c1 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -5156,7 +5156,7 @@ int ProcessMonitor::changeMyCnf(std::string type) system(cmd.c_str()); - cmd = "chown mysql:mysql " + mycnfFile + " >/dev/null 2>&1"; + cmd = "chown " + USER + ":" + USER + " " + mycnfFile + " >/dev/null 2>&1"; system(cmd.c_str()); @@ -5917,7 +5917,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = SUDO + "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + cmd = SUDO + "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); @@ -5950,7 +5950,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = SUDO + "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + cmd = SUDO + "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); return true; @@ -6024,7 +6024,7 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) system(cmd.c_str()); log.writeLog(__LINE__, "amazonVolumeCheck function , volume to mounted: " + volumeName, LOG_TYPE_DEBUG); - cmd = SUDO + "chown mysql:mysql -R " + startup::StartUp::installDir() + "/data" + oam.itoa(dbrootID); + cmd = SUDO + "chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/data" + oam.itoa(dbrootID); system(cmd.c_str()); return true; From ac7035e1cdc6bfee70145af8cfd1b641dd3cb82d Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 1 Nov 2018 09:53:53 -0500 Subject: [PATCH 102/127] MCOL-520 - sudo for mount amazon --- oam/install_scripts/module_installer.sh | 4 ++-- oamapps/postConfigure/mycnfUpgrade.cpp | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index ba810b15c..cfd52161d 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -93,8 +93,8 @@ if [ $module = "um" ]; then echo "Setup UM Volume Mount" device=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation UMVolumeDeviceName$mid` mkdir -p $COLUMNSTORE_INSTALL_DIR/mysql/db > /dev/null 2>&1 - mount $device $COLUMNSTORE_INSTALL_DIR/mysql/db -t ext2 -o noatime,nodiratime,noauto - chown mysql:mysql -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1 + sudo mount $device $COLUMNSTORE_INSTALL_DIR/mysql/db -t ext2 -o noatime,nodiratime,noauto,user + sudo chown $user:$user -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1 fi fi fi diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index afe59e742..6d2967a43 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -212,7 +212,14 @@ int main(int argc, char* argv[]) } } - string cmd = "chown mysql:mysql " + mycnfFile; + string USER = "mysql"; + + char* p = getenv("USER"); + + if (p && *p) + USER = p; + + string cmd = "chown " + USER + ":" + USER + " " + mycnfFile; system(cmd.c_str()); exit (0); From 1d56ed7a933c78bef1e56955c445126bec14c32f Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 1 Nov 2018 10:27:43 -0500 Subject: [PATCH 103/127] MCOL-520 - fix amazon login issue --- procmgr/processmanager.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 8591f1f37..60a00e640 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -5131,8 +5131,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string cmd = installDir + "/bin/remote_command.sh " + IPAddr + " " + password + " 'ls' 1 > " + loginTmp; int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) - { + if (!oam.checkLogStatus("/tmp/login_test.log", "README")) { //check for RSA KEY ISSUE and fix if (oam.checkLogStatus(loginTmp, "Host key verification failed")) { @@ -5141,10 +5140,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str system(cmd.c_str()); } - log.writeLog(__LINE__, "addModule - login failed, retry login test: " + moduleName, LOG_TYPE_DEBUG); - sleep(10); - continue; - } + log.writeLog(__LINE__, "addModule - login failed, retry login test: " + moduleName, LOG_TYPE_DEBUG); + sleep(10); + continue; + } // logged in break; From 59b887f9212946dc94a2401d9c7d89a22e9df4e0 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 1 Nov 2018 10:59:03 -0500 Subject: [PATCH 104/127] MCOL-520 - fix amazon login issue --- procmgr/processmanager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 60a00e640..550ebeb01 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -5131,7 +5131,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string cmd = installDir + "/bin/remote_command.sh " + IPAddr + " " + password + " 'ls' 1 > " + loginTmp; int rtnCode = system(cmd.c_str()); - if (!oam.checkLogStatus("/tmp/login_test.log", "README")) { + if (!oam.checkLogStatus(loginTmp, "README")) { //check for RSA KEY ISSUE and fix if (oam.checkLogStatus(loginTmp, "Host key verification failed")) { From daae3af7aeee383533b319629622d8941a93302f Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 1 Nov 2018 14:36:41 -0500 Subject: [PATCH 105/127] MCOL-520 - fix amazon login issue --- oamapps/postConfigure/postConfigure.cpp | 2 +- procmgr/processmanager.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 0d239af2a..2def8d11b 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4651,7 +4651,7 @@ bool createDbrootDirs(string DBRootStorageType, bool amazonInstall) // mount data1 and create directories if configured with storage if ( DBRootStorageType == "external" ) { - string cmd = "mount " + installDir + "/data1 > " + tmpDir + "/mount.txt 2>&1"; + string cmd = "mount " + installDir + "/data1 > " + tmpDir + "/mount.log 2>&1"; system(cmd.c_str()); if ( !rootUser) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 550ebeb01..c218fee46 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -5131,6 +5131,11 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string cmd = installDir + "/bin/remote_command.sh " + IPAddr + " " + password + " 'ls' 1 > " + loginTmp; int rtnCode = system(cmd.c_str()); +log.writeLog(__LINE__, "loginTmp " + loginTmp, LOG_TYPE_DEBUG); +log.writeLog(__LINE__, "cmd " + cmd, LOG_TYPE_DEBUG); +log.writeLog(__LINE__, "rtnCode " + oam.itoa(rtnCode), LOG_TYPE_DEBUG); + + if (!oam.checkLogStatus(loginTmp, "README")) { //check for RSA KEY ISSUE and fix if (oam.checkLogStatus(loginTmp, "Host key verification failed")) From 7f16ad644e8d93ef70d4154f24f27a2cf8e61b62 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 1 Nov 2018 15:06:45 -0500 Subject: [PATCH 106/127] MCOL-520 - default password to ssh for addmodule --- oamapps/mcsadmin/mcsadmin.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 6089abc69..060015053 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -5680,7 +5680,7 @@ int processCommand(string* arguments) string moduleType; string moduleName; int moduleCount; - string password; + string password = "ssh"; typedef std::vector inputNames; inputNames inputnames; typedef std::vector umStorageNames; @@ -5705,7 +5705,7 @@ int processCommand(string* arguments) hostArg = 3; //check for a non-distrubuted install setup, dont need password - if ( DistributedInstall == "y" ) + if ( DistributedInstall != "y" ) { if (arguments[4] != "") password = arguments[4]; @@ -5736,7 +5736,7 @@ int processCommand(string* arguments) hostArg = 2; //check for a non-distrubuted install setup, dont need password - if ( DistributedInstall == "y" ) + if ( DistributedInstall != "y" ) { if (arguments[3] != "") password = arguments[3]; From 3e17f0ce5b0c5ad975785d3aee425e44ff57c6df Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 2 Nov 2018 10:52:11 -0500 Subject: [PATCH 107/127] MCOL-520 - add in the mariadb upgrade code --- oam/install_scripts/module_installer.sh | 21 +++----- oamapps/postConfigure/postConfigure.cpp | 13 ++++- procmgr/processmanager.cpp | 28 ++-------- procmon/processmonitor.cpp | 72 +++++++++++++++++++------ procmon/processmonitor.h | 2 +- 5 files changed, 82 insertions(+), 54 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index cfd52161d..b089e0989 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -98,6 +98,14 @@ if [ $module = "um" ]; then fi fi fi + + #setup rc.local + RCFILE=/etc/rc.d/rc.local + + if [ $user != "root" ]; then + echo "uncomment runuser in rc.local, amazon AMI" + sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 + fi fi #if pm, create dbroot directories @@ -168,19 +176,6 @@ if [ $user == "root" ]; then fi fi -#setup rc.local -if [ -f /etc/rc.d ]; then - RCFILE=/etc/rc.d/rc.local -else - RCFILE=/etc/rc.local -fi -touch $RCFILE - -if [ $user != "root" ]; then - echo "uncomment runuser in rc.local, amazon AMI" - sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 -fi - echo "!!!Module Installation Successfully Completed!!!" exit 0 diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 2def8d11b..550741bbe 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4056,7 +4056,18 @@ int main(int argc, char* argv[]) { if ( oam.checkLogStatus(dbbuilderLog, "System catalog appears to exist") ) { - cout.flush(); + cout << endl << "Run MariaDB Upgrade.. "; + cout.flush(); + + //send message to procmon's to run upgrade script + int status = sendUpgradeRequest(IserverTypeInstall, pmwithum); + + if ( status != 0 ) { + cout << endl << "MariaDB Columnstore Install Failed" << endl << endl; + exit(1); + } + else + cout << " DONE" << endl; } else { diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index c218fee46..3fe88d1b2 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -5107,10 +5107,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } //wait until login is success until continuing or fail if can't login - log.writeLog(__LINE__, "addModule - Successfully Launch of new Instance, retry login test: " + moduleName, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "addModule - Successfully Launch of new Instance, perform login test: " + moduleName, LOG_TYPE_DEBUG); int retry = 0; - for ( ; retry < 18 ; retry++) + for ( ; retry < 30 ; retry++) { IPAddr = oam.getEC2InstanceIpAddress(hostName); @@ -5123,7 +5123,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str if (IPAddr == "stopped") { - sleep(10); + sleep(5); continue; } @@ -5131,11 +5131,6 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string cmd = installDir + "/bin/remote_command.sh " + IPAddr + " " + password + " 'ls' 1 > " + loginTmp; int rtnCode = system(cmd.c_str()); -log.writeLog(__LINE__, "loginTmp " + loginTmp, LOG_TYPE_DEBUG); -log.writeLog(__LINE__, "cmd " + cmd, LOG_TYPE_DEBUG); -log.writeLog(__LINE__, "rtnCode " + oam.itoa(rtnCode), LOG_TYPE_DEBUG); - - if (!oam.checkLogStatus(loginTmp, "README")) { //check for RSA KEY ISSUE and fix if (oam.checkLogStatus(loginTmp, "Host key verification failed")) @@ -5146,7 +5141,7 @@ log.writeLog(__LINE__, "rtnCode " + oam.itoa(rtnCode), LOG_TYPE_DEBUG); } log.writeLog(__LINE__, "addModule - login failed, retry login test: " + moduleName, LOG_TYPE_DEBUG); - sleep(10); + sleep(5); continue; } @@ -5154,7 +5149,7 @@ log.writeLog(__LINE__, "rtnCode " + oam.itoa(rtnCode), LOG_TYPE_DEBUG); break; } - if ( retry >= 18 ) + if ( retry >= 30 ) { log.writeLog(__LINE__, "addModule - Failed to log in to Instance: " + hostName, LOG_TYPE_ERROR); pthread_mutex_unlock(&THREAD_LOCK); @@ -5546,19 +5541,6 @@ log.writeLog(__LINE__, "rtnCode " + oam.itoa(rtnCode), LOG_TYPE_DEBUG); setStandbyModule(remoteModuleName, false); } - //set root password - if (amazon) - { - cmd = startup::StartUp::installDir() + "/bin/remote_command.sh " + remoteModuleIP + " " + password + " '/root/.scripts/updatePassword.sh " + password + "' > " + tmpLogDir + "/password_change.log"; - log.writeLog(__LINE__, "addModule - cmd: " + cmd, LOG_TYPE_DEBUG); - int rtnCode = system(cmd.c_str()); - - if (WEXITSTATUS(rtnCode) == 0) - log.writeLog(__LINE__, "addModule - update root password: " + remoteModuleName, LOG_TYPE_DEBUG); - else - log.writeLog(__LINE__, "addModule - ERROR: update root password: " + remoteModuleName, LOG_TYPE_DEBUG); - } - //default string binaryInstallDir = installDir; diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 242df76c1..df72c5c4d 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1710,27 +1710,26 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO break; } + case RUNUPGRADE: + { + log.writeLog(__LINE__, "MSG RECEIVED: Run upgrade script "); - /* case RUNUPGRADE: - { - log.writeLog(__LINE__, "MSG RECEIVED: Run upgrade script "); + string mysqlpw; + msg >> mysqlpw; - string mysqlpw; - msg >> mysqlpw; + // run upgrade script + int ret = runUpgrade(mysqlpw); - // run upgrade script - int ret = runUpgrade(mysqlpw); + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) RUNUPGRADE; + ackMsg << (ByteStream::byte) ret; + mq.write(ackMsg); - ackMsg << (ByteStream::byte) ACK; - ackMsg << (ByteStream::byte) RUNUPGRADE; - ackMsg << (ByteStream::byte) ret; - mq.write(ackMsg); + log.writeLog(__LINE__, "RUNUPGRADE: ACK back to ProcMgr return status = " + oam.itoa((int) ret)); - log.writeLog(__LINE__, "RUNUPGRADE: ACK back to ProcMgr return status = " + oam.itoa((int) ret)); - - break; - } - */ + break; + } + case PROCUNMOUNT: { string dbrootID; @@ -6578,6 +6577,47 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) return oam::API_SUCCESS; } +/****************************************************************************************** +* @brief runUpgrade +* +* purpose: run upgrade script +* +******************************************************************************************/ +int ProcessMonitor::runUpgrade(std::string mysqlpw) +{ + Oam oam; + + string tmpLog = tmpLogDir + "/mysql_upgrade.log"; + + for ( int i = 0 ; i < 10 ; i++ ) + { + //run upgrade script + string cmd = startup::StartUp::installDir() + "/mysql/bin/mysql_upgrade --defaults-file=" + startup::StartUp::installDir() + "/mysql/my.cnf --password=" + + mysqlpw + " > " + tmpLog + " 2>&1"; + int retCode = system(cmd.c_str()); + + if ( retCode == 0 ) { + log.writeLog(__LINE__, "mysql_upgrade.sh: Successful return", LOG_TYPE_DEBUG); + return oam::API_SUCCESS; + } + else { + if (oam.checkLogStatus(cmd, "ERROR 1045") ) { + log.writeLog(__LINE__, "mysql_upgrade.sh: Missing Password error, return success", LOG_TYPE_DEBUG); + return oam::API_SUCCESS; + } + + log.writeLog(__LINE__, "mysql_upgrade.sh: Error return, check log " + tmpLog, LOG_TYPE_ERROR); + //restart mysqld and retry + try { + oam.actionMysqlCalpont(MYSQL_RESTART); + } + catch(...) + {} + sleep(1); + } + } + return oam::API_FAILURE; +} } //end of namespace diff --git a/procmon/processmonitor.h b/procmon/processmonitor.h index 5ec131d42..67c30f0a7 100644 --- a/procmon/processmonitor.h +++ b/procmon/processmonitor.h @@ -493,7 +493,7 @@ public: /** *@brief run upgrade script */ -// int runUpgrade(std::string mysqlpw); + int runUpgrade(std::string mysqlpw); /** *@brief change my.cnf From 1699f05ce65f1f6894be0f8cecbe20ede315571e Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 2 Nov 2018 14:22:16 -0500 Subject: [PATCH 108/127] MCOL-520 - add in the mariadb upgrade code --- oam/install_scripts/module_installer.sh | 14 +++++++------- procmon/processmonitor.cpp | 8 ++++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index b089e0989..ed3a55e36 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -97,14 +97,14 @@ if [ $module = "um" ]; then sudo chown $user:$user -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1 fi fi - fi - - #setup rc.local - RCFILE=/etc/rc.d/rc.local + + #setup rc.local + RCFILE=/etc/rc.d/rc.local - if [ $user != "root" ]; then - echo "uncomment runuser in rc.local, amazon AMI" - sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 + if [ $user != "root" ]; then + echo "uncomment runuser in rc.local, amazon AMI" + sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 + fi fi fi diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index df72c5c4d..9855c231c 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -6589,11 +6589,15 @@ int ProcessMonitor::runUpgrade(std::string mysqlpw) string tmpLog = tmpLogDir + "/mysql_upgrade.log"; + string passwordOption = " --password=" + mysqlpw; + if ( mysqlpw == oam::UnassignedName ) + passwordOption = ""; + for ( int i = 0 ; i < 10 ; i++ ) { //run upgrade script - string cmd = startup::StartUp::installDir() + "/mysql/bin/mysql_upgrade --defaults-file=" + startup::StartUp::installDir() + "/mysql/my.cnf --password=" + - mysqlpw + " > " + tmpLog + " 2>&1"; + string cmd = startup::StartUp::installDir() + "/mysql/bin/mysql_upgrade --defaults-file=" + startup::StartUp::installDir() + "/mysql/my.cnf " + + passwordOption + " > " + tmpLog + " 2>&1"; int retCode = system(cmd.c_str()); if ( retCode == 0 ) { From 23e5602496a742bbbec7d2a94f2b3688985a8ce4 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 2 Nov 2018 14:22:44 -0500 Subject: [PATCH 109/127] MCOL-520 - add in the mariadb upgrade code --- procmon/processmonitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 9855c231c..a7dd6732f 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -6598,8 +6598,8 @@ int ProcessMonitor::runUpgrade(std::string mysqlpw) //run upgrade script string cmd = startup::StartUp::installDir() + "/mysql/bin/mysql_upgrade --defaults-file=" + startup::StartUp::installDir() + "/mysql/my.cnf " + passwordOption + " > " + tmpLog + " 2>&1"; - int retCode = system(cmd.c_str()); + int retCode = system(cmd.c_str()); if ( retCode == 0 ) { log.writeLog(__LINE__, "mysql_upgrade.sh: Successful return", LOG_TYPE_DEBUG); return oam::API_SUCCESS; From 988c573166b8911a7e5a3e24632a9b4d7ec6bc80 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 2 Nov 2018 15:13:58 -0500 Subject: [PATCH 110/127] MCOL-520 - add in the mariadb upgrade code --- procmon/processmonitor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index a7dd6732f..c423c7d63 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -6599,6 +6599,7 @@ int ProcessMonitor::runUpgrade(std::string mysqlpw) string cmd = startup::StartUp::installDir() + "/mysql/bin/mysql_upgrade --defaults-file=" + startup::StartUp::installDir() + "/mysql/my.cnf " + passwordOption + " > " + tmpLog + " 2>&1"; + log.writeLog(__LINE__, "runUpgrade, cmd = " + cmd, LOG_TYPE_DEBUG); int retCode = system(cmd.c_str()); if ( retCode == 0 ) { log.writeLog(__LINE__, "mysql_upgrade.sh: Successful return", LOG_TYPE_DEBUG); From cf6820d3424fa965f2103cce6b7c4bdc7f1661a7 Mon Sep 17 00:00:00 2001 From: David Hill Date: Sat, 3 Nov 2018 17:06:36 -0500 Subject: [PATCH 111/127] MCOL-520 - added in mysql_upgrade script and fix issue with shared memory not cleared at shutdown --- oam/install_scripts/module_installer.sh | 18 +++++++++--------- oam/oamcpp/liboamcpp.cpp | 5 +---- .../columnstoreSupport/columnstoreSupport.cpp | 13 +++++++++++++ oamapps/postConfigure/helpers.cpp | 4 +--- oamapps/postConfigure/postConfigure.cpp | 1 - procmgr/processmanager.cpp | 9 +++++++++ procmon/processmonitor.cpp | 15 +++++++-------- procmon/processmonitor.h | 2 +- 8 files changed, 41 insertions(+), 26 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index ed3a55e36..0e116a597 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -84,8 +84,8 @@ test -f $COLUMNSTORE_INSTALL_DIR/post/functions && . $COLUMNSTORE_INSTALL_DIR/po mid=`module_id` #if um, cloud, separate system type, external um storage, then setup mount -if [ $module = "um" ]; then - if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then +if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then + if [ $module = "um" ]; then systemtype=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation ServerTypeInstall` if [ $systemtype = "1" ]; then umstoragetype=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation UMStorageType` @@ -97,14 +97,14 @@ if [ $module = "um" ]; then sudo chown $user:$user -R $COLUMNSTORE_INSTALL_DIR/mysql > /dev/null 2>&1 fi fi - - #setup rc.local - RCFILE=/etc/rc.d/rc.local + fi - if [ $user != "root" ]; then - echo "uncomment runuser in rc.local, amazon AMI" - sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 - fi + #setup rc.local for amazon + RCFILE=/etc/rc.d/rc.local + + if [ $user != "root" ]; then + echo "uncomment runuser in rc.local, amazon AMI" + sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1 fi fi diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 14b601cfb..a8a9bfd3d 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -9396,9 +9396,6 @@ int Oam::checkGlusterLog(std::string logFile, std::string& msg) ******************************************************************************************/ std::string Oam::getMySQLPassword() { - - return oam::UnassignedName; - string mysqlUser = "root"; string USER = "root"; @@ -9422,7 +9419,7 @@ std::string Oam::getMySQLPassword() if (!file) { writeLog("getMySQLPassword: doesn't exist: " + fileName, LOG_TYPE_DEBUG); - exceptionControl("getMySQLPassword", API_FILE_OPEN_ERROR); + return oam::UnassignedName; } char line[400]; diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp index 0232cd9c6..05fe17445 100644 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -305,6 +305,19 @@ void reportThread(string reporttype) cmd = installDir + "/bin/mcsadmin getSystemInfo >> " + outputFile; system(cmd.c_str()); + cmd = "echo ' ' >> " + outputFile; + system(cmd.c_str()); + cmd = "echo '******************** System Directories ********************' >> " + outputFile; + system(cmd.c_str()); + cmd = "echo ' ' >> " + outputFile; + system(cmd.c_str()); + cmd = "echo '################# mcsadmin getSystemDirectories ################# ' >> " + outputFile; + system(cmd.c_str()); + cmd = "echo ' ' >> " + outputFile; + system(cmd.c_str()); + cmd = installDir + "/bin/mcsadmin getSystemDirectories >> " + outputFile; + system(cmd.c_str()); + cmd = "echo ' ' >> " + outputFile; system(cmd.c_str()); cmd = "echo '******************** System Configuration File ********************' >> " + outputFile; diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index 0392979ee..1a7941f1f 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -39,7 +39,6 @@ using namespace installer; #include "installdir.h" -extern string mysqlpw; string pwprompt = " "; string masterLogFile = oam::UnassignedName; @@ -347,7 +346,6 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) ByteStream::byte requestID = RUNUPGRADE; msg << requestID; - msg << mysqlpw; int returnStatus = oam::API_SUCCESS; @@ -385,7 +383,7 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) { string tmpDir = startup::StartUp::tmpDir(); - cout << "ERROR: Error return in running the MariDB Columnstore Upgrade, check " + tmpDir + "/upgrade*.logs on " << (*pt).DeviceName << endl; + cout << "ERROR: Error return in running the MariDB Columnstore Upgrade, check " + tmpDir + "/mysql_upgrade.log on " << (*pt).DeviceName << endl; return returnStatus; } } diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 550741bbe..da3977257 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -200,7 +200,6 @@ string tmpDir; string HOME = "/root"; string SUDO = ""; extern string pwprompt; -string mysqlpw = oam::UnassignedName; extern const char* pcommand; extern string prompt; diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 3fe88d1b2..3203c67b2 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -1502,6 +1502,15 @@ void processMSG(messageqcpp::IOSocket* cfIos) cmd = "touch " + startup::StartUp::installDir() + "/local/moveDbrootTransactionLog"; system(cmd.c_str()); + //clear shared memory + cmd = startup::StartUp::installDir() + "/bin/clearShm > /dev/null 2>&1"; + int rtnCode = system(cmd.c_str()); + + if (WEXITSTATUS(rtnCode) != 1) + log.writeLog(__LINE__, "Successfully ran DBRM clearShm", LOG_TYPE_DEBUG); + else + log.writeLog(__LINE__, "Error running DBRM clearShm", LOG_TYPE_ERROR); + // now do local module processManager.shutdownModule(config.moduleName(), graceful, manualFlag); diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index c423c7d63..2f6a5ab9d 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1714,11 +1714,8 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO { log.writeLog(__LINE__, "MSG RECEIVED: Run upgrade script "); - string mysqlpw; - msg >> mysqlpw; - // run upgrade script - int ret = runUpgrade(mysqlpw); + int ret = runUpgrade(); ackMsg << (ByteStream::byte) ACK; ackMsg << (ByteStream::byte) RUNUPGRADE; @@ -6583,15 +6580,17 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) * purpose: run upgrade script * ******************************************************************************************/ -int ProcessMonitor::runUpgrade(std::string mysqlpw) +int ProcessMonitor::runUpgrade() { Oam oam; string tmpLog = tmpLogDir + "/mysql_upgrade.log"; - string passwordOption = " --password=" + mysqlpw; - if ( mysqlpw == oam::UnassignedName ) - passwordOption = ""; + string mysqlpw = oam.getMySQLPassword(); + + string passwordOption = passwordOption = ""; + if ( mysqlpw != oam::UnassignedName ) + passwordOption = " --password=" + mysqlpw; for ( int i = 0 ; i < 10 ; i++ ) { diff --git a/procmon/processmonitor.h b/procmon/processmonitor.h index 67c30f0a7..e8c458006 100644 --- a/procmon/processmonitor.h +++ b/procmon/processmonitor.h @@ -493,7 +493,7 @@ public: /** *@brief run upgrade script */ - int runUpgrade(std::string mysqlpw); + int runUpgrade(); /** *@brief change my.cnf From cf181bb39a2fc096b38622983dda25f361eb2d32 Mon Sep 17 00:00:00 2001 From: David Hill Date: Sat, 3 Nov 2018 17:38:17 -0500 Subject: [PATCH 112/127] MCOL-520 - fix non-root cluster test issues --- oamapps/postConfigure/postConfigure.cpp | 2 +- .../clusterTester/columnstoreClusterTester.sh | 61 ++++--------------- 2 files changed, 12 insertions(+), 51 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index da3977257..1b0771bd7 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4055,7 +4055,7 @@ int main(int argc, char* argv[]) { if ( oam.checkLogStatus(dbbuilderLog, "System catalog appears to exist") ) { - cout << endl << "Run MariaDB Upgrade.. "; + cout << endl << "Run MariaDB Server Upgrade.. "; cout.flush(); //send message to procmon's to run upgrade script diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index b426b8584..887ed9af7 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -267,31 +267,17 @@ checkLocalDir() # Non-root User directory permissions check # echo "" - echo "** Run Non-root User directory permissions check on Local Node" + echo "** Run Non-root User directory permissions check on Local Node (dev/shm)" echo "" - #remove any check tmp files from previous runs - `rm -f ${tmpDir}/*_check > /dev/null 2>&1` - - #check ${tmpDir} and /dev/shm - pass=true - `touch ${tmpDir}/cs_check > /dev/null 2>&1` - if [ "$?" -eq 0 ]; then - echo "Local Node permission test on ${tmpDir} : Passed" - `rm -f ${tmpDir}/cs_check` - else - echo "Local Node permission test on ${tmpDir} : ${bold}Failed${normal}, change permissions to 777 and re-test" - exit 1 - fi - `touch /dev/shm/cs_check > /dev/null 2>&1` if [ "$?" -eq 0 ]; then - echo "Local Node permission test on /dev/shm : Passed" - `rm -f /dev/shm/cs_check` + echo "Local Node permission test on /dev/shm : Passed" + `rm -f /dev/shm/cs_check` else - echo "Local Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test" - pass=false - REPORTPASS=false + echo "Local Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test" + pass=false + REPORTPASS=false fi fi } @@ -348,39 +334,14 @@ checkSSH() checkRemoteDir() { - # - # remove old _check tmp files from remote servers - - `rm -f ${tmpDir}/*_check > /dev/null 2>&1` - - for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f ${tmpDir}/*_check > /dev/null 2>&1' 1 > ${tmpDir}/remote_command_check 2>&1` - done if [ "$USER" != "root" ]; then # Non-root User directory permissions check # echo "" - echo "** Run Non-root User directory permissions check on remote nodes" + echo "** Run Non-root User directory permissions check on remote nodes (/dev/shm)" echo "" - for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch ${tmpDir}/cs_check' 1 > ${tmpDir}/remote_command_check 2>&1` - rc="$?" - if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then - `grep "Permission denied" ${tmpDir}/remote_command_check > /dev/null 2>&1` - if [ "$?" -eq 0 ]; then - echo "$ipadd Node permission test on ${tmpDir} : ${bold}Failed${normal}, change permissions to 777 and re-test" - exit 1 - else - echo "$ipadd Node permission test on ${tmpDir} : Passed" - fi - else - echo "Error running remote_command.sh to $ipadd Node, check ${tmpDir}/remote_command_check" - pass=false - REPORTPASS=false - fi - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > ${tmpDir}/remote_command_check 2>&1` rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then @@ -464,10 +425,10 @@ checkLocale() pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > ${tmpDir}/locale_check 2>&1' 1 > ${tmpDir}/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > locale_check 2>&1' 1 > ${tmpDir}/remote_command_check` rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/locale_check > ${tmpDir}/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD locale_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" exit 1 @@ -758,14 +719,14 @@ checkTime() #get local epoch time localTime=`date +%s` for ipadd in "${NODE_IPADDRESS[@]}"; do - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'date +%s > ${tmpDir}/time_check' > ${tmpDir}/time_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'date +%s > time_check' > ${tmpDir}/time_check` rc="$?" if [ $rc -ne 0 ] ; then echo $ipadd " Node ${bold}Failed${normal} date/time check failed, check ${tmpDir}/time_check" pass=false REPORTPASS=false else - `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/time_check > ${tmpDir}/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD time_check > ${tmpDir}/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check" else From 0193cbf91d938e972ad99cb75a34a8840725cae7 Mon Sep 17 00:00:00 2001 From: David Hill Date: Sat, 3 Nov 2018 18:43:19 -0500 Subject: [PATCH 113/127] MCOL-520 - fix non-root supportReport issues --- oamapps/columnstoreSupport/columnstoreSupport.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp index 05fe17445..ccf7714c4 100644 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -58,6 +58,7 @@ string rootPassword = ""; string debug_flag = "0"; string mysqlpw = " "; string tmpDir; +string ProfileFile; int runningThreads = 0; pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; @@ -130,9 +131,10 @@ void childReportThread(threadInfo_t& st) cout.flush(); - string cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + rootPassword + " '" + + string cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + rootPassword + " '. " + ProfileFile + ";" + installDir + "/bin/" + reportType + "Report.sh " + remoteModuleName + " " + installDir + "' " + debug_flag + " - forcetty"; + int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) @@ -669,6 +671,17 @@ int main(int argc, char* argv[]) cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl; exit(-1); } + + //Get Profile file + try + { + ProfileFile = sysConfig->getConfig(InstallSection, "ProfileFile"); + } + catch (...) + { + cout << "ERROR: Problem getting ProfileFile" << endl; + exit(-1); + } string ModuleSection = "SystemModuleConfig"; From 5549f8b54b837a98ea3d71b0e6714da76c8446f2 Mon Sep 17 00:00:00 2001 From: David Hill Date: Sun, 4 Nov 2018 22:50:05 -0600 Subject: [PATCH 114/127] MCOL-520 - added back in the clearShm into the service script on stop --- oam/install_scripts/columnstore | 1 + 1 file changed, 1 insertion(+) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index bba8ca0fa..1ba4a2c1f 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -129,6 +129,7 @@ stop() { touch ${tmpDir}/StopColumnstore pkill ProcMon pkill ProcMgr + $InstallDir/bin/clearShm RETVAL=$? rm -f $lockdir/columnstore fuser -k 8604/tcp > /dev/null 2>&1 From 26774ecd903e89ac32e2483ede87cddff9ba5a35 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 5 Nov 2018 12:35:12 -0600 Subject: [PATCH 115/127] MCOL-520 - add in extra comments --- oam/install_scripts/pre-uninstall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index bfaac6603..633cd83f0 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -146,7 +146,8 @@ else cat < Date: Tue, 6 Nov 2018 08:50:47 -0600 Subject: [PATCH 116/127] MCOL-520 - add mysql upgrade to single server install --- oamapps/postConfigure/helpers.h | 2 +- oamapps/postConfigure/installer.cpp | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/oamapps/postConfigure/helpers.h b/oamapps/postConfigure/helpers.h index 8d79beb64..5f7b1631a 100644 --- a/oamapps/postConfigure/helpers.h +++ b/oamapps/postConfigure/helpers.h @@ -38,7 +38,7 @@ extern bool waitForActive(); extern void dbrmDirCheck(); extern void mysqlSetup(); extern int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeout ); -extern int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum); +extern int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum = false); extern int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pmwithum); extern void checkFilesPerPartion(int DBRootCount, Config* sysConfig); extern void checkMysqlPort( string& mysqlPort, Config* sysConfig); diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 044d6cb37..8c1751481 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -894,15 +894,31 @@ int main(int argc, char* argv[]) cout << endl << "System Catalog Successfull Created" << endl; else { - if ( !oam.checkLogStatus(logFile, "System catalog appears to exist") ) + if ( oam.checkLogStatus(logFile, "System catalog appears to exist") ) + { + cout << endl << "Run MariaDB Server Upgrade.. "; + cout.flush(); + + //send message to procmon's to run upgrade script + int status = sendUpgradeRequest(IserverTypeInstall); + + if ( status != 0 ) { + cout << endl << "MariaDB Columnstore Install Failed" << endl << endl; + exit(1); + } + else + cout << " DONE" << endl; + } + else { cout << endl << "System Catalog Create Failure" << endl; cout << "Check latest log file in " << logFile << endl; cout << " IMPORTANT: Once issue has been resolved, rerun postConfigure" << endl << endl; + exit (1); } - } - + } + cout << endl << "MariaDB ColumnStore Install Successfully Completed, System is Active" << endl << endl; cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; From f46250112bd29de7d18243a9e10e2533e71b66ed Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 6 Nov 2018 08:59:18 -0600 Subject: [PATCH 117/127] MCOL-520 - change to exit on invalid command line argument --- oam/install_scripts/post-install | 4 +++- oam/install_scripts/pre-uninstall | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index ba12ab186..a08a17d0a 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -28,7 +28,9 @@ for arg in "$@"; do elif [ `expr -- "$arg" : '--plugin='` -eq 9 ]; then plugin="`echo $arg | awk -F= '{print $2}'`" else - echo "post-install: ignoring unknown argument: $arg" 1>&2 + echo "post-install: invalid unknown argument: $arg" 1>&2 + echo "exiting..." + exit 1 fi done diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 633cd83f0..3154e1db4 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -2,7 +2,7 @@ # # $Id: post-uninstall 421 2007-04-05 15:46:55Z dhill $ # -# Post-uninstall steps for columnstore install +# pre-uninstall steps for columnstore install prefix=/usr/local installdir=$prefix/mariadb/columnstore @@ -26,7 +26,9 @@ for arg in "$@"; do elif [ `expr -- "$arg" : '--quiet'` -eq 7 ]; then quiet=1 else - echo "post-uninstall: ignoring unknown argument: $arg" 1>&2 + echo "pre-install: invalid unknown argument: $arg" 1>&2 + echo "exiting..." + exit 1 fi done From 3289fd2483e0bba124cb29d367d8830b485be247 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 6 Nov 2018 09:01:22 -0600 Subject: [PATCH 118/127] MCOL-520 - fix speeling of MariaDB --- oamapps/postConfigure/helpers.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index 1a7941f1f..75f0d43ec 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -217,7 +217,7 @@ void dbrmDirCheck() cout << "If the files were copied from " << dbrmrootPrevDir << " to " << dbrmrootDir << endl; cout << "you will need to edit the file BRM_saves_current to contain the current path of" << endl; cout << dbrmrootDir << endl << endl; - cout << "Please reference the MariDB Columnstore Installation Guide on Upgrade Installs for" << endl; + cout << "Please reference the MariaDB Columnstore Installation Guide on Upgrade Installs for" << endl; cout << "addition information, if needed." << endl << endl; while (true) @@ -383,7 +383,7 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum) { string tmpDir = startup::StartUp::tmpDir(); - cout << "ERROR: Error return in running the MariDB Columnstore Upgrade, check " + tmpDir + "/mysql_upgrade.log on " << (*pt).DeviceName << endl; + cout << "ERROR: Error return in running the MariaDB Columnstore Upgrade, check " + tmpDir + "/mysql_upgrade.log on " << (*pt).DeviceName << endl; return returnStatus; } } @@ -804,7 +804,7 @@ void checkMysqlPort( std::string& mysqlPort, Config* sysConfig ) if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl; + cout << "ERROR: Failed trying to update MariaDB Columnstore System Configuration file" << endl; exit(1); } @@ -949,7 +949,7 @@ void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl; + cout << "ERROR: Failed trying to update MariaDB Columnstore System Configuration file" << endl; exit(1); } From 5dd8bbd63436ca3ed05cea7ee6d9201054aa2d27 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 6 Nov 2018 12:10:51 -0600 Subject: [PATCH 119/127] MCOL-520 - moved mysql_upgrade to post-mysql-install --- oam/install_scripts/post-mysql-install | 25 ++++++++++++++++++++++++- oamapps/postConfigure/installer.cpp | 4 +++- oamapps/postConfigure/postConfigure.cpp | 4 +++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 6f99b2284..f70c3fc93 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -44,7 +44,7 @@ checkForError() { prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install -password= +password=" " pwprompt= for arg in "$@"; do if [ $(expr -- "$arg" : '--prefix=') -eq 9 ]; then @@ -57,6 +57,8 @@ for arg in "$@"; do prefix=$(dirname $installdir) elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then tmpdir="$(echo $arg | awk -F= '{print $2}')" + elif [ $(expr -- "$arg" : '--password=') -eq 11 ]; then + password="$(echo $arg | awk -F= '{print $2}')" else echo "ignoring unknown argument: $arg" 1>&2 fi @@ -103,6 +105,27 @@ if [ -x $installdir/mysql/mysql-Columnstore ]; then $installdir/mysql/mysql-Columnstore start sleep 5 + + # Run MariaDB (mysql) upgrade script, if it exist + if [ -x $installdir/mysql/bin/mysql_upgrade ]; then + if [[ ${password} == " " ]]; then + $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf > $tmpdir/mysql_upgrade.log + if [ $? -ne 0 ]; then + echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" + $installdir/mysql/mysql-Columnstore stop + sleep 2 + exit 2; + fi + else + $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf --password=$password > $tmpdir/mysql_upgrade.log + if [ $? -ne 0 ]; then + echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" + $installdir/mysql/mysql-Columnstore stop + sleep 2 + exit 2; + fi + fi + fi # Install various Calpont stuff... $installdir/mysql/install_calpont_mysql.sh --installdir=$installdir --tmpdir=$tmpdir diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 8c1751481..94850deae 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -896,7 +896,7 @@ int main(int argc, char* argv[]) { if ( oam.checkLogStatus(logFile, "System catalog appears to exist") ) { - cout << endl << "Run MariaDB Server Upgrade.. "; +/* cout << endl << "Run MariaDB Server Upgrade.. "; cout.flush(); //send message to procmon's to run upgrade script @@ -908,6 +908,8 @@ int main(int argc, char* argv[]) } else cout << " DONE" << endl; +*/ + cout.flush(); } else { diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1b0771bd7..8c4cd73c6 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4055,7 +4055,7 @@ int main(int argc, char* argv[]) { if ( oam.checkLogStatus(dbbuilderLog, "System catalog appears to exist") ) { - cout << endl << "Run MariaDB Server Upgrade.. "; +/* cout << endl << "Run MariaDB Server Upgrade.. "; cout.flush(); //send message to procmon's to run upgrade script @@ -4067,6 +4067,8 @@ int main(int argc, char* argv[]) } else cout << " DONE" << endl; +*/ + cout.flush(); } else { From 5d0458470419e87cb925a50859e7bd9979bfe388 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 6 Nov 2018 15:35:47 -0600 Subject: [PATCH 120/127] MCOL-520 - changes for mysql_upgrade requiring password --- oam/install_scripts/module_installer.sh | 12 +++++++++++- oamapps/postConfigure/helpers.cpp | 8 +++++++- procmon/main.cpp | 9 ++++++++- procmon/processmonitor.cpp | 2 +- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 0e116a597..c2c48af43 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -15,6 +15,7 @@ rpmmode=install user=`whoami 2>/dev/null` quiet=0 shiftcnt=0 +password=" " for arg in "$@"; do if [ $(expr -- "$arg" : '--prefix=') -eq 9 ]; then @@ -40,6 +41,9 @@ for arg in "$@"; do elif [ $(expr -- "$arg" : '--module') -eq 8 ]; then module="$(echo $arg | awk -F= '{print $2}')" ((shiftcnt++)) + elif [ $(expr -- "$arg" : '--password') -eq 10 ]; then + password="$(echo $arg | awk -F= '{print $2}')" + ((shiftcnt++)) fi done shift $shiftcnt @@ -155,9 +159,15 @@ if [ $module = "um" ] || ( [ $module = "pm" ] && [ $PMwithUM = "y" ] ) || [ $Ser echo "ERROR: post-mysqld-install failed: check ${tmpDir}/post-mysqld-install.log" exit 1 fi + echo "Run post-mysql-install" - $COLUMNSTORE_INSTALL_DIR/bin/post-mysql-install --installdir=$COLUMNSTORE_INSTALL_DIR --tmpdir=${tmpDir} > ${tmpDir}/post-mysql-install.log 2>&1 + mysqlPassword=" " + if [[ $password != " " ]]; then + mysqlPassword="--password="$password + fi + + $COLUMNSTORE_INSTALL_DIR/bin/post-mysql-install --installdir=$COLUMNSTORE_INSTALL_DIR $mysqlPassword --tmpdir=${tmpDir} > ${tmpDir}/post-mysql-install.log 2>&1 if [ $? -ne 0 ]; then echo "ERROR: post-mysql-install failed: check ${tmpDir}/post-mysql-install.log" exit 1 diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index 75f0d43ec..322ecb6c3 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -282,8 +282,14 @@ void mysqlSetup() if (p && *p) HOME = p; } + + string mysqlpw = oam.getMySQLPassword(); - cmd = installDir + "/bin/post-mysql-install --installdir=" + installDir + " --tmpdir=" + tmpDir + " > " + tmpDir + "/post-mysql-install.log"; + string passwordOption = ""; + if ( mysqlpw != oam::UnassignedName ) + passwordOption = " --password=" + mysqlpw; + + cmd = installDir + "/bin/post-mysql-install --installdir=" + installDir + " --tmpdir=" + tmpDir + " " + passwordOption + " > " + tmpDir + "/post-mysql-install.log"; rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) == 2) diff --git a/procmon/main.cpp b/procmon/main.cpp index 02ad7abbb..0d2d2ab4a 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -242,9 +242,16 @@ int main(int argc, char** argv) } string modType = config.moduleType(); + + string mysqlpw = oam.getMySQLPassword(); + + string passwordOption = ""; + if ( mysqlpw != oam::UnassignedName ) + passwordOption = " --password=" + mysqlpw; + //run the module install script - string cmd = startup::StartUp::installDir() + "/bin/module_installer.sh " + " --installdir=" + startup::StartUp::installDir() + " --module=" + modType + " > " + tmpLogDir + "/module_installer.log 2>&1"; + string cmd = startup::StartUp::installDir() + "/bin/module_installer.sh " + " --installdir=" + startup::StartUp::installDir() + " --module=" + modType + " " + passwordOption + " > " + tmpLogDir + "/module_installer.log 2>&1"; log.writeLog(__LINE__, "run module_installer.sh", LOG_TYPE_DEBUG); log.writeLog(__LINE__, cmd, LOG_TYPE_DEBUG); diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 2f6a5ab9d..b92ab2b6f 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -6588,7 +6588,7 @@ int ProcessMonitor::runUpgrade() string mysqlpw = oam.getMySQLPassword(); - string passwordOption = passwordOption = ""; + string passwordOption = ""; if ( mysqlpw != oam::UnassignedName ) passwordOption = " --password=" + mysqlpw; From 16e956db023e8a3bffb9565b016e4415ad18ab9b Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 6 Nov 2018 16:35:44 -0600 Subject: [PATCH 121/127] MCOL-520 - move mysql_upgrade --- oam/install_scripts/module_installer.sh | 15 ++++++----- oam/install_scripts/post-mysql-install | 5 +--- oam/install_scripts/post-mysqld-install | 36 +++++++++++++++++++++++++ oamapps/postConfigure/helpers.cpp | 16 +++++------ 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index c2c48af43..0f8d6ba14 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -153,8 +153,14 @@ fi # if um, run mysql install scripts if [ $module = "um" ] || ( [ $module = "pm" ] && [ $PMwithUM = "y" ] ) || [ $ServerTypeInstall = "2" ]; then + + mysqlPassword=" " + if [[ $password != " " ]]; then + mysqlPassword="--password="$password + fi + echo "Run post-mysqld-install" - $COLUMNSTORE_INSTALL_DIR/bin/post-mysqld-install --installdir=$COLUMNSTORE_INSTALL_DIR > ${tmpDir}/post-mysqld-install.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/post-mysqld-install --installdir=$COLUMNSTORE_INSTALL_DIR $mysqlPassword > ${tmpDir}/post-mysqld-install.log 2>&1 if [ $? -ne 0 ]; then echo "ERROR: post-mysqld-install failed: check ${tmpDir}/post-mysqld-install.log" exit 1 @@ -162,12 +168,7 @@ if [ $module = "um" ] || ( [ $module = "pm" ] && [ $PMwithUM = "y" ] ) || [ $Ser echo "Run post-mysql-install" - mysqlPassword=" " - if [[ $password != " " ]]; then - mysqlPassword="--password="$password - fi - - $COLUMNSTORE_INSTALL_DIR/bin/post-mysql-install --installdir=$COLUMNSTORE_INSTALL_DIR $mysqlPassword --tmpdir=${tmpDir} > ${tmpDir}/post-mysql-install.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/post-mysql-install --installdir=$COLUMNSTORE_INSTALL_DIR --tmpdir=${tmpDir} > ${tmpDir}/post-mysql-install.log 2>&1 if [ $? -ne 0 ]; then echo "ERROR: post-mysql-install failed: check ${tmpDir}/post-mysql-install.log" exit 1 diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index f70c3fc93..8a4c0291d 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -24,7 +24,7 @@ checkForError() { echo "checking for engine columnstore..." $installdir/mysql/bin/mysql \ --defaults-extra-file=$installdir/mysql/my.cnf \ - --user=root $pwprompt \ + --user=root \ --execute='show engines;' \ calpontsys | grep -i columnstore @@ -45,7 +45,6 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install password=" " -pwprompt= for arg in "$@"; do if [ $(expr -- "$arg" : '--prefix=') -eq 9 ]; then prefix="$(echo $arg | awk -F= '{print $2}')" @@ -57,8 +56,6 @@ for arg in "$@"; do prefix=$(dirname $installdir) elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then tmpdir="$(echo $arg | awk -F= '{print $2}')" - elif [ $(expr -- "$arg" : '--password=') -eq 11 ]; then - password="$(echo $arg | awk -F= '{print $2}')" else echo "ignoring unknown argument: $arg" 1>&2 fi diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index e712b2813..27eba3d42 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -8,6 +8,10 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install user=mysql + +password=" " +pwprompt= + if [ $EUID -ne 0 ]; then USER=`whoami 2>/dev/null` user=$USER @@ -24,6 +28,8 @@ for arg in "$@"; do prefix=`dirname $installdir` elif [ `expr -- "$arg" : '--user='` -eq 7 ]; then user="`echo $arg | awk -F= '{print $2}'`" + elif [ $(expr -- "$arg" : '--password=') -eq 11 ]; then + password="$(echo $arg | awk -F= '{print $2}')" else echo "post-mysqld-install: ignoring unknown argument: $arg" 1>&2 fi @@ -68,6 +74,36 @@ fi # InfiniDB testing hook... test -x /usr/local/bin/idb-testing-mysql-pre-start && /usr/local/bin/idb-testing-mysql-pre-start $installdir/mysql +if [ -x $installdir/mysql/mysql-Columnstore ]; then + # Restart in the same way that mysqld will be started normally. + $installdir/mysql/mysql-Columnstore stop >/dev/null 2>&1 + sleep 2 + $installdir/mysql/mysql-Columnstore start + + sleep 5 + + # Run MariaDB (mysql) upgrade script, if it exist + if [ -x $installdir/mysql/bin/mysql_upgrade ]; then + if [[ ${password} == " " ]]; then + $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf > $tmpdir/mysql_upgrade.log + if [ $? -ne 0 ]; then + echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" + $installdir/mysql/mysql-Columnstore stop + sleep 2 + exit 2; + fi + else + $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf --password=$password > $tmpdir/mysql_upgrade.log + if [ $? -ne 0 ]; then + echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" + $installdir/mysql/mysql-Columnstore stop + sleep 2 + exit 2; + fi + fi + fi +fi + ### Don't give the user the notes, we'll fix them ourselves... $installdir/mysql/scripts/mysql_install_db --rpm --user=$user --defaults-extra-file=$installdir/mysql/my.cnf --basedir=$installdir/mysql >/dev/null # Change permissions again to fix any new files. diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index 322ecb6c3..deedd4c60 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -254,7 +254,13 @@ void mysqlSetup() string tmpDir = startup::StartUp::tmpDir(); - cmd = installDir + "/bin/post-mysqld-install --installdir=" + installDir + " > " + tmpDir + "/post-mysqld-install.log 2>&1"; + string mysqlpw = oam.getMySQLPassword(); + + string passwordOption = ""; + if ( mysqlpw != oam::UnassignedName ) + passwordOption = " --password=" + mysqlpw; + + cmd = installDir + "/bin/post-mysqld-install --installdir=" + installDir + " " + passwordOption + " > " + tmpDir + "/post-mysqld-install.log 2>&1"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) @@ -283,13 +289,7 @@ void mysqlSetup() HOME = p; } - string mysqlpw = oam.getMySQLPassword(); - - string passwordOption = ""; - if ( mysqlpw != oam::UnassignedName ) - passwordOption = " --password=" + mysqlpw; - - cmd = installDir + "/bin/post-mysql-install --installdir=" + installDir + " --tmpdir=" + tmpDir + " " + passwordOption + " > " + tmpDir + "/post-mysql-install.log"; + cmd = installDir + "/bin/post-mysql-install --installdir=" + installDir + " --tmpdir=" + tmpDir + " > " + tmpDir + "/post-mysql-install.log"; rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) == 2) From b090339cab7645187d2534e355b34a2059c8ba9b Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 6 Nov 2018 16:58:33 -0600 Subject: [PATCH 122/127] MCOL-520 - move mysql_upgrade --- oam/install_scripts/module_installer.sh | 2 +- oam/install_scripts/post-mysql-install | 2 ++ oam/install_scripts/post-mysqld-install | 3 +++ oamapps/postConfigure/helpers.cpp | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 0f8d6ba14..038611675 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -160,7 +160,7 @@ if [ $module = "um" ] || ( [ $module = "pm" ] && [ $PMwithUM = "y" ] ) || [ $Ser fi echo "Run post-mysqld-install" - $COLUMNSTORE_INSTALL_DIR/bin/post-mysqld-install --installdir=$COLUMNSTORE_INSTALL_DIR $mysqlPassword > ${tmpDir}/post-mysqld-install.log 2>&1 + $COLUMNSTORE_INSTALL_DIR/bin/post-mysqld-install --installdir=$COLUMNSTORE_INSTALL_DIR $mysqlPassword --tmpdir=${tmpDir} > ${tmpDir}/post-mysqld-install.log 2>&1 if [ $? -ne 0 ]; then echo "ERROR: post-mysqld-install failed: check ${tmpDir}/post-mysqld-install.log" exit 1 diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 8a4c0291d..57e950f68 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -45,6 +45,8 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install password=" " +tmpdir="/tmp" + for arg in "$@"; do if [ $(expr -- "$arg" : '--prefix=') -eq 9 ]; then prefix="$(echo $arg | awk -F= '{print $2}')" diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index 27eba3d42..3976c5466 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -8,6 +8,7 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install user=mysql +tmpdir="/tmp" password=" " pwprompt= @@ -30,6 +31,8 @@ for arg in "$@"; do user="`echo $arg | awk -F= '{print $2}'`" elif [ $(expr -- "$arg" : '--password=') -eq 11 ]; then password="$(echo $arg | awk -F= '{print $2}')" + elif [ $(expr -- "$arg" : '--tmpdir=') -eq 9 ]; then + tmpdir="$(echo $arg | awk -F= '{print $2}')" else echo "post-mysqld-install: ignoring unknown argument: $arg" 1>&2 fi diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index deedd4c60..3238f9c57 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -260,7 +260,7 @@ void mysqlSetup() if ( mysqlpw != oam::UnassignedName ) passwordOption = " --password=" + mysqlpw; - cmd = installDir + "/bin/post-mysqld-install --installdir=" + installDir + " " + passwordOption + " > " + tmpDir + "/post-mysqld-install.log 2>&1"; + cmd = installDir + "/bin/post-mysqld-install --installdir=" + installDir + " " + passwordOption + " --tmpdir=" + tmpDir + " > " + tmpDir + "/post-mysqld-install.log 2>&1"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) != 0) From a3e78d6b639f43af8a4a56bf0d1e1a3c89fe04f6 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 6 Nov 2018 17:41:11 -0600 Subject: [PATCH 123/127] MCOL-520 - move mysql_upgrade --- oam/install_scripts/post-mysqld-install | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index 3976c5466..3d9ce2b72 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -105,6 +105,10 @@ if [ -x $installdir/mysql/mysql-Columnstore ]; then fi fi fi + + # stop + $installdir/mysql/mysql-Columnstore stop >/dev/null 2>&1 + sleep 2 fi ### Don't give the user the notes, we'll fix them ourselves... From f49ad9a4a178e3d40bff90b0059183d0c584ed9a Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 6 Nov 2018 18:42:44 -0600 Subject: [PATCH 124/127] MCOL-520 - move mysql_upgrade --- oam/install_scripts/post-mysql-install | 22 --------- oam/install_scripts/post-mysqld-install | 61 +++++++++++++------------ 2 files changed, 33 insertions(+), 50 deletions(-) diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 57e950f68..ae5970ffe 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -44,7 +44,6 @@ checkForError() { prefix=/usr/local installdir=$prefix/mariadb/columnstore rpmmode=install -password=" " tmpdir="/tmp" for arg in "$@"; do @@ -105,27 +104,6 @@ if [ -x $installdir/mysql/mysql-Columnstore ]; then sleep 5 - # Run MariaDB (mysql) upgrade script, if it exist - if [ -x $installdir/mysql/bin/mysql_upgrade ]; then - if [[ ${password} == " " ]]; then - $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf > $tmpdir/mysql_upgrade.log - if [ $? -ne 0 ]; then - echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" - $installdir/mysql/mysql-Columnstore stop - sleep 2 - exit 2; - fi - else - $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf --password=$password > $tmpdir/mysql_upgrade.log - if [ $? -ne 0 ]; then - echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" - $installdir/mysql/mysql-Columnstore stop - sleep 2 - exit 2; - fi - fi - fi - # Install various Calpont stuff... $installdir/mysql/install_calpont_mysql.sh --installdir=$installdir --tmpdir=$tmpdir checkForError diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index 3d9ce2b72..939f48006 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -77,38 +77,43 @@ fi # InfiniDB testing hook... test -x /usr/local/bin/idb-testing-mysql-pre-start && /usr/local/bin/idb-testing-mysql-pre-start $installdir/mysql -if [ -x $installdir/mysql/mysql-Columnstore ]; then - # Restart in the same way that mysqld will be started normally. - $installdir/mysql/mysql-Columnstore stop >/dev/null 2>&1 - sleep 2 - $installdir/mysql/mysql-Columnstore start +# If DB exist, run upgrade script if it exist +if [ -d $installdir/mysql/db/calpontsys ]; then - sleep 5 - - # Run MariaDB (mysql) upgrade script, if it exist - if [ -x $installdir/mysql/bin/mysql_upgrade ]; then - if [[ ${password} == " " ]]; then - $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf > $tmpdir/mysql_upgrade.log - if [ $? -ne 0 ]; then - echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" - $installdir/mysql/mysql-Columnstore stop - sleep 2 - exit 2; - fi - else - $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf --password=$password > $tmpdir/mysql_upgrade.log - if [ $? -ne 0 ]; then - echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" - $installdir/mysql/mysql-Columnstore stop - sleep 2 - exit 2; + if [ -x $installdir/mysql/mysql-Columnstore ]; then + # Restart in the same way that mysqld will be started normally. + $installdir/mysql/mysql-Columnstore stop >/dev/null 2>&1 + sleep 2 + $installdir/mysql/mysql-Columnstore start + + sleep 5 + + # Run MariaDB (mysql) upgrade script, if it exist + if [ -x $installdir/mysql/bin/mysql_upgrade ]; then + echo "Running mysql_upgrade script" + if [[ ${password} == " " ]]; then + $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf > $tmpdir/mysql_upgrade.log + if [ $? -ne 0 ]; then + echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" + $installdir/mysql/mysql-Columnstore stop + sleep 2 + exit 2; + fi + else + $installdir/mysql/bin/mysql_upgrade --defaults-file=$installdir/mysql/my.cnf --password=$password > $tmpdir/mysql_upgrade.log + if [ $? -ne 0 ]; then + echo "ERROR: mysql_upgrade failure, check $tmpdir/mysql_upgrade.log" + $installdir/mysql/mysql-Columnstore stop + sleep 2 + exit 2; + fi fi fi + + # stop + $installdir/mysql/mysql-Columnstore stop + sleep 2 fi - - # stop - $installdir/mysql/mysql-Columnstore stop >/dev/null 2>&1 - sleep 2 fi ### Don't give the user the notes, we'll fix them ourselves... From 142f04aeebfc1e878f7573527c177b0fa090813c Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 7 Nov 2018 09:01:22 -0600 Subject: [PATCH 125/127] MCOL-520 - change to only do sudo change to rc.local for non-root amazon --- oamapps/postConfigure/postConfigure.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 8c4cd73c6..15a39973f 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1471,12 +1471,12 @@ int main(int argc, char* argv[]) cout << "Check Amazon Install Documenation for additional information, exiting..." << endl; exit (1); } - } - // setup to start on reboot, for non-root amazon installs - if ( !rootUser ) - { - system("sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); + // setup to start on reboot, for non-root amazon installs + if ( !rootUser ) + { + system("sudo sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); + } } if ( !writeConfig(sysConfig) ) From deba5d2a1036d6237da2ebd1ab3c4b2889a3982f Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 7 Nov 2018 09:29:55 -0600 Subject: [PATCH 126/127] MCOL-520 - check storage type before unmount --- procmon/processmonitor.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index b92ab2b6f..752961854 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1035,10 +1035,20 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO cacheutils::dropPrimProcFdCache(); flushInodeCache(); - string cmd = SUDO + "umount " + startup::StartUp::installDir() + "/data* -l > /dev/null 2>&1"; + try + { + string DBRootStorageType = "internal"; + oam.getSystemConfig("DBRootStorageType", DBRootStorageType); - system(cmd.c_str()); - sleep(1); + if ( DBRootStorageType == "external" ) + { + string cmd = SUDO + "umount " + startup::StartUp::installDir() + "/data* -l > /dev/null 2>&1"; + + system(cmd.c_str()); + sleep(1); + } + } + catch (...) {} system("rpm -e --nodeps $(rpm -qa | grep '^mariadb-columnstore')"); system("dpkg -P $(dpkg --get-selections | grep '^mariadb-columnstore')"); From 3727203b8ff06723efab823ad59f3a077444c5bc Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 7 Nov 2018 10:13:00 -0600 Subject: [PATCH 127/127] MCOL-520 - remove sudo for checking gluster log --- oamapps/postConfigure/postConfigure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 15a39973f..bb828b5be 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -5544,7 +5544,7 @@ bool storageSetup(bool amazonInstall) // if gluster if ( storageType == "3" ) { - string command = SUDO + "stat /var/run/glusterd.pid > /dev/null 2>&1"; + string command = "stat /var/run/glusterd.pid > /dev/null 2>&1"; int status = system(command.c_str()); if (WEXITSTATUS(status) != 0 )