database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-08 14:22:09 +03:00
Code Activity

Merge branch 'develop' into MCOL-1793

Browse Source
This commit is contained in:
David Hall
2018-11-20 16:30:22 -06:00
parent 9c1bc910a5 410734821b
commit 3709fe92a5
181 changed files with 2839 additions and 21116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
utils/clusterTester/columnstoreClusterTester.sh
View File

@@ -18,6 +18,9 @@ if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then
COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore
fi
#get temp directory
tmpDir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir`
checkContinue() {
if [ "$CHECK" = false ]; then
@@ -69,7 +72,6 @@ helpPrint () {
echo " Firewall settings"
echo " Date/time settings"
echo " Dependent packages installed"
echo " For non-root user install - test permissions on /tmp and /dev/shm"
echo ""
echo "Usage: $0 [options]"
echo "OPTIONS:"
@@ -203,23 +205,23 @@ checkLocalOS()
echo ""
#get local OS
`$COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh > /tmp/os_detect 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh > ${tmpDir}/os_detect 2>&1`
if [ "$?" -eq 0 ]; then
localOS=`cat /tmp/os_detect | grep "Operating System name" | cut -f2 -d '"'`
localOS=`cat ${tmpDir}/os_detect | grep "Operating System name" | cut -f2 -d '"'`
echo "Local Node OS System Name : $localOS"
if [ "$OS" != "" ] ; then
echo ""
echo "Local Node OS Versions doesn't match the command line OS argument"
echo "Contining using the Detected Local Node OS Version"
OS=`cat /tmp/os_detect | grep "Operating System tag" | cut -f4 -d " "`
OS=`cat ${tmpDir}/os_detect | grep "Operating System tag" | cut -f4 -d " "`
echo "Local Node OS Version : $OS"
else
OS=`cat /tmp/os_detect | grep "Operating System tag" | cut -f4 -d " "`
OS=`cat ${tmpDir}/os_detect | grep "Operating System tag" | cut -f4 -d " "`
fi
else
localOS=`cat /tmp/os_detect | grep "Operating System name" | cut -f2 -d '"'`
localOS=`cat ${tmpDir}/os_detect | grep "Operating System name" | cut -f2 -d '"'`
echo "Local Node OS System Name : $localOS"
if [ "$OS" == "" ] ; then
@@ -265,31 +267,17 @@ checkLocalDir()
# Non-root User directory permissions check
#
echo ""
echo "** Run Non-root User directory permissions check on Local Node"
echo "** Run Non-root User directory permissions check on Local Node (dev/shm)"
echo ""
#remove any check tmp files from previous runs
`sudo rm -f /tmp/*_check > /dev/null 2>&1`
#check /tmp and /dev/shm
pass=true
`touch /tmp/cs_check > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "Local Node permission test on /tmp : Passed"
`rm -f /tmp/cs_check`
else
echo "Local Node permission test on /tmp : ${bold}Failed${normal}, change permissions to 777 and re-test"
exit 1
fi
`touch /dev/shm/cs_check > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "Local Node permission test on /dev/shm : Passed"
`rm -f /dev/shm/cs_check`
echo "Local Node permission test on /dev/shm : Passed"
`rm -f /dev/shm/cs_check`
else
echo "Local Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test"
pass=false
REPORTPASS=false
echo "Local Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test"
pass=false
REPORTPASS=false
fi
fi
}
@@ -346,56 +334,29 @@ checkSSH()
checkRemoteDir()
{
#
# remove old _check tmp files from remote servers
`sudo rm -f /tmp/*_check > /dev/null 2>&1`
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'sudo rm -f /tmp/*_check > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1`
done
if [ "$USER" != "root" ]; then
# Non-root User directory permissions check
#
echo ""
echo "** Run Non-root User directory permissions check on remote nodes"
echo "** Run Non-root User directory permissions check on remote nodes (/dev/shm)"
echo ""
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /tmp/cs_check' 1 > /tmp/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
`grep "Permission denied" /tmp/remote_command_check > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "$ipadd Node permission test on /tmp : ${bold}Failed${normal}, change permissions to 777 and re-test"
exit 1
else
echo "$ipadd Node permission test on /tmp : Passed"
fi
else
echo "Error running remote_command.sh to $ipadd Node, check /tmp/remote_command_check"
pass=false
REPORTPASS=false
fi
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > /tmp/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
`grep "Permission denied" /tmp/remote_command_check > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "$ipadd Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test"
pass=false
REPORTPASS=false
else
echo "$ipadd Node permission test on /dev/shm : Passed"
fi
else
echo "Error running remote_command.sh to $ipadd Node, check /tmp/remote_command_check"
pass=false
REPORTPASS=false
fi
done
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > ${tmpDir}/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
`grep "Permission denied" ${tmpDir}/remote_command_check > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "$ipadd Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test"
pass=false
REPORTPASS=false
else
echo "$ipadd Node permission test on /dev/shm : Passed"
fi
else
echo "Error running remote_command.sh to $ipadd Node, check ${tmpDir}/remote_command_check"
pass=false
REPORTPASS=false
fi
if ! $pass; then
checkContinue
@@ -415,20 +376,20 @@ checkOS()
echo ""
pass=true
`/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh /tmp/.`
`/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh ${tmpDir}/.`
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD /tmp/os_detect.sh 1 > /tmp/remote_scp_put_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD ${tmpDir}/os_detect.sh 1 > ${tmpDir}/remote_scp_put_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_put.sh to $ipadd Node, check /tmp/remote_scp_put_check"
echo "Error running remote_scp_put.sh to $ipadd Node, check ${tmpDir}/remote_scp_put_check"
exit 1
else
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD /tmp/os_detect.sh 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD ${tmpDir}/os_detect.sh 1 > ${tmpDir}/remote_command_check`
rc="$?"
if [ "$?" -ne 0 ]; then
echo "Error running remote_command.sh /tmp/os_detect.sh on $ipadd Node, check /tmp/remote_command_check"
echo "Error running remote_command.sh ${tmpDir}/os_detect.sh on $ipadd Node, check ${tmpDir}/remote_command_check"
exit 1
else
remoteOS=`cat /tmp/remote_command_check | grep "Operating System name" | cut -f2 -d '"'`
remoteOS=`cat ${tmpDir}/remote_command_check | grep "Operating System name" | cut -f2 -d '"'`
echo "$ipadd Node OS Version : $remoteOS"
if [ $localOS != $remoteOS ]; then
echo "${bold}Failed${normal}, $ipadd has a different OS than local node"
@@ -453,25 +414,25 @@ checkLocale()
echo ""
#get local Locale
`locale | grep LANG= > /tmp/locale_check 2>&1`
`locale | grep LANG= > ${tmpDir}/locale_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "Local Node Locale : `cat /tmp/locale_check`"
echo "Local Node Locale : `cat ${tmpDir}/locale_check`"
else
echo "Error running 'locale' command on local node"
fi
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > /tmp/locale_check 2>&1' 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > locale_check 2>&1' 1 > ${tmpDir}/remote_command_check`
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/locale_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD locale_check > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check"
exit 1
else
echo "$ipadd Node Locale : `cat locale_check`"
`diff /tmp/locale_check locale_check > /dev/null 2>&1`
`diff ${tmpDir}/locale_check locale_check > /dev/null 2>&1`
if [ "$?" -ne 0 ]; then
echo "${bold}Failed${normal}, $ipadd has a different Locale setting than local node"
pass=false
@@ -480,7 +441,7 @@ checkLocale()
`rm -f locale_check`
fi
else
echo "Error running remote_command.sh to $ipadd Node, check /tmp/remote_command_check"
echo "Error running remote_command.sh to $ipadd Node, check ${tmpDir}/remote_command_check"
exit 1
pass=false
REPORTPASS=false
@@ -538,7 +499,7 @@ checkLocalSELINUX()
pass=true
#check local SELINUX
if [ -f /etc/selinux/config ]; then
`cat /etc/selinux/config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
`cat /etc/selinux/config | grep SELINUX | grep enforcing > ${tmpDir}/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, Local Node SELINUX setting is Enabled, check port test results"
pass=false
@@ -565,9 +526,9 @@ checkUMASK()
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > ${tmpDir}/remote_command_check 2>&1`
if [ "$?" -eq 0 ]; then
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> /tmp/remote_scp_get 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> ${tmpDir}/remote_scp_get 2>&1`
if [ "$?" -eq 0 ]; then
permission=`cat test.log`
result=${permission:4:1}
@@ -584,11 +545,11 @@ checkUMASK()
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check /tmp/remote_scp_get"
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check ${tmpDir}/remote_scp_get"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check /tmp/remote_command_check"
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check ${tmpDir}/remote_command_check"
pass=false
fi
`rm -f test.log`
@@ -611,11 +572,11 @@ checkSELINUX()
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "$ipadd Node SELINUX setting is Not Enabled"
else
`cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
`cat config | grep SELINUX | grep enforcing > ${tmpDir}/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results"
pass=false
@@ -644,12 +605,12 @@ checkFirewalls()
#check local FIREWALLS
for firewall in "${FIREWALL_LIST[@]}"; do
pass=true
`service $firewall status > /tmp/firewall1_check 2>&1`
`service $firewall status > ${tmpDir}/firewall1_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results"
pass=false
else
`systemctl status $firewall > /tmp/firewall1_check 2>&1`
`systemctl status $firewall > ${tmpDir}/firewall1_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results"
pass=false
@@ -667,12 +628,12 @@ checkFirewalls()
# 'sysconfig not on remote node
for firewall in "${FIREWALL_LIST[@]}"; do
pass=true
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > ${tmpDir}/firewall_check 2>&1" 1 > ${tmpDir}/remote_command_check`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results"
pass=false
else
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > ${tmpDir}/firewall_check 2>&1" 1 > ${tmpDir}/remote_command_check`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results"
pass=false
@@ -696,7 +657,7 @@ checkFirewalls()
pass=true
#check local IPTABLES
`/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1`
`/sbin/rcSuSEfirewall2 status > ${tmpDir}/rcSuSEfirewall2_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node rcSuSEfirewall2 service is Enabled, check port test results"
pass=false
@@ -705,7 +666,7 @@ checkFirewalls()
fi
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > ${tmpDir}/rcSuSEfirewall2_check 2>&1' 1 > ${tmpDir}/remote_command_check`
rc="$?"
if [ $rc -eq 0 ] ; then
echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, check port test results"
@@ -728,12 +689,12 @@ checkPorts()
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`sudo nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > /tmp/port_test`
`nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > ${tmpDir}/port_test`
if [ "$?" -ne 0 ]; then
echo $ipadd " Node Passed port test"
else
echo $ipadd " Node ${bold}Failed${normal} port test, check and disable any firewalls or open ports in firewall"
cat /tmp/port_test
cat ${tmpDir}/port_test
pass=false
REPORTPASS=false
fi
@@ -756,16 +717,16 @@ checkTime()
#get local epoch time
localTime=`date +%s`
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'date +%s > /tmp/time_check' > /tmp/time_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'date +%s > time_check' > ${tmpDir}/time_check`
rc="$?"
if [ $rc -ne 0 ] ; then
echo $ipadd " Node ${bold}Failed${normal} date/time check failed, check /tmp/time_check"
echo $ipadd " Node ${bold}Failed${normal} date/time check failed, check ${tmpDir}/time_check"
pass=false
REPORTPASS=false
else
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/time_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD time_check > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check"
else
remoteTime=`cat time_check`
timeDiff=`echo "$(($remoteTime-$localTime))"`
@@ -803,17 +764,17 @@ checkMysqlPassword()
fi
if [ "$IPADDRESSES" != "" ]; then
`/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/mariadb-command-line.sh /tmp/.`
`/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/mariadb-command-line.sh ${tmpDir}/.`
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD $COLUMNSTORE_INSTALL_DIR/mysql/mysql-Columnstore start > /dev/null 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD /tmp/mariadb-command-line.sh 1 > /tmp/remote_scp_put_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD ${tmpDir}/mariadb-command-line.sh 1 > ${tmpDir}/remote_scp_put_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_put.sh to $ipadd Node, check /tmp/remote_scp_put_check"
echo "Error running remote_scp_put.sh to $ipadd Node, check ${tmpDir}/remote_scp_put_check"
exit 1
else
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD /tmp/mariadb-command-line.sh 1 > /tmp/remote_command_check`
`cat /tmp/remote_command_check | grep "ERROR - PASSWORD" > /dev/null 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD ${tmpDir}/mariadb-command-line.sh 1 > ${tmpDir}/remote_command_check`
`cat ${tmpDir}/remote_command_check | grep "ERROR - PASSWORD" > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node MariaDB login failed with missing password file, /root/.my.cnf"
pass=false
@@ -839,7 +800,8 @@ checkPackages()
echo "** Run MariaDB ColumnStore Dependent Package Check"
echo ""
declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "numactl-libs")
declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "libaio" "rsync" "snappy" "net-tools" "numactl-libs")
declare -a CENTOS_PKG_NOT=("mariadb-libs")
if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then
@@ -859,8 +821,8 @@ checkPackages()
REPORTPASS=false
fi
else
`yum list installed "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep Installed > /dev/null 2>&1`
`yum list installed "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep Installed > /dev/null 2>&1`
if [ "$?" -ne 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install"
pass=false
@@ -879,8 +841,8 @@ checkPackages()
#check for package that shouldnt be installed
pass=true
for PKG in "${CENTOS_PKG_NOT[@]}"; do
`yum list installed "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep Installed > /dev/null 2>&1`
`yum list installed "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep Installed > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install"
pass=false
@@ -900,14 +862,14 @@ checkPackages()
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${CENTOS_PKG[@]}"; do
if [ $OS == "centos6" ] && [ $PKG == "boost" ]; then
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'ls /usr/lib/libboost_regex.so > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'ls /usr/lib/libboost_regex.so > /dev/null 2>&1' 1 > ${tmpDir}/remote_command_check 2>&1`
if [ $? -ne 0 ] ; then
echo "${bold}Failed${normal}, $ipadd Node ${bold}boost libraries${normal} not installed"
pass=false
REPORTPASS=false
fi
else
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 2 ] ; then
echo "${bold}Failed${normal}, $ipadd Node, 'yum' not installed"
@@ -932,7 +894,7 @@ checkPackages()
#check for package that shouldnt be installed
for PKG in "${CENTOS_PKG_NOT[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 2 ] ; then
echo "${bold}Failed${normal}, $ipadd Node, 'yum' not installed"
@@ -958,7 +920,8 @@ checkPackages()
fi
fi
declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "libnuma1")
declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "libaio1" "rsync" "libsnappy1" "net-tools" "libnuma1")
declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18")
if [ "$OS" == "suse12" ]; then
@@ -970,8 +933,8 @@ checkPackages()
pass=true
#check centos packages on local node
for PKG in "${SUSE_PKG[@]}"; do
`rpm -qi "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep "not installed" > /dev/null 2>&1`
`rpm -qi "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep "not installed" > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install"
pass=false
@@ -988,8 +951,8 @@ checkPackages()
#check for package that shouldnt be installed
pass=true
for PKG in "${SUSE_PKG_NOT[@]}"; do
`rpm -qi "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep "not installed" > /dev/null 2>&1`
`rpm -qi "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep "not installed" > /dev/null 2>&1`
if [ "$?" -ne 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install"
pass=false
@@ -1009,7 +972,7 @@ checkPackages()
if [ "$IPADDRESSES" != "" ]; then
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${SUSE_PKG[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
rc="$?"
if [ $rc -ne 0 ] ; then
echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is not installed, please install"
@@ -1028,7 +991,7 @@ checkPackages()
#check for package that shouldnt be installed
for PKG in "${SUSE_PKG_NOT[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 0 ] ; then
echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install"
@@ -1049,7 +1012,7 @@ checkPackages()
fi
fi
declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" )
declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" )
declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18")
if [ "$OS" == "ubuntu16" ] || [ "$OS" == "ubuntu18" ]; then
@@ -1061,8 +1024,8 @@ checkPackages()
pass=true
#check centos packages on local node
for PKG in "${UBUNTU_PKG[@]}"; do
`dpkg -s "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
`dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
if [ "$?" -ne 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install"
pass=false
@@ -1079,8 +1042,8 @@ checkPackages()
#check for package that shouldnt be installed
pass=true
for PKG in "${UBUNTU_PKG_NOT[@]}"; do
`dpkg -s "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
`dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install"
pass=false
@@ -1100,12 +1063,12 @@ checkPackages()
if [ "$IPADDRESSES" != "" ]; then
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${UBUNTU_PKG[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check"
else
`cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1`
`cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed"
pass=false
@@ -1132,12 +1095,12 @@ checkPackages()
#check for package that shouldnt be installed
for PKG in "${UBUNTU_PKG_NOT[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check"
else
`cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1`
`cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed"
pass=false
@@ -1166,7 +1129,8 @@ checkPackages()
fi
fi
declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libnuma1")
declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libnuma1")
declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server")
if [ "$OS" == "debian8" ]; then
@@ -1178,8 +1142,8 @@ checkPackages()
pass=true
#check centos packages on local node
for PKG in "${DEBIAN_PKG[@]}"; do
`dpkg -s "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
`dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
if [ "$?" -ne 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install"
pass=false
@@ -1196,8 +1160,8 @@ checkPackages()
#check for package that shouldnt be installed
pass=true
for PKG in "${DEBIAN_PKG_NOT[@]}"; do
`dpkg -s "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
`dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install"
pass=false
@@ -1217,12 +1181,12 @@ checkPackages()
if [ "$IPADDRESSES" != "" ]; then
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${DEBIAN_PKG[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check"
else
`cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1`
`cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed"
pass=false
@@ -1249,12 +1213,12 @@ checkPackages()
#check for package that shouldnt be installed
for PKG in "${DEBIAN_PKG_NOT[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check"
else
`cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1`
`cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed"
pass=false
@@ -1283,7 +1247,8 @@ checkPackages()
fi
fi
declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1" "libnuma1")
declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1" "libnuma1")
declare -a DEBIAN9_PKG_NOT=("libmariadb18" "mariadb-server")
if [ "$OS" == "debian9" ]; then
@@ -1295,8 +1260,8 @@ checkPackages()
pass=true
#check centos packages on local node
for PKG in "${DEBIAN9_PKG[@]}"; do
`dpkg -s "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
`dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
if [ "$?" -ne 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is not installed, please install"
pass=false
@@ -1313,8 +1278,8 @@ checkPackages()
#check for package that shouldnt be installed
pass=true
for PKG in "${DEBIAN9_PKG_NOT[@]}"; do
`dpkg -s "$PKG" > /tmp/pkg_check 2>&1`
`cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
`dpkg -s "$PKG" > ${tmpDir}/pkg_check 2>&1`
`cat ${tmpDir}/pkg_check | grep 'install ok installed' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install"
pass=false
@@ -1335,12 +1300,12 @@ checkPackages()
if [ "$IPADDRESSES" != "" ]; then
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${DEBIAN9_PKG[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check"
else
`cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1`
`cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed"
pass=false
@@ -1367,12 +1332,12 @@ checkPackages()
#check for package that shouldnt be installed
for PKG in "${DEBIAN9_PKG_NOT[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > ${tmpDir}/pkg_check 2>&1" 1 > ${tmpDir}/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD ${tmpDir}/pkg_check > ${tmpDir}/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_scp_get.sh to $ipadd Node, check ${tmpDir}/remote_scp_get_check"
else
`cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1`
`cat ${tmpDir}/remote_command_check | grep 'command not found' > /dev/null 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed"
pass=false

4
utils/compress/version1.cpp
View File

@@ -188,13 +188,9 @@ void initCtlShm()
}
catch (bi::interprocess_exception&)
{
#if BOOST_VERSION < 104500
bi::shared_memory_object shm(bi::create_only, DecomShmName.c_str(), bi::read_write);
#else
bi::permissions perms;
perms.set_unrestricted();
bi::shared_memory_object shm(bi::create_only, DecomShmName.c_str(), bi::read_write, perms);
#endif
shm.truncate(sizeof(CtlShmImage));
bi::mapped_region region(shm, bi::read_write);
tmpptr = new (region.get_address()) CtlShmImage;

4
utils/configcpp/configcpp.cpp
View File

@@ -97,7 +97,7 @@ Config* Config::makeConfig(const char* cf)
cf = cfStr.c_str();
#else
cf = getenv("CALPONT_CONFIG_FILE");
cf = getenv("COLUMNSTORE_CONFIG_FILE");
#endif
if (cf == 0 || *cf == 0)
@@ -419,7 +419,7 @@ void Config::writeConfig(const string& configFile) const
value = c1->getConfig("SystemConfig", "SystemName");
//good read, save copy, copy temp file tp tmp then to Columnstore.xml
//move to /tmp to get around a 'same file error' in mv command
//move to get around a 'same file error' in mv command
try
{
if (exists(scft)) fs::remove(scft);

8
utils/configcpp/writeonce.cpp
View File

@@ -50,12 +50,16 @@ namespace config
void WriteOnceConfig::initializeDefaults()
{
string tmpDir = startup::StartUp::tmpDir();
fLBID_Shift = make_pair("13", false);
fDBRootCount = make_pair("1", false);
fDBRMRoot = make_pair("/mnt/OAM/dbrm/BRM_saves", false);
fSharedMemoryTmpFile1 = make_pair("/tmp/CalpontShm,", false);
string file = tmpDir + "/ColumnstoreShm";
fSharedMemoryTmpFile1 = make_pair(file, false);
fTxnIDFile = make_pair("/mnt/OAM/dbrm/SMTxnID", false);
fSharedMemoryTmpFile2 = make_pair("/tmp/CalpontSessionMonitorShm", false);
file = tmpDir + "/CalpontSessionMonitorShm";
fSharedMemoryTmpFile2 = make_pair(file, false);
}
void WriteOnceConfig::setup()

7
utils/funcexp/func_concat.cpp
View File

@@ -55,11 +55,14 @@ string Func_concat::getStrVal(Row& row,
bool& isNull,
CalpontSystemCatalog::ColType&)
{
string ret = stringValue(parm[0], row, isNull);
string ret;
string tmp;
stringValue(parm[0], row, isNull, ret);
for ( unsigned int id = 1 ; id < parm.size() ; id++)
{
ret.append( stringValue(parm[id], row, isNull) );
stringValue(parm[id], row, isNull, tmp);
ret.append(tmp);
}
return ret;

13
utils/funcexp/func_concat_ws.cpp
View File

@@ -49,8 +49,8 @@ string Func_concat_ws::getStrVal(Row& row,
bool& isNull,
CalpontSystemCatalog::ColType&)
{
string delim = stringValue(parm[0], row, isNull);
string delim;
stringValue(parm[0], row, isNull, delim);
if (isNull)
return "";
@@ -63,8 +63,8 @@ string Func_concat_ws::getStrVal(Row& row,
for ( unsigned int id = 1 ; id < parm.size() ; id++)
{
string tstr = stringValue(parm[id], row, isNull);
string tstr;
stringValue(parm[id], row, isNull, tstr);
if (isNull)
{
isNull = false;
@@ -94,10 +94,11 @@ string Func_concat_ws::getStrVal(Row& row,
#else
string str;
string tmp;
for ( uint32_t i = 1 ; i < parm.size() ; i++)
{
str += string(stringValue(parm[i], row, isNull).c_str());
string(stringValue(parm[i], row, isNull).c_str(), tmp);
str += tmp;
if (isNull)
{

4
utils/funcexp/func_elt.cpp
View File

@@ -100,7 +100,9 @@ string Func_elt::getStrVal(rowgroup::Row& row,
return "";
}
return stringValue(parm[number], row, isNull);
std::string ret;
stringValue(parm[number], row, isNull, ret);
return ret;
}

8
utils/funcexp/func_insert.cpp
View File

@@ -88,13 +88,13 @@ std::string Func_insert::getStrVal(rowgroup::Row& row,
bool& isNull,
execplan::CalpontSystemCatalog::ColType&)
{
const string& tstr = stringValue(fp[0], row, isNull);
string tstr;
stringValue(fp[0], row, isNull, tstr);
if (isNull)
return "";
const string& tnewstr = stringValue(fp[3], row, isNull);
string tnewstr;
stringValue(fp[3], row, isNull, tnewstr);
if (isNull)
return "";

4
utils/funcexp/func_repeat.cpp
View File

@@ -61,7 +61,9 @@ std::string Func_repeat::getStrVal(rowgroup::Row& row,
bool& isNull,
execplan::CalpontSystemCatalog::ColType& op_ct)
{
string str = stringValue(fp[0], row, isNull);
string str;
stringValue(fp[0], row, isNull, str);
if (str.empty() || str == "")
return "";

3
utils/funcexp/func_reverse.cpp
View File

@@ -65,7 +65,8 @@ std::string Func_reverse::getStrVal(rowgroup::Row& row,
bool& isNull,
execplan::CalpontSystemCatalog::ColType&)
{
string str = stringValue(fp[0], row, isNull);
string str;
stringValue(fp[0], row, isNull, str);
// We used to reverse in the string buffer, but that doesn't
// work for all compilers as some re-use the buffer on simple

8
utils/funcexp/functor_str.h
View File

@@ -100,7 +100,7 @@ public:
}
protected:
const std::string& stringValue(execplan::SPTP& fp, rowgroup::Row& row, bool& isNull)
const void stringValue(execplan::SPTP& fp, rowgroup::Row& row, bool& isNull, std::string& fFloatStr)
{
// Bug3788, use the shorter of fixed or scientific notation for floating point values.
// [ the default format in treenode.h is fixed-point notation ]
@@ -120,7 +120,8 @@ protected:
break;
default:
return fp->data()->getStrVal(row, isNull);
fFloatStr = fp->data()->getStrVal(row, isNull);
return;
break;
}
@@ -140,10 +141,7 @@ protected:
fFloatStr += buf;
}
return fFloatStr;
}
std::string fFloatStr;
};

8
utils/idbdatafile/IDBLogger.cpp
View File

@@ -1,4 +1,5 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2018 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@@ -28,6 +29,8 @@
#include "IDBLogger.h"
#include "installdir.h"
#include <sstream>
#include <fstream>
#include <time.h>
@@ -55,7 +58,10 @@ void IDBLogger::writeLog( const std::string& logmsg )
pthread_t threadid = pthread_self();
#endif
ostringstream fname;
fname << "/tmp/idbdf-log-" << pid << "-" << threadid << ".csv";
string tmpDir = startup::StartUp::tmpDir();
fname << tmpDir + "/idbdf-log-" << pid << "-" << threadid << ".csv";
ofstream output;
output.open( fname.str().c_str(), ios::out | ios::app );

25
utils/idbdatafile/IDBPolicy.cpp
View File

@@ -34,6 +34,9 @@
#include "utils_utf8.h"
#endif
#include "installdir.h"
using namespace std;
namespace idbdatafile
@@ -60,6 +63,7 @@ void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string&
if ( hdfsRdwrScratch.length() > 0 )
{
// TODO-check to make sure this directory has sufficient space, whatever that means.
boost::filesystem::path tmpfilepath( hdfsRdwrScratch );
if (boost::filesystem::exists(tmpfilepath))
@@ -76,6 +80,8 @@ void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string&
}
else
{
cout << tmpfilepath << endl;
if (!boost::filesystem::create_directory(tmpfilepath))
{
// We failed to create the scratch directory
@@ -213,23 +219,10 @@ void IDBPolicy::configIDBPolicy()
}
// Directory in which to place file buffer temporary files.
string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch");
string tmpDir = startup::StartUp::tmpDir();
if ( hdfsRdwrScratch.length() == 0 )
{
string tmpPath = cf->getConfig("SystemConfig", "TempDiskPath");
if ( tmpPath.length() == 0 )
{
hdfsRdwrScratch = "/tmp/hdfsscratch";
}
else
{
hdfsRdwrScratch = tmpPath;
hdfsRdwrScratch += "/hdfsscratch";
}
}
string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch");
string hdfsRdwrScratch = tmpDir + scratch;
IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize );

9
utils/joiner/joinpartition.cpp
View File

@@ -1,4 +1,5 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2018 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@@ -22,6 +23,7 @@
#include "joinpartition.h"
#include "tuplejoiner.h"
#include "atomicops.h"
#include "installdir.h"
using namespace std;
using namespace utils;
@@ -128,12 +130,9 @@ JoinPartition::JoinPartition(const JoinPartition& jp, bool splitMode) :
// Instead, each will double in size, giving a capacity of 8GB -> 16 -> 32, and so on.
// bucketCount = jp.bucketCount;
bucketCount = 2;
filenamePrefix = config->getConfig("HashJoin", "TempFilePath");
filenamePrefix = startup::StartUp::tmpDir();
if (filenamePrefix.empty())
filenamePrefix = "/tmp/infinidb";
filenamePrefix += "/Infinidb-join-data-";
filenamePrefix += "/Columnstore-join-data-";
uniqueID = atomicops::atomicInc(&uniqueNums);
uint32_t tmp = uniqueID;

3
utils/libmysql_client/libmysql_client.cpp
View File

@@ -104,6 +104,7 @@ int LibMySQL::run(const char* query)
{
fErrStr = "fatal error runing mysql_real_query() in libmysql_client lib";
ret = -1;
return ret;
}
fRes = mysql_use_result(fCon);
@@ -117,7 +118,7 @@ int LibMySQL::run(const char* query)
return ret;
}
void LibMySQL::handleMySqlError(const char* errStr, unsigned int errCode)
void LibMySQL::handleMySqlError(const char* errStr, int errCode)
{
ostringstream oss;

2
utils/libmysql_client/libmysql_client.h
View File

@@ -38,7 +38,7 @@ public:
// run the query
int run(const char* q);
void handleMySqlError(const char*, unsigned int);
void handleMySqlError(const char*, int);
MYSQL* getMySqlCon()
{

35
utils/rwlock/rwlock.cpp
View File

@@ -52,6 +52,8 @@ using namespace boost::posix_time;
#include "shmkeys.h"
#include "installdir.h"
namespace
{
using namespace rwlock;
@@ -143,19 +145,9 @@ RWLockShmImpl::RWLockShmImpl(int key, bool excl)
try
{
#if BOOST_VERSION < 104500
bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write);
#ifdef __linux__
{
string pname = "/dev/shm/" + keyName;
chmod(pname.c_str(), 0666);
}
#endif
#else
bi::permissions perms;
perms.set_unrestricted();
bi::shared_memory_object shm(bi::create_only, keyName.c_str(), bi::read_write, perms);
#endif
shm.truncate(sizeof(struct State));
fStateShm.swap(shm);
bi::mapped_region region(fStateShm, bi::read_write);
@@ -174,16 +166,25 @@ RWLockShmImpl::RWLockShmImpl(int key, bool excl)
new (&fState->sems[RWLock::READERS]) bi::interprocess_semaphore(0);
new (&fState->sems[RWLock::WRITERS]) bi::interprocess_semaphore(0);
}
catch (bi::interprocess_exception&)
catch (bi::interprocess_exception &e)
{
if (excl)
{
//don't think we can get here anymore...
throw not_excl();
if (e.get_error_code() == bi::security_error) {
cerr << "RWLock: Failed to create the lock. Check perms on /dev/shm; should be 1777" << endl;
throw;
}
if (e.get_error_code() == bi::already_exists_error && excl)
throw not_excl();
if (e.get_error_code() != bi::already_exists_error)
throw;
bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write);
fStateShm.swap(shm);
try {
bi::shared_memory_object shm(bi::open_only, keyName.c_str(), bi::read_write);
fStateShm.swap(shm);
}
catch (exception &e) {
cerr << "RWLock failed to attach to the " << keyName << " shared mem segment, got " << e.what() << endl;
throw;
}
bi::mapped_region region(fStateShm, bi::read_write);
fRegion.swap(region);
fState = static_cast<State*>(fRegion.get_address());

78
utils/startup/installdir.cpp
View File

@@ -31,6 +31,13 @@ using namespace boost;
#include "idbregistry.h"
#endif
#include "installdir.h"
#include "configcpp.h"
using namespace config;
#include <iostream>
using namespace std;
namespace startup
{
@@ -71,6 +78,77 @@ const string StartUp::installDir()
return *fInstallDirp;
}
/* static */
mutex StartUp::fTmpDirLock;
/* static */
string* StartUp::fTmpDirp = 0;
/* static */
const string StartUp::tmpDir()
{
mutex::scoped_lock lk(fTmpDirLock);
if (fTmpDirp)
return *fTmpDirp;
#ifdef _MSC_VER
fTmpDirp = new string("C:\\Calpont\Tmp");
string cfStr = IDBreadRegistry("");
if (!cfStr.empty())
*fTmpDirp = cfStr;
return *fTmpDirp;
#else
//check for non-root user
/* const char* p = getenv("HOME");
string homedir = p;
if (homedir == "/root")
{
Config* sysConfig = Config::makeConfig();
string TempFileDir;
try
{
TempFileDir = sysConfig->getConfig("SystemConfig", "TempFileDir");
}
catch (...)
{}
fTmpDirp = new string("/tmp");
*fTmpDirp = *fTmpDirp + TempFileDir;
}
else
{
// non-root user
fTmpDirp = new string(homedir);
*fTmpDirp = *fTmpDirp + "/.tmp";
}
*/
Config* sysConfig = Config::makeConfig();
string TempFileDir;
try
{
TempFileDir = sysConfig->getConfig("SystemConfig", "SystemTempFileDir");
}
catch (...)
{}
return TempFileDir;
#endif
}
}
// vim:ts=4 sw=4:

3
utils/startup/installdir.h
View File

@@ -39,6 +39,7 @@ public:
~StartUp() {}
static const std::string installDir();
static const std::string tmpDir();
private:
StartUp(const StartUp& rhs);
@@ -46,6 +47,8 @@ private:
static boost::mutex fInstallDirLock;
static std::string* fInstallDirp;
static boost::mutex fTmpDirLock;
static std::string* fTmpDirp;
};
}

4
utils/udfsdk/README.txt
View File

@@ -11,7 +11,7 @@ files to create your own function or just try that function as is.
$ make
- Copy the libudf_mysql.so.1.0.0 and libudfsdk.so.1.0.0 file to /usr/local/mariadb/columnstore/lib on
every columnstore node.
$ sudo cp libudf_mysql.so.1.0.0 libudfsdk.so.1.0.0 /usr/local/mariadb/columnstore/lib/
$ cp libudf_mysql.so.1.0.0 libudfsdk.so.1.0.0 /usr/local/mariadb/columnstore/lib/
- Restart ColumnStore
$ mcsadmin restartsystem y
- Using the mcsmysql client add the user defined function, e.g,
@@ -20,4 +20,4 @@ files to create your own function or just try that function as is.
> create function mcs_isnull returns string soname 'libudf_mysql.so';
You should now be able to use the mcs_add(arg1, arg2) and mcs_isnull(arg) functions in the select and/or where clauses
of SQL statements.
of SQL statements.

53
utils/udfsdk/udfmysql.cpp
View File

@@ -572,6 +572,59 @@ extern "C"
struct avgx_data* data = (struct avgx_data*)initid->ptr;
return data->sumx / data->cnt;
}
/**
* distinct_count connector stub
*/
#ifdef _MSC_VER
__declspec(dllexport)
#endif
my_bool distinct_count_init(UDF_INIT* initid, UDF_ARGS* args, char* message)
{
if (args->arg_count != 1)
{
strcpy(message,"distinct_count() requires one argument");
return 1;
}
return 0;
}
#ifdef _MSC_VER
__declspec(dllexport)
#endif
void distinct_count_deinit(UDF_INIT* initid)
{
// free(initid->ptr);
}
#ifdef _MSC_VER
__declspec(dllexport)
#endif
void
distinct_count_clear(UDF_INIT* initid, char* is_null __attribute__((unused)),
char* message __attribute__((unused)))
{
}
#ifdef _MSC_VER
__declspec(dllexport)
#endif
void
distinct_count_add(UDF_INIT* initid, UDF_ARGS* args,
char* is_null,
char* message __attribute__((unused)))
{
}
#ifdef _MSC_VER
__declspec(dllexport)
#endif
long long distinct_count(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)),
char* is_null, char* error __attribute__((unused)))
{
return 0;
}
}
// vim:ts=4 sw=4: