From 32ebc3e392fdba4a984288291531bcfb7acbd5fd Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 11 Dec 2017 10:22:20 -0600 Subject: [PATCH 01/41] MCOL-1047 - fixed debian 9 lib dep --- utils/clusterTester/columnstoreClusterTester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index acc39d23a..78fa3f889 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -978,7 +978,7 @@ checkPackages() fi fi - declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libioa1") + declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") if [ "$OS" == "debian9" ]; then if [ ! `which dpkg 2>/dev/null` ] ; then From bea47cc657f3f46ac359a11f32b1ac1e25f3d19e Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 12 Dec 2017 22:53:33 +0300 Subject: [PATCH 02/41] MCOL-939 Moved columnstore version information to columnstoreversion.h for 1.1.X --- CMakeLists.txt | 5 +- build/BuildCalpontVersion.bat | 18 -- build/bootstrap | 60 ----- build/bootstrap.4.5 | 2 +- build/build_rpms | 143 ------------ build/genVersion.sh | 49 ---- build/make_src_tar | 135 ----------- build/releasenum.in | 4 +- columnstoreversion.h.in | 7 + dbcon/mysql/ha_calpont.cpp | 6 +- dbcon/mysql/ha_calpont_impl.cpp | 4 +- oam/install_scripts/softwareReport.sh | 9 - oam/oamcpp/liboamcpp.cpp | 68 +----- oamapps/calpontConsole/calpontConsole.cpp | 21 +- oamapps/calpontSupport/softwareReport.sh | 53 ----- oamapps/columnstoreSupport/softwareReport.sh | 9 +- oamapps/mcsadmin/mcsadmin.cpp | 15 +- oamapps/postConfigure/amazonInstaller.cpp | 11 +- oamapps/postConfigure/postConfigure.cpp | 1 + oamapps/serverMonitor/diskMonitor.cpp | 2 +- procmgr/main.cpp | 20 +- procmgr/processmanager.cpp | 7 +- procmon/processmonitor.cpp | 15 +- tools/configMgt/buildCalpontPackages.sh | 48 ---- tools/configMgt/calpont-build | 102 --------- tools/configMgt/dm_parent_installer.sh | 194 ---------------- tools/configMgt/dm_parent_installer_v3.sh | 191 ---------------- tools/configMgt/parent_binary_installer.sh | 215 ------------------ tools/configMgt/parent_binary_installer_v3.sh | 212 ----------------- tools/configMgt/pm_parent_installer.sh | 188 --------------- tools/configMgt/pm_parent_installer_v3.sh | 179 --------------- tools/configMgt/stackReleaseChecker.cpp | 177 -------------- utils/configcpp/libconfigcpp.vcxproj | 4 +- utils/configcpp/libconfigcpp.vcxproj.filters | 4 +- 34 files changed, 53 insertions(+), 2125 deletions(-) delete mode 100644 build/BuildCalpontVersion.bat delete mode 100755 build/bootstrap delete mode 100755 build/build_rpms delete mode 100755 build/genVersion.sh delete mode 100755 build/make_src_tar create mode 100644 columnstoreversion.h.in delete mode 100755 oamapps/calpontSupport/softwareReport.sh delete mode 100755 tools/configMgt/buildCalpontPackages.sh delete mode 100755 tools/configMgt/calpont-build delete mode 100755 tools/configMgt/dm_parent_installer.sh delete mode 100755 tools/configMgt/dm_parent_installer_v3.sh delete mode 100755 tools/configMgt/parent_binary_installer.sh delete mode 100755 tools/configMgt/parent_binary_installer_v3.sh delete mode 100755 tools/configMgt/pm_parent_installer.sh delete mode 100755 tools/configMgt/pm_parent_installer_v3.sh delete mode 100644 tools/configMgt/stackReleaseChecker.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 51d2e7a8a..1e46e2ad3 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,8 @@ ENDIF(NOT CMAKE_BUILD_TYPE) #set( CMAKE_VERBOSE_MAKEFILE on ) INCLUDE(columnstore_version.cmake) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) + SET (PACKAGE columnstore) SET (PACKAGE_NAME columnstore) @@ -76,8 +78,7 @@ ENDIF("${isSystemDir}" STREQUAL "-1") INCLUDE (configureEngine.cmake) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum @ONLY IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL_ENGINE} COMPONENT platform) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) INCLUDE(bison.cmake) diff --git a/build/BuildCalpontVersion.bat b/build/BuildCalpontVersion.bat deleted file mode 100644 index b2e8c62dd..000000000 --- a/build/BuildCalpontVersion.bat +++ /dev/null @@ -1,18 +0,0 @@ -for /f "delims= skip=1" %%x in (releasenum) do (set "%%x") -set packager="%USERNAME% <%USERNAME%@calpont.com>" -echo %packager% -set builddate="%DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2% %TIME:~0,2%:%TIME:~3,2%:%TIME:~6,2%" -set buildmachine=%computername% -copy /Y .\CalpontVersion.txt.in ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@PACKAGER@@/%packager%/ ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@BUILDDATE@@/%builddate%/ ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@VERSION@@/%version%/ ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@RELEASE@@/%release%/ ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@BUILDMACHINE@@/%buildmachine%/ ..\utils\winport\CalpontVersion.txt -:: For some unknown reason, sed removes permissions. Put them back. -icacls ..\utils\winport\CalpontVersion.txt /grant SYSTEM:F administrators:F -cd .. -git status | grep "On branch" >> utils\winport\CalpontVersion.txt -git log -1 | head -3 | egrep "^(commit|Date)" >> utils\winport\CalpontVersion.txt - -cd build diff --git a/build/bootstrap b/build/bootstrap deleted file mode 100755 index e7dfc1f97..000000000 --- a/build/bootstrap +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# $Id: bootstrap 1333 2011-01-18 17:25:26Z rdempsey $ -# -# This script is responsible for setting up a fresh development tree -# following a git checkout. It copies static files and shared include -# files to the export tree prior to building the software. - -set -e - -mkdir -p export/mysql -#FIXME rename this -cp dbcon/mysql/install_calpont_mysql.sh export/mysql/ - -#TODO: see if this my.cnf is necessary -cp dbcon/mysql/my.cnf export/mysql/. - -#TODO: the init.d service file should not be mixed case -#TODO: the init.d service file should not contain calpont -cp dbcon/mysql/mysql-Columnstore export/mysql/ && \ -cp build/files.lst export/mysql/ && \ -cp dbcon/mysql/install_calpont_mysql.sh export/mysql/ && \ -cp dbcon/mysql/my.cnf export/mysql/ && \ -cp dbcon/mysql/mysql-Columnstore export/mysql/ - -#FIXME: -#This is not going to update the version in configure.ac -echo Generating version header -./build/genVersion.sh -if [ $? -ne 0 ]; then - echo Could not generate version header - exit $? -fi - -echo Running GNU autotools to generate build environment -aclocal && \ -autoconf && \ -autoheader && \ -libtoolize && \ -automake --add-missing && \ -autoreconf - -if [ $? -ne 0 ]; then - echo GNU autotools generation failed - exit $? -fi - - -echo GNU autotools environment generation completed successfully - -#cd net-snmp/net-snmp -# ./configure --prefix=$PREFIX --with-mib-modules=disman/event-mib --with-cc=gcc \ -# --with-logfile=$PREFIX/log/snmpd.log \ -# --with-sys-location=Unknown \ -# --with-sys-contact=root@localhost.localdomain \ -# --with-default-snmp-version=3 \ -# --with-persistent-directory=$PREFIX/var/net-snmp \ -# --without-openssl \ -# --with-ldflags="-Wl,-rpath -Wl,$PREFIX/lib" - diff --git a/build/bootstrap.4.5 b/build/bootstrap.4.5 index b8be8a899..ac400b225 100755 --- a/build/bootstrap.4.5 +++ b/build/bootstrap.4.5 @@ -49,7 +49,7 @@ for lib in libnetsnmpagent.so libnetsnmphelpers.so libnetsnmpmibs.so libnetsnmp. done popd >/dev/null -./build/genVersion.sh +#./build/genVersion.sh for x in \ $(find . -name '[mM]akefile' \ diff --git a/build/build_rpms b/build/build_rpms deleted file mode 100755 index 7af7e689b..000000000 --- a/build/build_rpms +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash -# -# $Id: build_rpms 1734 2012-10-25 18:07:16Z dlhill $ -# - -verbose=0 - -prefix=/usr/local/mariadb/columnstore -#mysqldir=$prefix/mariadb-columnstore/mysql -columnstoredir=$prefix - -# set default as community -community=true - -usage() -{ -cat <&2 - fi -done - -#try to find project root -while [ ! -d dbcon ]; do - cd .. - curdir=$(pwd) - if [ $curdir = / -o $curdir = $HOME ]; then - echo "I could not find the project root directory: I can't continue!" - exit 1 - fi -done - -if [ ! -f ./build/releasenum ]; then - echo "I could not find the file 'releasesum' in the build directory: I can't continue!" - exit 1 -fi - -. ./build/releasenum - -# make the install package -buildroot=$HOME/rpm/BUILD/mariadb-columnstore-$version.$release - -sudo rm -rf $buildroot -mkdir -p $buildroot -mkdir -p $buildroot/mariadb/columnstore -echo -n "Writing binary files to $buildroot..." -sudo chown -R builder:builder $columnstoredir -/bin/cp -f -r $columnstoredir/* $buildroot/mariadb/columnstore/. - -/bin/cp -r build/releasenum $buildroot/mariadb/columnstore/. - -#get mysql server files -#pushd $mysqldir > /dev/null -#mkdir -p $buildroot/mariadb/columnstore/mysql -#cpio -pdmu --quiet $buildroot/mariadb/columnstore/mysql < files.lst -#popd >/dev/null - -#clean up some files from buildroot -rm -f $buildroot/mariadb/columnstore/etc/Doxyfile -for file in $buildroot/mariadb/columnstore/lib/*; do - if [ -h $file ]; then - rm -f $file - fi -done - -#temporarily clean up these BRM thingies -for file in dbrm script-fcns; do - rm -f $buildroot/mariadb/columnstore/bin/$file -done - -#clean up these bulkload scripts -for file in bulkload.sh cleanup.sh dbbuilder.sh dbload.sh cpimport.sh bulkloadp.sh; do - rm -f $buildroot/mariadb/columnstore/bin/$file -done - -rm -f $buildroot/mariadb/columnstore/lib/libidbboot.a -rm -f $buildroot/mariadb/columnstore/lib/libiberty.a - -mkdir -p $HOME/rpm/BUILD $HOME/rpm/RPMS $HOME/rpm/SRPMS $HOME/rpm/SPECS $HOME/rpm/tmp - -if [ ! -f $HOME/.rpmmacros ]; then - cat > $HOME/.rpmmacros < /tmp/rpmbuild.out.$$ 2>&1 - -rc=$? -echo done. -if [ $verbose -gt 0 -o $rc -ne 0 ]; then - cat /tmp/rpmbuild.out.$$ -fi - -rm -f /tmp/*.$$ - -exit $rc - diff --git a/build/genVersion.sh b/build/genVersion.sh deleted file mode 100755 index 4d6c2eefd..000000000 --- a/build/genVersion.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# $Id: genVersion.sh 1019 2009-12-18 14:42:35Z rdempsey $ -# - -#prefix=export -#for arg in "$@"; do -# if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then -# prefix="`echo $arg | awk -F= '{print $2}'`" -# else -# echo "ignoring unknown argument: $arg" 1>&2 -# fi -#done -# -##try to find project root -#while [ ! -d dbcon ]; do -# cd .. -# curdir=$(pwd) -# if [ $curdir = / -o $curdir = $HOME ]; then -# echo "I could not find the project root directory: I can't continue!" -# exit 1 -# fi -#done -# -#if [ ! -f ./build/releasenum ]; then -# echo "I could not find the file 'releasesum' in the build directory: I can't continue!" -# exit 1 -#fi - -. ./build/releasenum - -#mkdir -p ${prefix}/include - -echo " -#ifndef VERSIONNUMBER_H_ -#define VERSIONNUMBER_H_ -#include -const std::string idb_version(\"$version\"); -const std::string idb_release(\"$release\"); -#endif -" > dbcon/mysql/versionnumber.h.tmp - -diff -bBq dbcon/mysql/versionnumber.h.tmp dbcon/mysql/versionnumber.h >/dev/null 2>&1 -if [ $? -ne 0 ]; then - cp dbcon/mysql/versionnumber.h.tmp dbcon/mysql/versionnumber.h -fi - -rm -f dbcon/mysql/versionnumber.h.tmp - diff --git a/build/make_src_tar b/build/make_src_tar deleted file mode 100755 index 098d75ff7..000000000 --- a/build/make_src_tar +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash -# -# $Id: make_src_tar 1932 2013-06-20 18:26:27Z dhill $ -# - -# basedir is the working dir for all the activity -basedir=$HOME -# vname is the name of the top-level dir in the tar file -vname= -# ofile is the output tar file name and is relative to a hidden dir, so it should be absolute -ofile= -# branch is the branch to checkout (develop by default) -branch= -for arg in "$@"; do - if [ $(expr -- "$arg" : '--basedir=') -eq 10 ]; then - basedir="$(echo $arg | awk -F= '{print $2}')" - elif [ $(expr -- "$arg" : '--vname=') -eq 8 ]; then - vname="$(echo $arg | awk -F= '{print $2}')" - elif [ $(expr -- "$arg" : '--ofile=') -eq 8 ]; then - ofile="$(echo $arg | awk -F= '{print $2}')" - elif [ $(expr -- "$arg" : '--branch=') -eq 9 ]; then - branch="$(echo $arg | awk -F= '{print $2}')" - else - echo "ignoring unknown argument: $arg" 1>&2 - fi -done - -if [ -f releasenum ]; then - . releasenum -elif [ -f build/releasenum ]; then - . build/releasenum -else - echo "Couldn't locate 'releasenum' file in the build dir, make sure running in top-level dir" -fi - -if [ -z "$version" ]; then - echo "Version defaulting to 4.0-develop" - version=4.0 - release=develop -fi - -if [ -z "$release" ]; then - release=0 -fi - -blddir=bld.$$ -if [ -z "$vname" ]; then - vname=infinidb-${version} -fi -if [ -z "$ofile" ]; then - ofile=${basedir}/${vname}-${release}.src.tar.gz -fi -co_dir=$blddir/$vname - -trap "{ echo; cd ${basedir}; rm -rf $blddir; exit 1; }" TERM INT ERR - -cd ${basedir} -rm -rf $blddir -mkdir -p $co_dir - -cd $co_dir - -branch_arg= -if [ ! -z "$branch" ]; then - branch_arg="-b $branch" -else - branch=develop -fi - -echo -n "getting git source from $branch..." -git clone http://srvengcm1.calpont.com/repos/infinidb.git infinidb -cd infinidb -git checkout $branch -echo "done." - -test -d build || exit 1 - -echo -n "cleaning git source for " -echo -n "release..." -rm -rf .git .gitignore -rm -rf qa export -rm -f rules.mak -find . -name Makefile | xargs rm -f -find . -name \*.xmi | xargs rm -f -rm -f dbcon/joblist/bandeddl.h.set dbcon/joblist/bandeddl.h.vector primitives/blockcache/iomanager.cpp.single-read \ - primitives/udf/Makefile.sdk -rm -rf utils/scenarios -cd mysql -rm -rf scripts queries storage/maria -cd .. -rm -rf writeengine/test/* -rm -rf utils/net-snmp-built-libs -rm -rf utils/multicast -rm -rf utils/openpgm -rm -rf tools/reserveStacks -rm -rf dbcon/doc dbcon/ingres dbcon/plsql -cd tools -rm -rf brmtest dataPump evalidx fixdate nettest pgm pingproc rebuildEM snconvert updateGenii valprint -rm -rf cfread dbgen dbgen-ssb ddldriver dmldriver idb_comp sendPlan vbgen -cd .. -rm -rf utils/libstatgrab utils/libxml utils/testbc utils/configcpp/md5 -echo "done." - -cd .. - -echo -n "getting MySQL..." -git clone https://github.com/infinidb/mysql mysql -cd mysql -git checkout $branch -echo "done." - -cd .. - -echo -n "building tarball..." -cd infinidb -cp -r utils/autoconf/* . -cp INSTALL INSTALL.keep -cp install-sh install-sh.keep -rm -rf utils/autoconf -autoreconf --force --install -rm -f INSTALL install-sh -mv INSTALL.keep INSTALL -mv install-sh.keep install-sh -cd .. -ln -s infinidb/INSTALL -ln -s infinidb/README -cd .. -tar -zcf $ofile $vname -echo "done." - -echo "tarball left in $ofile" - -cd ${basedir} -rm -rf $blddir - diff --git a/build/releasenum.in b/build/releasenum.in index 1f5e4b4dc..7461afcf3 100644 --- a/build/releasenum.in +++ b/build/releasenum.in @@ -1,2 +1,2 @@ -version=@PACKAGE_VERSION@ -release=@PACKAGE_RELEASE@ \ No newline at end of file +version=${PACKAGE_VERSION} +release=${PACKAGE_RELEASE} diff --git a/columnstoreversion.h.in b/columnstoreversion.h.in new file mode 100644 index 000000000..ec642aa92 --- /dev/null +++ b/columnstoreversion.h.in @@ -0,0 +1,7 @@ +#ifndef VERSIONNUMBER_H_ +#define VERSIONNUMBER_H_ +#include +const std::string columnstore_version("${VERSION}"); +const std::string columnstore_release("${PACKAGE_RELEASE}"); +#endif + diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 16e1e0def..2cdc093f3 100755 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -108,7 +108,7 @@ */ #include "ha_calpont.h" -#include "versionnumber.h" +#include "columnstoreversion.h" #define NEED_CALPONT_EXTERNS #include "ha_calpont_impl.h" @@ -201,7 +201,7 @@ static int columnstore_init_func(void *p) tm.tm_year % 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", idb_version.c_str(), idb_release.c_str()); + fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(), columnstore_release.c_str()); calpont_hton= (handlerton *)p; #ifndef _MSC_VER @@ -234,7 +234,7 @@ static int infinidb_init_func(void *p) tm.tm_year % 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - fprintf(stderr, "InfiniDB: Started; Version: %s-%s\n", idb_version.c_str(), idb_release.c_str()); + fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(), columnstore_release.c_str()); calpont_hton= (handlerton *)p; diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 781047106..d4669c8d5 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -141,7 +141,7 @@ using namespace logging; using namespace funcexp; #include "installdir.h" -#include "versionnumber.h" +#include "columnstoreversion.h" namespace cal_impl_if { @@ -2286,7 +2286,7 @@ const char* calgetversion(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, char* error) { - string version(idb_version); + string version(columnstore_version); *length = version.size(); memcpy(result, version.c_str(), *length); return result; diff --git a/oam/install_scripts/softwareReport.sh b/oam/install_scripts/softwareReport.sh index ee65f7332..09fcb4db6 100755 --- a/oam/install_scripts/softwareReport.sh +++ b/oam/install_scripts/softwareReport.sh @@ -36,15 +36,6 @@ echo "################# mcsadmin getcolumnstoresoftwareinfo #################" echo " " $INSTALLDIR/bin/mcsadmin getsoftwareinfo -if [ -e $INSTALLDIR/releasenum ];then - echo " " - echo "-- Columnstore Release Number file --" - echo " " - echo "################# cat $INSTALLDIR/releasenum #################" - echo " " - cat $INSTALLDIR/releasenum -fi - echo " " echo "-- Columnstore Storage Configuration --" echo " " diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index e28e21aa6..7bdfb27b8 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -46,6 +46,7 @@ #include #include +#include "columnstoreversion.h" #include "ddlpkg.h" #include "../../dbcon/dmlpackage/dmlpkg.h" #define LIBOAM_DLLEXPORT @@ -148,74 +149,13 @@ namespace oam /******************************************************************** * * get System Software information - * + * (for backward compatibility only) ********************************************************************/ void Oam::getSystemSoftware(SystemSoftware& systemsoftware) { - // parse releasenum file - - string rn = InstallDir + "/releasenum"; - ifstream File(rn.c_str()); - if (!File) - // Open File error - return; - - char line[400]; - string buf; - while (File.getline(line, 400)) - { - buf = line; - for ( unsigned int i = 0;; i++) - { - if ( SoftwareData[i] == "") - //end of list - break; - - string data = ""; - string::size_type pos = buf.find(SoftwareData[i],0); - if (pos != string::npos) - { - string::size_type pos1 = buf.find("=",pos); - if (pos1 != string::npos) - { - data = buf.substr(pos1+1, 80); - } - else - // parse error - exceptionControl("getSystemSoftware", API_FAILURE); - - //strip off any leading or trailing spaces - for(;;) - { - string::size_type pos = data.find(' ',0); - if (pos == string::npos) - // no more found - break; - // strip leading - if (pos == 0) { - data = data.substr(pos+1,10000); - } - else - { // strip trailing - data = data.substr(0, pos); - } - } - - switch(i) - { - case(0): // line up with SoftwareData[] - systemsoftware.Version = data; - break; - case(1): - systemsoftware.Release = data; - break; - } - } - } //end of for loop - } //end of while - - File.close(); + systemsoftware.Version = columnstore_version; + systemsoftware.Release = columnstore_release; } /******************************************************************** * diff --git a/oamapps/calpontConsole/calpontConsole.cpp b/oamapps/calpontConsole/calpontConsole.cpp index 74c8e633a..7838284b1 100644 --- a/oamapps/calpontConsole/calpontConsole.cpp +++ b/oamapps/calpontConsole/calpontConsole.cpp @@ -25,6 +25,7 @@ #include extern int h_errno; +#include "columnstoreversion.h" #include "calpontConsole.h" #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" @@ -703,7 +704,6 @@ int processCommand(string* arguments) case 4: // getSystemConfig { - SystemSoftware systemsoftware; SystemConfig systemconfig; string returnValue; @@ -714,14 +714,13 @@ int processCommand(string* arguments) try { - oam.getSystemSoftware(systemsoftware); oam.getSystemConfig(systemconfig); cout << endl << "System Configuration" << endl << endl; cout << "SystemName = " << systemconfig.SystemName << endl; - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; cout << "ParentOAMModuleName = " << systemconfig.ParentOAMModule << endl; cout << "StandbyOAMModuleName = " << systemconfig.StandbyOAMModule << endl; @@ -4836,21 +4835,15 @@ int processCommand(string* arguments) if (oam.checkLogStatus("/tmp/calpont.txt", "Status: install")) system("cat /tmp/calpont.txt"); else { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; } } } else { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; } cout << endl; break; diff --git a/oamapps/calpontSupport/softwareReport.sh b/oamapps/calpontSupport/softwareReport.sh deleted file mode 100755 index 81c57c715..000000000 --- a/oamapps/calpontSupport/softwareReport.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# -# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - INSTALLDIR=$2 -else - INSTALLDIR="/usr/local/mariadb/columnstore" -fi - -if [ $USER = "root" ]; then - SUDO=" " -else - SUDO="sudo" -fi - -$SUDO rm -f /tmp/${MODULE}_softwareReport.txt - -{ -echo " " -echo "******************** Software Report for ${MODULE} ********************" -echo " " - -echo " " -echo "-- Calpont Package Details --" -echo " " -echo "################# mcsadmin getcalpontsoftwareinfo #################" -echo " " -$INSTALLDIR/bin/mcsadmin getcalpontsoftwareinfo - -echo " " -echo "-- Calpont Release Number file --" -echo " " -echo "################# cat $INSTALLDIR/releasenum #################" -echo " " -cat $INSTALLDIR/releasenum - -echo " " -echo "-- Calpont Storage Configuration --" -echo " " -echo "################# mcsadmin getStorageConfig #################" -echo " " -$INSTALLDIR/bin/mcsadmin getStorageConfig - -} > /tmp/${MODULE}_softwareReport.txt - -exit 0 \ No newline at end of file diff --git a/oamapps/columnstoreSupport/softwareReport.sh b/oamapps/columnstoreSupport/softwareReport.sh index cb1468813..b7a6e90b0 100755 --- a/oamapps/columnstoreSupport/softwareReport.sh +++ b/oamapps/columnstoreSupport/softwareReport.sh @@ -33,14 +33,7 @@ echo "-- Columnstore Package Details --" echo " " echo "################# mcsadmin getcolumnstoresoftwareinfo #################" echo " " -$INSTALLDIR/bin/mcsadmin getcolumnstoresoftwareinfo - -echo " " -echo "-- Columnstore Release Number file --" -echo " " -echo "################# cat $INSTALLDIR/releasenum #################" -echo " " -cat $INSTALLDIR/releasenum +$INSTALLDIR/bin/mcsadmin getsoftwareinfo echo " " echo "-- Columnstore Storage Configuration --" diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 93dd701df..9c436a3a6 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -26,6 +26,7 @@ #include extern int h_errno; +#include "columnstoreversion.h" #include "mcsadmin.h" #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" @@ -5046,21 +5047,15 @@ int processCommand(string* arguments) if (WEXITSTATUS(rtnCode) == 0) system("cat /tmp/columnstore.txt"); else { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; } } } else { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; } cout << endl; break; diff --git a/oamapps/postConfigure/amazonInstaller.cpp b/oamapps/postConfigure/amazonInstaller.cpp index 9268d0f28..48f5f3899 100644 --- a/oamapps/postConfigure/amazonInstaller.cpp +++ b/oamapps/postConfigure/amazonInstaller.cpp @@ -46,6 +46,7 @@ #include #include +#include "columnstoreversion.h" #include "liboamcpp.h" #include "configcpp.h" #include "alarmmanager.h" @@ -220,10 +221,7 @@ int main(int argc, char *argv[]) postConfigureLog = true; } else if( string("-v") == argv[i] ) { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << endl << "columnstore Version: " << systemsoftware.Version << "-" << systemsoftware.Release << endl; + cout << "columnstore Version: " << columnstore_version << "-" << columnstore_release << endl; exit (0); } else if( string("-pc") == argv[i] ) { @@ -1289,10 +1287,7 @@ int main(int argc, char *argv[]) cout << "===== Setting up system '" + systemName + "' based on these settings ===== " << endl << endl; - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "columnstore Version = " << systemsoftware.Version << "-" << systemsoftware.Release << endl; + cout << "columnstore Version = " << columnstore_version << "-" << columnstore_release << endl; cout << "System Type = " << systemType << endl; if ( subnetID != oam::UnassignedName ) { diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 8a543f82b..9f7505f6b 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -69,6 +69,7 @@ #include "boost/filesystem/path.hpp" #include "boost/tokenizer.hpp" +#include "columnstoreversion.h" #include "liboamcpp.h" #include "configcpp.h" diff --git a/oamapps/serverMonitor/diskMonitor.cpp b/oamapps/serverMonitor/diskMonitor.cpp index af55b4172..2edcaa2f6 100644 --- a/oamapps/serverMonitor/diskMonitor.cpp +++ b/oamapps/serverMonitor/diskMonitor.cpp @@ -223,7 +223,7 @@ void diskMonitor() string fileName; // check local if ( deviceName == "/") { - fileName = deviceName + "usr/local/mariadb/columnstore/releasenum"; + fileName = deviceName + "usr/local/mariadb/columnstore/"; } else { diff --git a/procmgr/main.cpp b/procmgr/main.cpp index b6f51a609..860775780 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -27,6 +27,7 @@ #include +#include "columnstoreversion.h" #include "processmanager.h" #include "installdir.h" @@ -507,24 +508,7 @@ static void startMgrProcessThread() log.writeLog(__LINE__, "startMgrProcessThread launched", LOG_TYPE_DEBUG); //get calpont software version and release - SystemSoftware systemsoftware; - string softwareVersion; - string softwareRelease; - - try - { - oam.getSystemSoftware(systemsoftware); - - softwareVersion = systemsoftware.Version; - softwareRelease = systemsoftware.Release; - } - catch (exception& e) { - cout << endl << "ProcMon Construct Error reading getSystemSoftware = " << e.what() << endl; - exit(-1); - } - - string localSoftwareInfo = softwareVersion + softwareRelease; - + string localSoftwareInfo = columnstore_version + columnstore_release; //get systemStartupOffline string systemStartupOffline = "n"; try { diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 403bbbdda..118591c09 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -24,6 +24,7 @@ //#define NDEBUG #include +#include "columnstoreversion.h" #include "processmanager.h" #include "installdir.h" #include "dbrm.h" @@ -4565,12 +4566,12 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } if ( packageType == "rpm") - calpontPackage = homedir + "/mariadb-columnstore*" + systemsoftware.Version + "-" + systemsoftware.Release + "*.rpm.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm.tar.gz"; else if ( packageType == "deb") - calpontPackage = homedir + "/mariadb-columnstore*" + systemsoftware.Version + "-" + systemsoftware.Release + "*.deb.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb.tar.gz"; else - calpontPackage = homedir + "/mariadb-columnstore*" + systemsoftware.Version + "-" + systemsoftware.Release + "*.bin.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.bin.tar.gz"; if ( DistributedInstall == "y" ) { diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 90679ee28..148546a72 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -23,6 +23,7 @@ #include +#include "columnstoreversion.h" #include "IDBDataFile.h" #include "IDBPolicy.h" #include "processmonitor.h" @@ -110,18 +111,8 @@ MonitorConfig::MonitorConfig() // } //get calpont software version and release - SystemSoftware systemsoftware; - - try - { - oam.getSystemSoftware(systemsoftware); - - fsoftwareVersion = systemsoftware.Version; - fsoftwareRelease = systemsoftware.Release; - } - catch (exception& e) { - cout << endl << "ProcMon Construct Error reading getSystemSoftware = " << e.what() << endl; - } + fsoftwareVersion = columnstore_version; + fsoftwareRelease = columnstore_release; } diff --git a/tools/configMgt/buildCalpontPackages.sh b/tools/configMgt/buildCalpontPackages.sh deleted file mode 100755 index 4ed8ebe51..000000000 --- a/tools/configMgt/buildCalpontPackages.sh +++ /dev/null @@ -1,48 +0,0 @@ -#! /bin/sh -# -# $1 - release number or 'Latest' (default 'Latest') -# -if [ "$1" = "" ] ; then - DIR=Latest - REL="3.6-0" -elif [ "$1" = "Latest" ] ; then - DIR=Latest - REL="3.6-0" -elif [ "$2" = "" ] ; then - DIR=$1 - REL=$1 -else - DIR=$1 - REL=$2 -fi -# -if [[ $REL != *-* ]] ; then - REL=$REL"-0" -fi -# -{ -cd /root/autoOAM/ -rpm -e infinidb-libs infinidb-platform infinidb-enterprise --nodeps -rpm -e infinidb-storage-engine infinidb-mysql --nodeps -rpm -e infinidb-mysql --nodeps -rm -rf /usr/local/Calpont -rm -rf calpont* -rm -f *gz -# -smbclient //srvhill01/shared -WMARIADB -Uroot%Calpont1 -c "cd packages/$DIR;prompt OFF;mget *.rpm" -rpm -ivh calpont*.x86_64.rpm --nodeps -rpm -iq calpont >> /usr/local/Calpont/releasenum -cd /usr/local/ -tar -zcvf calpont-infinidb-ent-$REL.x86_64.bin.tar.gz Calpont -mv calpont-infinidb-ent-$REL.x86_64.bin.tar.gz /root/autoOAM/ -cd /root/autoOAM/ -alien -ck calpont*.x86_64.rpm -tar -zcvf calpont-infinidb-ent-$REL.x86_64.rpm.tar.gz *$REL*.rpm -tar -zcvf calpont-infinidb-ent-$REL.amd64.deb.tar.gz *$REL*.deb - -smbclient //srvhill01/shared -WMARIADB -Uroot%Calpont1 -c "cd packages/$DIR;mkdir packages;cd packages;prompt OFF;del calpont-infinidb-ent*gz;mput *gz" -} > /root/autoOAM/buildCalpontPackages-$DIR.log 2>&1 -# -echo "Calpont Packages Build Successfully Completed" -exit 0 - diff --git a/tools/configMgt/calpont-build b/tools/configMgt/calpont-build deleted file mode 100755 index 6b0408e15..000000000 --- a/tools/configMgt/calpont-build +++ /dev/null @@ -1,102 +0,0 @@ -#! /bin/sh -# -# $Id: calpont-build 421 2007-14-15 15:46:55Z dhill $ -# -# build calpont rpm and install on call6500 -# $1 - release number or 'Latest' -# $2 - svn branch -# -if [ "$2" = "" ] ; then - echo "enter release and svn-branch arguments" - exit 0 -fi -# !!! set for OS type 32 or 64 bit !!! - -OS=x86_64 - -# -REL=$1 -DIR=$2 - -#remove mysqld before new one is built -#prevents old version from being packaged in new build -rm -f /usr/local/Calpont/mysql/libexec/* - -if [ $DIR = "genii" ] ; then - ./mysql-build-genii > mysql-build-genii.log & -else - ./mysql-build-branch $DIR > mysql-build-$DIR.log & -fi -# -HOME=/home/nightly/$DIR -rm -rf $HOME -rm -fr /home/nightly/rpm/ -# - -if [ $DIR = "genii" ] ; then - su - nightly -c "export CXXFLAGS='-O3 -fno-strict-aliasing ';mkdir $DIR;updateGenii.pl > ~/$DIR/updateGenii.log 2>&1;cd ~/$DIR;make clean;make > ~/$DIR/make.log 2>&1;make > ~/$DIR/make1.log 2>&1;./build/build_rpms > ~/$DIR/build_rpms.log 2>&1" -else - su - nightly -c "export CXXFLAGS='-O3 -fno-strict-aliasing ';mkdir $DIR;updateGenii.pl -b $DIR > ~/$DIR/updateGenii.log 2>&1;cd ~/$DIR;make clean;make > ~/$DIR/make.log 2>&1;make > ~/$DIR/make1.log 2>&1;./build/build_rpms > ~/$DIR/build_rpms.log 2>&1" -fi - -# -# generate rpm info text files - -if [ ! -f $HOME/build/releasenum ]; then - #default - version=3.0 -else - . $HOME/build/releasenum -fi - -cd /home/nightly/rpm/ > /dev/null 2>&1 -cd RPMS/$OS/ > /dev/null 2>&1 -test -f infinidb-platform-$version*.rpm || echo "no rpm" -test -f infinidb-platform-$version*.rpm || exit -1 -rpm -qi -p infinidb-platform-$version*.rpm > infinidb-platform-$OS-rpm-info.txt -echo " " >> infinidb-platform-$OS-rpm-info.txt -echo "MD5SUM" >> infinidb-platform-$OS-rpm-info.txt -md5sum infinidb-platform-$version*.rpm >> infinidb-platform-$OS-rpm-info.txt -echo " " >> infinidb-platform-$OS-rpm-info.txt -# -test -f infinidb-storage-engine-*.rpm || echo "no rpm" -test -f infinidb-storage-engine-*.rpm || exit -1 -rpm -qi -p infinidb-storage-engine-*.rpm > infinidb-storage-engine-$OS-rpm-info.txt -echo " " >> infinidb-storage-engine-$OS-rpm-info.txt -echo "MD5SUM" >> infinidb-storage-engine-$OS-rpm-info.txt -md5sum infinidb-storage-engine-*.rpm >> infinidb-storage-engine-$OS-rpm-info.txt -echo " " >> infinidb-storage-engine-$OS-rpm-info.txt -# -test -f infinidb-mysql-*.rpm || echo "no rpm" -test -f infinidb-mysql-*.rpm || exit -1 -rpm -qi -p infinidb-mysql-*.rpm > infinidb-mysql-$OS-rpm-info.txt -echo " " >> infinidb-mysql-$OS-rpm-info.txt -echo "MD5SUM" >> infinidb-mysql-$OS-rpm-info.txt -md5sum infinidb-mysql-*.rpm >> infinidb-mysql-$OS-rpm-info.txt -echo " " >> infinidb-mysql-$OS-rpm-info.txt -# -{ -if [ $REL != "Latest" ] ; then - # copy rpms to calweb - DATE=`date +%F` - cd /home/nightly/rpm/RPMS/$OS/ - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations;mkdir $REL;cd $REL;rm *.$OS.rpm;prompt OFF;mput *" - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/nightly;mkdir $REL-$DATE;cd $REL-$DATE;rm *.$OS.rpm;prompt OFF;mput *" - cd /home/nightly/$DIR/export/etc/ - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/$REL;rm Columnstore.xml;prompt OFF;mput Columnstore.xml" - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/nightly/$REL-$DATE;rm Columnstore.xml;prompt OFF;mput Columnstore.xml" - -else - # copy nightly on calweb - DATE=`date +%F` - cd /home/nightly/rpm/RPMS/$OS/ - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/Latest;rm *.$OS.rpm;prompt OFF;mput *" - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/nightly;mkdir genii-$DATE;cd genii-$DATE;prompt OFF;mput *" - cd /home/nightly/$DIR/export/etc/ - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/Latest;rm Columnstore.xml;prompt OFF;mput Columnstore.xml" - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/nightly/genii-$DATE;prompt OFF;mput Columnstore.xml" -fi -} 2> /root/infinidb-build-$REL-$DIR.log -# -echo "Build Successfully Completed" -exit 0 diff --git a/tools/configMgt/dm_parent_installer.sh b/tools/configMgt/dm_parent_installer.sh deleted file mode 100755 index 57498924c..000000000 --- a/tools/configMgt/dm_parent_installer.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/expect -# -# $Id$ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -#set PACKAGE [lindex $argv 2] -set PACKAGE *.rpm -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set USERNAME [lindex $argv 5] -set CEFLAG [lindex $argv 6] -set PACKAGENAME [lindex $argv 7] -set INSTALLLOCATION [lindex $argv 8] -set DEBUG [lindex $argv 9] - -set INSTALLDIR "/usr/local" - -set SHARED "//srvhill01/shared" - -log_user $DEBUG -spawn -noecho /bin/bash - -set CONFIG "Calpont" -if { $PACKAGENAME != "columnstore"} { - set CONFIG "Calpont" -} - - -set timeout 10 -send "ssh $USERNAME@$SERVER 'rm -f /root/Calpont-*.rpm /root/$PACKAGENAME*.rpm'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } -} -expect { - -re {[$#] } { } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -# -# erase InfiniDB packages -# -set timeout 60 -send_user "Erase Old InfiniDB Packages " -send "ssh $USERNAME@$SERVER 'pkill -9 mysqld'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re {[$#] } { } -} -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^mariadb-columnstore') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } - -re {[$#] } { } - -re "not installed" { } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/$INSTALLLOCATION/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall && $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re {[$#] } { send_user "DONE" } - -re "uninstall completed" { send_user "DONE" } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# get the InfiniDB package -# -send_user "Get InfiniDB Packages " -send "smbclient $SHARED -WMARIADB -Uroot%Calpont1 -c 'cd packages/$RELEASE/;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } -} -send_user "\n" -# -# send the InfiniDB package -# -set timeout 120 -send_user "Copy InfiniDB Packages " -send "scp -q $PACKAGE $USERNAME@$SERVER:/root/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re {[$#] } { send_user "DONE" } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -#send "rm -f $PACKAGE\n" -# -# install InfiniDB package -# -send_user "Install New InfiniDB Packages " -send "ssh $USERNAME@$SERVER ' rpm -iv --nodeps --force $PACKAGENAME-*'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { expect { - -re "completed" { expect { - -re "completed" { send_user "DONE" } - } - } - } - } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -set timeout 120 -expect -re {[$#] } -# -if { $CONFIGFILE != "NULL"} { - # - # copy over Columnstore.xml file - # - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/$INSTALLLOCATION/etc/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send_user "Copy InfiniDB Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/$INSTALLLOCATION/etc/$CONFIG.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE"} - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - #do a dummy scp command - send "scp $CONFIGFILE $USERNAME@$SERVER:/tmp/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user " " } - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved InfiniDB Configuration File " - send "ssh $USERNAME@$SERVER 'cd /usr/local/$INSTALLLOCATION/etc/;mv -f Columnstore.xml Columnstore.xml.install;cp -v Columnstore.xml.rpmsave Columnstore.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "Columnstore.xml" { send_user "DONE" } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -# -exit - diff --git a/tools/configMgt/dm_parent_installer_v3.sh b/tools/configMgt/dm_parent_installer_v3.sh deleted file mode 100755 index 8bb899705..000000000 --- a/tools/configMgt/dm_parent_installer_v3.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -set PACKAGE [lindex $argv 2] -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set USERNAME [lindex $argv 5] -set CEFLAG [lindex $argv 6] -set DEBUG [lindex $argv 7] - -set CALPONTPACKAGE calpont-$PACKAGE -set MYSQLPACKAGE calpont-mysql-$PACKAGE -set MYSQLDPACKAGE calpont-mysqld-$PACKAGE -set INSTALLDIR "/usr/local" - -#set SHARED "//cal6500/shared" -set SHARED "//calweb/shared" - -log_user $DEBUG -spawn -noecho /bin/bash -send "rm -f $CALPONTPACKAGE\n" -# -# delete and erase all old packages from Director Module -# -set timeout 10 -send "ssh $USERNAME@$SERVER 'rm -f /root/$CALPONTPACKAGE'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } abort - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } abort -} -expect { - -re "#" { } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -# -# erase calpont packages -# -set timeout 60 -send_user "Erase Old Calpont Packages " -send "ssh $USERNAME@$SERVER 'pkill -9 mysqld'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { } abort -} -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^calpont') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } abort - -re "# " { } abort - -re "not installed" { } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/Calpont/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/Calpont/bin/pre-uninstall && $INSTALLDIR/Calpont/bin/pre-uninstall'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { send_user "DONE" } abort - -re "uninstall completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# get the calpont package -# -send_user "Get Calpont Packages " -send "smbclient $SHARED -Wcalpont -Uoamuser%Calpont1 -c 'cd Iterations/$RELEASE/;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } abort -} -send_user "\n" -# -# send the calpont package -# -send_user "Copy Calpont Packages " -send "scp -q $PACKAGE $USERNAME@$SERVER:/root/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re "#" { send_user "DONE" } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -#send "rm -f $PACKAGE\n" -# -# install calpont package -# -set timeout 120 -send_user "Install New Calpont Packages " -send "ssh $USERNAME@$SERVER ' rpm -iv --nodeps /root/$CALPONTPACKAGE'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { expect { - -re "completed" { expect { - -re "completed" { send_user "DONE" } abort - } - } - } - } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -set timeout 120 -expect -re "# " -# -if { $CONFIGFILE != "NULL"} { - # - # copy over Columnstore.xml file - # - send_user "Copy Calpont Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/mariadb/columnstore/etc/Columnstore.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/mariadb/columnstore/etc/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:/tmp/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user " " } abort - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved Calpont Configuration File " - send "ssh $USERNAME@$SERVER 'cd /usr/local/mariadb/columnstore/etc/;mv -f Columnstore.xml Columnstore.xml.install;cp -v Columnstore.xml.rpmsave Columnstore.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "Columnstore.xml" { send_user "DONE" } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -# -exit - diff --git a/tools/configMgt/parent_binary_installer.sh b/tools/configMgt/parent_binary_installer.sh deleted file mode 100755 index dc87420e6..000000000 --- a/tools/configMgt/parent_binary_installer.sh +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -set PACKAGE [lindex $argv 2] -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set USERNAME [lindex $argv 5] -set INSTALLDIR [lindex $argv 6] -set PACKAGENAME [lindex $argv 7] -set INSTALLLOCATION [lindex $argv 8] -set DEBUG [lindex $argv 9] - -set CALPONTPACKAGE infinidb-ent-*$PACKAGE - -set SHARED "//srvhill01/shared" - -set INSTALLDIRARG " " -set HOME "/root" -if { $USERNAME != "root" } { - set INSTALLDIRARG "--installdir=$INSTALLDIR/$INSTALLLOCATION" - set HOME $INSTALLDIR -} - -log_user $DEBUG -spawn -noecho /bin/bash -send "rm -f $PACKAGE\n" -# -# delete binary package on Parent OAM Module -# -set timeout 30 -send_user "Remove Calpont Packages from System " -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/$INSTALLLOCATION*.gz;rm -f /root/$INSTALLLOCATION*.rpm;rm -f /root/$INSTALLLOCATION*.gz'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } abort - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } abort -} -expect { - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "DONE\n" -# -# get the calpont package -# -send_user "Get Calpont Packages " -send "smbclient $SHARED -WMARIADB -Uroot%Calpont1 -c 'cd packages/$RELEASE/packages;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } abort -} -send_user "\n" -# -# send the calpont package -# -send_user "Copy Calpont Packages " -send "scp $PACKAGE $USERNAME@$SERVER:$HOME/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -# -# uninstall calpont package -# -send_user "Erase Old Calpont Package " -send "ssh $USERNAME@$SERVER 'pkill -9 mysqld'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { } abort -} -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^calpont') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } abort - -re "# " { } abort - -re "not installed" { } abort - -re "Failed dependencies" { } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/$INSTALLLOCATION/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall && $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall $INSTALLDIRARG'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { send_user "DONE" } abort - -re "uninstall completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# install calpont package -# -set timeout 30 -send_user "Install New Calpont Package " -send "ssh $USERNAME@$SERVER 'tar -C $INSTALLDIR --exclude db -zxf $HOME/$CALPONTPACKAGE;cat $INSTALLDIR/$INSTALLLOCATION/releasenum'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { send_user "DONE" } abort - -re "release=" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "exiting now" { send_user "FAILED: Error in tar command\n" ; exit -1 } - -} -send_user "\n" -set timeout 30 -send "rm -f $PACKAGE\n" -# -send_user "Run post-install script " -send " \n" -send date\n -send "ssh $USERNAME@$SERVER '$INSTALLDIR/$INSTALLLOCATION/bin/post-install $INSTALLDIRARG'\n" -set timeout 10 -expect { - -re "word: " { send "$PASSWORD\n" } abort - -re "passphrase" { send "$PASSWORD\n" } abort -} -set timeout 60 -# check return -expect { - -re "InfiniDB syslog logging not working" { send_user "ERROR: InfiniDB System logging not setup\n" ; exit 1 } - -re "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } - -re "Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1} - -re "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } - -re "closed" { send_user "ERROR: Connection closed\n" ; exit 1 } - -re "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } - -re "postConfigure" { send_user "DONE" } abort - -re "# " { send_user "DONE" } abort -} -send_user "\n" -sleep 10 -# -if { $CONFIGFILE != "NULL"} { - # - # copy over $CONFIG.xml file - # - send_user "Copy Calpont Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:$INSTALLDIR/$INSTALLLOCATION/etc/$CONFIG.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:$INSTALLDIR/$INSTALLLOCATION/etc/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved Calpont Configuration File " - send "ssh $USERNAME@$SERVER 'cd $INSTALLDIR/$INSTALLLOCATION/etc/;mv -f $CONFIG.xml $CONFIG.xml.install;cp -v $CONFIG.xml.rpmsave $CONFIG.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "$CONFIG.xml" { send_user "DONE" } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -# -exit - diff --git a/tools/configMgt/parent_binary_installer_v3.sh b/tools/configMgt/parent_binary_installer_v3.sh deleted file mode 100755 index 155004a90..000000000 --- a/tools/configMgt/parent_binary_installer_v3.sh +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -set PACKAGE [lindex $argv 2] -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set USERNAME [lindex $argv 5] -set INSTALLDIR [lindex $argv 6] -set DEBUG [lindex $argv 7] - -set CALPONTPACKAGE calpont-infinidb-ent-*$PACKAGE - -set SHARED "//calweb/shared" - -set INSTALLDIRARG " " -set HOME "/root" -if { $USERNAME != "root" } { - set INSTALLDIRARG "--installdir=$INSTALLDIR/Calpont" - set HOME $INSTALLDIR -} - -log_user $DEBUG -spawn -noecho /bin/bash -send "rm -f $PACKAGE\n" -# -# delete binary package on Parent OAM Module -# -set timeout 30 -send_user "Remove Calpont Packages from System " -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/calpont*.gz;rm -f /root/calpont*.rpm;rm -f /root/calpont*.gz'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } abort - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } abort -} -expect { - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "DONE\n" -# -# get the calpont package -# -send_user "Get Calpont Packages " -send "smbclient $SHARED -Wcalpont -Uoamuser%Calpont1 -c 'cd Iterations/$RELEASE/packages;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } abort -} -send_user "\n" -# -# send the calpont package -# -send_user "Copy Calpont Packages " -send "scp $PACKAGE $USERNAME@$SERVER:$HOME/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -# -# uninstall calpont package -# -send_user "Erase Old Calpont Package " -send "ssh $USERNAME@$SERVER 'pkill -9 mysqld'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { } abort -} -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^calpont') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n"expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } abort - -re "# " { } abort - -re "not installed" { } abort - -re "Failed dependencies" { } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/Calpont/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/Calpont/bin/pre-uninstall && $INSTALLDIR/Calpont/bin/pre-uninstall $INSTALLDIRARG'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { send_user "DONE" } abort - -re "uninstall completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# install calpont package -# -set timeout 30 -send_user "Install New Calpont Package " -send "ssh $USERNAME@$SERVER 'tar -C $INSTALLDIR --exclude db -zxf $HOME/$CALPONTPACKAGE;cat $INSTALLDIR/Calpont/releasenum'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { send_user "DONE" } abort - -re "release=" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "exiting now" { send_user "FAILED: Error in tar command\n" ; exit -1 } - -} -send_user "\n" -set timeout 30 -send "rm -f $PACKAGE\n" -# -send_user "Run post-install script " -send " \n" -send date\n -send "ssh $USERNAME@$SERVER '$INSTALLDIR/Calpont/bin/post-install $INSTALLDIRARG'\n" -set timeout 10 -expect { - -re "word: " { send "$PASSWORD\n" } abort - -re "passphrase" { send "$PASSWORD\n" } abort -} -set timeout 60 -# check return -expect { - -re "InfiniDB syslog logging not working" { send_user "ERROR: InfiniDB System logging not setup\n" ; exit 1 } - -re "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } - -re "Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1} - -re "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } - -re "closed" { send_user "ERROR: Connection closed\n" ; exit 1 } - -re "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } - -re "postConfigure" { send_user "DONE" } abort - -re "# " { send_user "DONE" } abort -} -send_user "\n" -sleep 10 -# -if { $CONFIGFILE != "NULL"} { - # - # copy over Columnstore.xml file - # - send_user "Copy Calpont Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:$INSTALLDIR/Calpont/etc/Columnstore.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:$INSTALLDIR/Calpont/etc/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved Calpont Configuration File " - send "ssh $USERNAME@$SERVER 'cd $INSTALLDIR/Calpont/etc/;mv -f Columnstore.xml Columnstore.xml.install;cp -v Columnstore.xml.rpmsave Columnstore.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "Columnstore.xml" { send_user "DONE" } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -# -exit - diff --git a/tools/configMgt/pm_parent_installer.sh b/tools/configMgt/pm_parent_installer.sh deleted file mode 100755 index c8e135b9b..000000000 --- a/tools/configMgt/pm_parent_installer.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set USERNAME root -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -#set PACKAGE [lindex $argv 2] -set PACKAGE *.rpm -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set PREFIX [lindex $argv 5] -set USERNAME [lindex $argv 6] -set PACKAGENAME [lindex $argv 7] -set INSTALLLOCATION [lindex $argv 8] -set DEBUG [lindex $argv 9] - -set SHARED "//srvhill01/shared" -set INSTALLDIR "/usr/local" - -log_user $DEBUG -spawn -noecho /bin/bash - -set CONFIG "Calpont" -if { $PACKAGENAME != "columnstore"} { - set CONFIG "Calpont" -} - -send "rm -f $PACKAGE\n" -# -# delete and erase all old packages from Parent OAM Module -# -set timeout 30 -send "ssh $USERNAME@$SERVER 'rm -f /root/Calpont-*.rpm /root/$PACKAGENAME*.rpm /root/mariadb*.rpm '\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } -} -expect { - -re {[$#] } { } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -# -# get the InfiniDB package -# -expect -re {[$#] } -send_user "Get InfiniDB Packages " -send "smbclient $SHARED -WMARIADB -Uroot%Calpont1 -c 'cd packages/$RELEASE/;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } -} -send_user "\n" -# -# send the InfiniDB package -# -set timeout 120 -send_user "Copy InfiniDB Packages " -send "scp -q $PACKAGE $USERNAME@$SERVER:/root/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re {[$#] } { send_user "DONE" } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -#send "rm -f $PACKAGE\n" -# -# erase InfiniDB package -# -set timeout 60 -expect -re {[$#] } -send_user "Erase Old InfiniDB Package " -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^mariadb-columnstore') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } - -re {[$#] } { } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/$INSTALLLOCATION/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall && $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re {[$#] } { send_user "DONE" } - -re "uninstall completed" { send_user "DONE" } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# install InfiniDB package -# -set timeout 160 -send_user "Install New InfiniDB Packages " -send "ssh $USERNAME@$SERVER ' rpm -ivh --nodeps --force $PACKAGENAME-*'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { send_user "DONE" } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "File not found" { send_user "FAILED: File not found\n" ; exit -1 } -} -send_user "\n" -set timeout 120 -expect -re {[$#] } -send "rm -f $PACKAGE\n" -# -if { $CONFIGFILE != "NULL"} { - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/$INSTALLLOCATION/etc/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - # - # copy over $CONFIG.xml file - # - send_user "Copy InfiniDB Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/$INSTALLLOCATION/etc/$CONFIG.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - #do a dummy scp command - send "scp $CONFIGFILE $USERNAME@$SERVER:/tmp/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user " " } - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved InfiniDB Configuration File " - send "ssh $USERNAME@$SERVER 'cd /usr/local/$INSTALLLOCATION/etc/;mv -f $CONFIG.xml $CONFIG.xml.install;cp -v $CONFIG.xml.rpmsave $CONFIG.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "$CONFIG.xml" { send_user "DONE" } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -expect -re {[$#] } -# -exit - diff --git a/tools/configMgt/pm_parent_installer_v3.sh b/tools/configMgt/pm_parent_installer_v3.sh deleted file mode 100755 index 88931ba38..000000000 --- a/tools/configMgt/pm_parent_installer_v3.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set USERNAME root -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -set PACKAGE [lindex $argv 2] -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set PREFIX [lindex $argv 5] -set USERNAME [lindex $argv 6] -set DEBUG [lindex $argv 7] - -set CALPONTPACKAGE calpont-$PREFIX$PACKAGE -set MYSQLPACKAGE calpont-mysql-$PACKAGE -set MYSQLDPACKAGE calpont-mysqld-$PACKAGE -#set SHARED "//cal6500/shared" -set SHARED "//calweb/shared" -set INSTALLDIR "/usr/local" - -log_user $DEBUG -spawn -noecho /bin/bash -send "rm -f $PACKAGE\n" -# -# delete and erase all old packages from Parent OAM Module -# -set timeout 30 -send "ssh $USERNAME@$SERVER 'rm -f /root/calpont*.rpm /root/infinidb*.rpm'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } abort - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } abort -} -expect { - -re "#" { } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -# -# get the calpont package -# -expect -re "# " -send_user "Get Calpont Packages " -send "smbclient $SHARED -Wcalpont -Uoamuser%Calpont1 -c 'cd Iterations/$RELEASE/;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } abort -} -send_user "\n" -# -# send the calpont package -# -send_user "Copy Calpont Packages " -send "scp -q $PACKAGE $USERNAME@$SERVER:/root/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re "#" { send_user "DONE" } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -#send "rm -f $PACKAGE\n" -# -# erase calpont package -# -expect -re "# " -send_user "Erase Old Calpont Package " -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^calpont') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } abort - -re "# " { } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/Calpont/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/Calpont/bin/pre-uninstall && $INSTALLDIR/Calpont/bin/pre-uninstall'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { send_user "DONE" } abort - -re "uninstall completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# install calpont package -# -set timeout 160 -send_user "Install New Calpont Package " -send "ssh $USERNAME@$SERVER ' rpm -ivh --nodeps /root/$CALPONTPACKAGE'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -set timeout 120 -expect -re "# " -send "rm -f $PACKAGE\n" -# -if { $CONFIGFILE != "NULL"} { - # - # copy over Columnstore.xml file - # - send_user "Copy Calpont Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/mariadb/columnstore/etc/Columnstore.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/mariadb/columnstore/etc/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:/tmp/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user " " } abort - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved Calpont Configuration File " - send "ssh $USERNAME@$SERVER 'cd /usr/local/mariadb/columnstore/etc/;mv -f Columnstore.xml Columnstore.xml.install;cp -v Columnstore.xml.rpmsave Columnstore.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "Columnstore.xml" { send_user "DONE" } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -expect -re "# " -# -exit - diff --git a/tools/configMgt/stackReleaseChecker.cpp b/tools/configMgt/stackReleaseChecker.cpp deleted file mode 100644 index 404aff92a..000000000 --- a/tools/configMgt/stackReleaseChecker.cpp +++ /dev/null @@ -1,177 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporaton - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/****************************************************************************************** -* $Id: stackReleaseChecker.cpp 64 2006-10-12 22:21:51Z dhill $ -* -* -* List of files being updated by configure: -* Calpont/etc/Columnstore.xml -* -* -******************************************************************************************/ -/** - * @file - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "stdio.h" -#include "ctype.h" -#include - -#include -#include - -#include "liboamcpp.h" -#include "configcpp.h" - -using namespace std; -using namespace oam; -using namespace config; - -int main(int argc, char *argv[]) -{ - Oam oam; - string systemUser = "root"; - string installParentModuleHostName; - string password; - string debug_flag = "1"; - string systemName = "all"; - bool VERBOSE = false; - - Config* sysConfig = Config::makeConfig("./systems/CalpontSystems.xml"); - - for( int i = 1; i < argc; i++ ) - { - if( string("-h") == argv[i] ) { - cout << endl; - cout << "'stackReleaseChecker' retrieves and prints the release installed on a stack." << endl; - cout << endl; - cout << "The list of systems are located in './systems/CalpontSystems.xml" << endl; - cout << endl; - cout << "Usage: stackReleaseChecker [-h][-s system][-v]" << endl; - cout << " -s system-name or leave blank for all systems" << endl; - cout << " -v verbose will give the version numbers" << endl; - exit (0); - } - else if( string("-s") == argv[i] ) { - i++; - if ( argc == i ) { - cout << "ERROR: missing system argument" << endl; - exit(-1); - } - systemName = argv[i]; - } - else if( string("-v") == argv[i] ) - VERBOSE = true; - } - - int systemCount; - try { - systemCount = strtol(sysConfig->getConfig("NetworkConfig", "SystemCount").c_str(), 0, 0); - if ( systemCount == 0 ) { - cout << "ERROR: SystemCount in ./systems/CalpontSystems.xml equal to 0" << endl; - exit(-1); - } - } - catch(...) - { - cout << "ERROR: Problem getting SystemCount from ./systems/CalpontSystems.xml" << endl; - exit(-1); - } - - bool FOUND = false; - for ( int i = 1 ; i < systemCount+1 ; i++) - { - string SystemName = "SystemName" + oam.itoa(i); - string oamParentModule = "OAMParentModule" + oam.itoa(i); - string SystemPassword = "SystemPassword" + oam.itoa(i); - string SystemUser = "SystemUser" + oam.itoa(i); - - string tempSystem; - try { - tempSystem = sysConfig->getConfig("NetworkConfig", SystemName ); - } - catch(...) - { - cout << "ERROR: Problem getting SystemName from ./systems/CalpontSystems.xml" << endl; - exit(-1); - } - - if ( tempSystem == systemName || systemName == "all") { - try { - installParentModuleHostName = sysConfig->getConfig("NetworkConfig", oamParentModule ); - password = sysConfig->getConfig("NetworkConfig", SystemPassword ); - systemUser = sysConfig->getConfig("NetworkConfig", SystemUser ); - FOUND = true; - } - catch(...) - { - cout << "ERROR: Problem getting SystemName from ./systems/CalpontSystems.xml" << endl; - exit(-1); - } - - if ( tempSystem == "unassigned") - continue; - - //get if root or no-root user install - string installDir = "/usr/local"; - if ( systemUser != "root" ) - installDir = "/home/" + systemUser; - - installDir = installDir + "/Calpont"; - - string cmd; - if (!VERBOSE) - cmd = "./remote_command.sh " + installParentModuleHostName + " " + systemUser + " " + password + " 'cat " + installDir + "/releasenum' 'release=' cat: 10 " + debug_flag; - else - cmd = "./remote_command.sh " + installParentModuleHostName + " " + systemUser + " " + password + " '" + installDir + "/bin/mcsadmin getcalpontsoftware' 'tools' Error 10 " + debug_flag; - - cout << "***** Calpont InfiniDB Package Release Information for stack '" << tempSystem << "' *****" << endl << endl; - - system(cmd.c_str()); - - cout << endl; - - cout << "***** Status of stack '" << tempSystem << "' *****" << endl << endl; - - cmd = "./remote_command.sh " + installParentModuleHostName + " " + systemUser + " " + password + " '" + installDir + "/bin/mcsadmin getsystems' dummy Error 10 " + debug_flag; - - system(cmd.c_str()); - - cout << endl << endl; - } - } - - if ( !FOUND ) { - cout << "ERROR: System Name '" + systemName + "' not in ./systems/CalpontSystems.xml" << endl; - exit(-1); - } -} diff --git a/utils/configcpp/libconfigcpp.vcxproj b/utils/configcpp/libconfigcpp.vcxproj index 4856a76cd..4f7510e5f 100644 --- a/utils/configcpp/libconfigcpp.vcxproj +++ b/utils/configcpp/libconfigcpp.vcxproj @@ -272,7 +272,7 @@ - + @@ -300,4 +300,4 @@ - \ No newline at end of file + diff --git a/utils/configcpp/libconfigcpp.vcxproj.filters b/utils/configcpp/libconfigcpp.vcxproj.filters index 33ce5c88d..706be8841 100644 --- a/utils/configcpp/libconfigcpp.vcxproj.filters +++ b/utils/configcpp/libconfigcpp.vcxproj.filters @@ -41,7 +41,7 @@ Header Files - + Header Files @@ -56,4 +56,4 @@ Resource Files - \ No newline at end of file + From 44c06f2cc3002d0d2ecf39bc00add6c4dff72a00 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 12 Dec 2017 16:32:49 -0600 Subject: [PATCH 03/41] MCOL-1106 - increased mysqld startup timeouts --- dbcon/mysql/mysql-Columnstore | 7 +++- oam/install_scripts/binary_installer.sh | 54 +++---------------------- oam/install_scripts/user_installer.sh | 2 +- 3 files changed, 11 insertions(+), 52 deletions(-) diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore index de090329f..02a0de162 100755 --- a/dbcon/mysql/mysql-Columnstore +++ b/dbcon/mysql/mysql-Columnstore @@ -61,7 +61,7 @@ datadir=$basedir/db # Value here is overriden by value in my.cnf. # 0 means don't wait at all # Negative numbers mean to wait indefinitely -service_startup_timeout=30 +service_startup_timeout=90 # Lock directory for RedHat / SuSE. lockdir='/var/lock/subsys' @@ -262,6 +262,8 @@ wait_for_gone () { done log_failure_msg + kill_by_pid + return 1 } @@ -310,7 +312,7 @@ fi kill_by_pid() { # let's see if we can kill the 2 mysql procs by hand # get the our mysql from ps - eval $(ps -ef | grep "$COLUMNSTORE_INSTALL_DIR/mysql//bin/mysqld " | grep -v grep | head -1 | awk '{printf "pid=%d\n", $2}') + eval $(ps -ef | grep "$COLUMNSTORE_INSTALL_DIR/mysql/bin/mysqld" | grep -v grep | head -1 | awk '{printf "pid=%d\n", $2}') if [ -n "$pid" ]; then ppid=$(ps -o ppid= -p $pid) @@ -369,6 +371,7 @@ case "$mode" in wait_for_gone $mysqld_pid "$mysqld_pid_file_path"; return_value=$? else log_failure_msg "MySQL server process #$mysqld_pid is not running!" + kill_by_pid rm "$mysqld_pid_file_path" fi diff --git a/oam/install_scripts/binary_installer.sh b/oam/install_scripts/binary_installer.sh index ead0dde37..280b10c22 100644 --- a/oam/install_scripts/binary_installer.sh +++ b/oam/install_scripts/binary_installer.sh @@ -238,11 +238,15 @@ send_user "\n" # # Start module installer to setup Customer OS files # +if { $SERVERTYPE == "2" || $SERVERTYPE == "pmwithum" } { + set MODULETYPE "um" +} + send_user "Run Module Installer " send " \n" send date\n send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=$MODULETYPE --port=$MYSQLPORT --installdir=$INSTALLDIR'\n" -set timeout 60 +set timeout 120 expect { "word: " { send "$PASSWORD\n" exp_continue @@ -256,54 +260,6 @@ expect { } send_user "\n" -if { $MODULETYPE == "um" || $SERVERTYPE == "2" || $SERVERTYPE == "pmwithum" } { - # - # run mysql setup scripts - # - send_user "Run MySQL Setup Scripts on Module " - send " \n" - send date\n - send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/post-mysqld-install --installdir=$INSTALLDIR'\n" - set timeout 60 - expect { - "word: " { send "$PASSWORD\n" - exp_continue - } - "passphrase" { send "$PASSWORD\n" - exp_continue - } - "Exit status 0" { send_user "DONE" } - "Exit status 1" { send_user "ERROR: scp failed" ; exit 1 } - "ERROR" { send_user "ERROR: Daemon failed to run"; - exit 1 } - "FAILED" { send_user "ERROR: Daemon failed to run"; - exit 1 } - timeout { send_user "ERROR: Timeout\n" ; exit 2 } - } - - send " \n" - send date\n - send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/post-mysql-install --installdir=$INSTALLDIR'\n" - set timeout 120 - expect { - "word: " { send "$PASSWORD\n" - exp_continue - } - "passphrase" { send "$PASSWORD\n" - exp_continue - } - "Exit status 0" { send_user "DONE" } - "Exit status 1" { send_user "ERROR: scp failed" ; exit 1 } - "ERROR" { send_user "ERROR: Daemon failed to run"; - exit 1 } - "FAILED" { send_user "ERROR: Daemon failed to run"; - exit 1 } - timeout { send_user "ERROR: Timeout\n" ; exit 2 } - } - send_user "\n" -} - - send_user "\nInstallation Successfully Completed on '$MODULE'\n" exit 0 diff --git a/oam/install_scripts/user_installer.sh b/oam/install_scripts/user_installer.sh index a31367aed..4d30492d8 100644 --- a/oam/install_scripts/user_installer.sh +++ b/oam/install_scripts/user_installer.sh @@ -255,7 +255,7 @@ send_user "\n" # send_user "Run Module Installer " send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=um --port=$MYSQLPORT'\n" -set timeout 60 +set timeout 120 expect { "word: " { send "$PASSWORD\n" exp_continue From de24edcf8f1b7499525be9d896133f04b6a80bf3 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 14 Dec 2017 14:47:47 -0600 Subject: [PATCH 04/41] Update README --- README | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README b/README index 291da0ab7..8186d1ed7 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.11 -MariaDB ColumnStore 1.0.11 is the development version of MariaDB ColumnStore. +This is MariaDB ColumnStore 1.0.12 +MariaDB ColumnStore 1.0.12 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.11 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.12 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 9f5bd31e37dd330ae01b5da04123985c6450e3f3 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 14 Dec 2017 14:49:58 -0600 Subject: [PATCH 05/41] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 397a1b1d8..40c1e99ec 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.11 -MariaDB ColumnStore 1.0.11 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.12 +MariaDB ColumnStore 1.0.12 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.11 is an GA release. +#MariaDB ColumnStore 1.0.12 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. From e9aff27e59f8a9dc7fec58c35e3968aa2018ec46 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 15 Dec 2017 20:33:07 +0000 Subject: [PATCH 06/41] MCOL-1116 Fix I_S.CS_FILES for missing dbroot If a dbroot is missing/offline mysqld would crash on information_schema.columnstore_files due to not catching an exception. This patch now catches the exception. --- dbcon/mysql/is_columnstore_files.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index e9479e254..ce00b8aae 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -124,6 +124,17 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) iter++; continue; } + + try + { + oam_instance.getDbrootPmConfig(iter->dbRoot, pmId); + } + catch (std::runtime_error) + { + // MCOL-1116: If we are here a DBRoot is offline/missing + iter++; + continue; + } table->field[0]->store(oid); table->field[1]->store(iter->segmentNum); table->field[2]->store(iter->partitionNum); @@ -134,7 +145,7 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) std::string DbRootPath = config->getConfig("SystemConfig", DbRootName.str()); fileSize = compressedFileSize = 0; snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", DbRootPath.c_str(), oidDirName); - oam_instance.getDbrootPmConfig(iter->dbRoot, pmId); + std::ostringstream oss; oss << "pm" << pmId << "_WriteEngineServer"; std::string client = oss.str(); From cb2e929c13c21263ae4bfcebd80fc400d91af688 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 19 Dec 2017 15:01:14 -0600 Subject: [PATCH 07/41] Add new generated header file to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index ecb4702e8..f8246dc9f 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,5 @@ install_manifest_libs.txt install_manifest_platform.txt install_manifest_storage-engine.txt _CPack_Packages +columnstoreversion.h + From a2cda71dbee3d8ba9816ee619695c4d3f8e175d3 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 19 Dec 2017 15:02:04 -0600 Subject: [PATCH 08/41] remove releasenum from rpm packages --- cpackEngineRPM.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index a6013b280..dcc65e032 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -189,7 +189,6 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/etc/MessageFile.txt" "/usr/local/mariadb/columnstore/etc/ErrorMessage.txt" "/usr/local/mariadb/columnstore/local/module" -"/usr/local/mariadb/columnstore/releasenum" "/usr/local/mariadb/columnstore/bin/rollback" "/usr/local/mariadb/columnstore/bin/editem" "/usr/local/mariadb/columnstore/bin/getConfig" From 8e81135108392236d1429927d2da468d492bc937 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 19 Dec 2017 15:03:48 -0600 Subject: [PATCH 09/41] This is being done twice, removing one of them. --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e46e2ad3..482e8d19e 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,8 +37,6 @@ ENDIF(NOT CMAKE_BUILD_TYPE) #set( CMAKE_VERBOSE_MAKEFILE on ) INCLUDE(columnstore_version.cmake) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) - SET (PACKAGE columnstore) SET (PACKAGE_NAME columnstore) From c31c836352ee921141e8bbdc5264700b59ae97a9 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 3 Jan 2018 09:20:09 +0000 Subject: [PATCH 10/41] MCOL-1085 Add crash dump to daemons This patch adds an automated crash dump which logs in /var/log/mariadb/columnstore/trace/ when one of the ColumnStore daemons crashes. --- CMakeLists.txt | 8 ++--- ddlproc/CMakeLists.txt | 2 +- ddlproc/ddlproc.cpp | 9 +++++ dmlproc/CMakeLists.txt | 3 +- dmlproc/dmlproc.cpp | 11 ++++++ exemgr/CMakeLists.txt | 2 +- exemgr/main.cpp | 14 +++++++- oamapps/serverMonitor/CMakeLists.txt | 3 +- oamapps/serverMonitor/main.cpp | 10 ++++++ primitives/primproc/CMakeLists.txt | 3 +- primitives/primproc/primproc.cpp | 11 ++++++ procmgr/CMakeLists.txt | 2 +- procmgr/main.cpp | 12 +++++++ procmon/CMakeLists.txt | 2 +- procmon/main.cpp | 10 ++++++ utils/common/crashtrace.cpp | 51 ++++++++++++++++++++++++++++ utils/common/crashtrace.h | 18 ++++++++++ versioning/BRM/CMakeLists.txt | 4 +-- versioning/BRM/masternode.cpp | 9 +++++ versioning/BRM/slavenode.cpp | 9 +++++ writeengine/server/CMakeLists.txt | 3 +- writeengine/server/we_server.cpp | 11 ++++++ 22 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 utils/common/crashtrace.cpp create mode 100644 utils/common/crashtrace.h diff --git a/CMakeLists.txt b/CMakeLists.txt index df20c3e1b..b40c663a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,12 +98,12 @@ endif() FOREACH(BUILD_TYPE RELEASE RELWITHDEBINFO MINSIZEREL) - SET(CMAKE_CXX_FLAGS_${BUILD_TYPE} "-g -O3 -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H") - SET(CMAKE_C_FLAGS_${BUILD_TYPE} "-g -O3 -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H") + SET(CMAKE_CXX_FLAGS_${BUILD_TYPE} "-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H") + SET(CMAKE_C_FLAGS_${BUILD_TYPE} "-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H") ENDFOREACH() -SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H") -SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H") +SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H") +SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H") SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") diff --git a/ddlproc/CMakeLists.txt b/ddlproc/CMakeLists.txt index a56fb4683..7c7fdf70c 100644 --- a/ddlproc/CMakeLists.txt +++ b/ddlproc/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp) +set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp) add_executable(DDLProc ${DDLProc_SRCS}) diff --git a/ddlproc/ddlproc.cpp b/ddlproc/ddlproc.cpp index 45cf09327..2589f9c29 100644 --- a/ddlproc/ddlproc.cpp +++ b/ddlproc/ddlproc.cpp @@ -61,6 +61,7 @@ using namespace execplan; #include "IDBPolicy.h" #include "utils_utf8.h" +#include "crashtrace.h" namespace fs = boost::filesystem; @@ -97,6 +98,9 @@ int main(int argc, char* argv[]) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("DDLProc"); + setupCwd(); WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_DDLPROC ); @@ -116,6 +120,11 @@ int main(int argc, char* argv[]) sigaction(SIGHUP, &ign, 0); ign.sa_handler = SIG_IGN; sigaction(SIGPIPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif ddlprocessor::DDLProcessor ddlprocessor(1, 20); diff --git a/dmlproc/CMakeLists.txt b/dmlproc/CMakeLists.txt index 8ed489aa8..957cfdff6 100644 --- a/dmlproc/CMakeLists.txt +++ b/dmlproc/CMakeLists.txt @@ -8,7 +8,8 @@ set(DMLProc_SRCS dmlproc.cpp dmlprocessor.cpp dmlresultbuffer.cpp - batchinsertprocessor.cpp) + batchinsertprocessor.cpp + ../utils/common/crashtrace.cpp) add_executable(DMLProc ${DMLProc_SRCS}) diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index fe542e3a4..350cc22e3 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -82,6 +82,8 @@ using namespace joblist; #include "utils_utf8.h" +#include "crashtrace.h" + namespace fs = boost::filesystem; namespace @@ -473,6 +475,9 @@ int main(int argc, char* argv[]) //BUG 5362 systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("DMLProc"); + Config* cf = Config::makeConfig(); setupCwd(); @@ -578,6 +583,12 @@ int main(int argc, char* argv[]) sigaction(SIGHUP, &ign, 0); ign.sa_handler = SIG_IGN; sigaction(SIGPIPE, &ign, 0); + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif dmlserver.start(); diff --git a/exemgr/CMakeLists.txt b/exemgr/CMakeLists.txt index 5a9f197d7..c7276ad05 100644 --- a/exemgr/CMakeLists.txt +++ b/exemgr/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp) +set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) add_executable(ExeMgr ${ExeMgr_SRCS}) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 002ba6aab..ca837a33e 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -97,6 +97,8 @@ using namespace querytele; #include "utils_utf8.h" #include "boost/filesystem.hpp" +#include "crashtrace.h" + namespace { //If any flags other than the table mode flags are set, produce output to screeen @@ -1165,10 +1167,12 @@ public: } }; +#ifdef _MSC_VER void exit_(int) { exit(0); } +#endif void added_a_pm(int) { @@ -1213,7 +1217,6 @@ void printTotalUmMemory(int sig) void setupSignalHandlers() { #ifdef _MSC_VER - signal(SIGSEGV, exit_); signal(SIGINT, exit_); signal(SIGTERM, exit_); #else @@ -1229,6 +1232,12 @@ void setupSignalHandlers() sigaction(SIGHUP, &ign, 0); ign.sa_handler = printTotalUmMemory; sigaction(SIGUSR1, &ign, 0); + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif } @@ -1302,6 +1311,9 @@ int main(int argc, char* argv[]) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("ExeMgr"); + gDebug = 0; bool eFlg = false; int c; diff --git a/oamapps/serverMonitor/CMakeLists.txt b/oamapps/serverMonitor/CMakeLists.txt index 04296114a..81d14c9ea 100644 --- a/oamapps/serverMonitor/CMakeLists.txt +++ b/oamapps/serverMonitor/CMakeLists.txt @@ -13,7 +13,8 @@ set(ServerMonitor_SRCS procmonMonitor.cpp msgProcessor.cpp dbhealthMonitor.cpp - UMAutoSync.cpp) + UMAutoSync.cpp + ../../utils/common/crashtrace.cpp) add_executable(ServerMonitor ${ServerMonitor_SRCS}) diff --git a/oamapps/serverMonitor/main.cpp b/oamapps/serverMonitor/main.cpp index 10903d6a4..56b8e3825 100644 --- a/oamapps/serverMonitor/main.cpp +++ b/oamapps/serverMonitor/main.cpp @@ -18,6 +18,8 @@ #include "IDBPolicy.h" #include "serverMonitor.h" +#include "crashtrace.h" + using namespace std; using namespace servermonitor; using namespace oam; @@ -38,6 +40,14 @@ int main (int argc, char** argv) ServerMonitor serverMonitor; Oam oam; + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + //Launch Memory Monitor Thread and check if swap is in critical condition pthread_t memoryMonitorThread; pthread_create (&memoryMonitorThread, NULL, (void*(*)(void*)) &memoryMonitor, NULL); diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index ddfa56abc..cb5a7c163 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -18,7 +18,8 @@ set(PrimProc_SRCS primitiveserver.cpp pseudocc.cpp rtscommand.cpp - umsocketselector.cpp) + umsocketselector.cpp + ../../utils/common/crashtrace.cpp) #PrimProc_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS) diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 7fc0c36f2..200786e46 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -71,6 +71,8 @@ using namespace idbdatafile; #include "cgroupconfigurator.h" +#include "crashtrace.h" + namespace primitiveprocessor { @@ -126,6 +128,12 @@ void setupSignalHandlers() ign.sa_handler = SIG_IGN; sigaction(SIGUSR2, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + sigset_t sigset; sigemptyset(&sigset); sigaddset(&sigset, SIGPIPE); @@ -288,6 +296,9 @@ int main(int argc, char* argv[]) systemLang.find("UTF") != string::npos ) utf8 = true; + // This is unset due to the way we start it + program_invocation_short_name = const_cast("PrimProc"); + Config* cf = Config::makeConfig(); setupSignalHandlers(); diff --git a/procmgr/CMakeLists.txt b/procmgr/CMakeLists.txt index 7ced90a35..aa2fa15fc 100644 --- a/procmgr/CMakeLists.txt +++ b/procmgr/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ProcMgr_SRCS main.cpp processmanager.cpp) +set(ProcMgr_SRCS main.cpp processmanager.cpp ../utils/common/crashtrace.cpp) add_executable(ProcMgr ${ProcMgr_SRCS}) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 3c1a870fc..16c3ad5b0 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -32,6 +32,8 @@ #include "utils_utf8.h" +#include "crashtrace.h" + using namespace std; using namespace logging; using namespace messageqcpp; @@ -100,6 +102,16 @@ int main(int argc, char **argv) setlocale(LC_ALL, systemLang.c_str()); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("ProcMgr"); + + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + Oam oam; //check if root-user diff --git a/procmon/CMakeLists.txt b/procmon/CMakeLists.txt index cb6ecbb49..cb61e8044 100644 --- a/procmon/CMakeLists.txt +++ b/procmon/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ProcMon_SRCS main.cpp processmonitor.cpp) +set(ProcMon_SRCS main.cpp processmonitor.cpp ../utils/common/crashtrace.cpp) add_executable(ProcMon ${ProcMon_SRCS}) diff --git a/procmon/main.cpp b/procmon/main.cpp index 88deeeb77..25dc07d6a 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -26,6 +26,8 @@ namespace bi=boost::interprocess; #include "IDBPolicy.h" +#include "crashtrace.h" + using namespace std; using namespace messageqcpp; using namespace processmonitor; @@ -75,6 +77,14 @@ int main(int argc, char **argv) setuid(0); // set effective ID to root; ignore return status #endif + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + if (argc > 1 && string(argv[1]) == "--daemon") { if (fork() != 0) return 0; diff --git a/utils/common/crashtrace.cpp b/utils/common/crashtrace.cpp new file mode 100644 index 000000000..41626361e --- /dev/null +++ b/utils/common/crashtrace.cpp @@ -0,0 +1,51 @@ +/* Copyright (C) 2018 MariaDB Corporaton + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#include +#include +#include +#include +#include + +void fatalHandler(int sig) +{ + char filename[128]; + void* addrs[128]; + snprintf(filename, 128, "/var/log/mariadb/columnstore/trace/%s.%d.log", program_invocation_short_name, getpid()); + FILE* logfile = fopen(filename, "w"); + char s[30]; + struct tm tim; + time_t now; + now = time(NULL); + tim = *(localtime(&now)); + strftime(s,30,"%F %T",&tim); + fprintf(logfile, "Date/time: %s\n", s); + fprintf(logfile, "Signal: %d\n\n", sig); + fflush(logfile); + int fd = fileno(logfile); + int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0])); + backtrace_symbols_fd(addrs, count, fd); + fclose(logfile); + struct sigaction sigact; + memset(&sigact, 0, sizeof(sigact)); + sigact.sa_handler = SIG_DFL; + sigaction(sig, &sigact, NULL); + raise(sig); +} diff --git a/utils/common/crashtrace.h b/utils/common/crashtrace.h new file mode 100644 index 000000000..3b9cb4036 --- /dev/null +++ b/utils/common/crashtrace.h @@ -0,0 +1,18 @@ +/* Copyright (C) 2018 MariaDB Corporaton + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +void fatalHandler(int sig); diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 2ce60254d..2c904e872 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -41,7 +41,7 @@ install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) ########### next target ############### -set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp) +set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp ../../utils/common/crashtrace.cpp) add_executable(controllernode ${controllernode_SRCS}) @@ -52,7 +52,7 @@ install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### -set(workernode_SRCS slavenode.cpp) +set(workernode_SRCS slavenode.cpp ../../utils/common/crashtrace.cpp) add_executable(workernode ${workernode_SRCS}) diff --git a/versioning/BRM/masternode.cpp b/versioning/BRM/masternode.cpp index 3932f9a37..4bdf42553 100644 --- a/versioning/BRM/masternode.cpp +++ b/versioning/BRM/masternode.cpp @@ -35,6 +35,8 @@ #include "brmtypes.h" #include "utils_utf8.h" +#include "crashtrace.h" + #define MAX_RETRIES 10 BRM::MasterDBRMNode *m; @@ -128,6 +130,13 @@ int main(int argc, char **argv) signal(SIGUSR1, restart); signal(SIGPIPE, SIG_IGN); #endif + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); idbdatafile::IDBPolicy::configIDBPolicy(); diff --git a/versioning/BRM/slavenode.cpp b/versioning/BRM/slavenode.cpp index 2d36bc3a4..5ec20f69f 100644 --- a/versioning/BRM/slavenode.cpp +++ b/versioning/BRM/slavenode.cpp @@ -37,6 +37,8 @@ #include "utils_utf8.h" #include "IDBPolicy.h" +#include "crashtrace.h" + using namespace BRM; using namespace std; @@ -117,6 +119,13 @@ int main(int argc, char **argv) signal(SIGPIPE, SIG_IGN); #endif + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + if (!(argc >= 3 && (arg = argv[2]) == "fg")) err = fork(); diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index 365e98036..5029cdb15 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -14,7 +14,8 @@ set(WriteEngineServer_SRCS we_dmlcommandproc.cpp we_cleartablelockcmd.cpp we_cpifeederthread.cpp - we_getfilesizes.cpp) + we_getfilesizes.cpp + ../../utils/common/crashtrace.cpp) add_executable(WriteEngineServer ${WriteEngineServer_SRCS}) diff --git a/writeengine/server/we_server.cpp b/writeengine/server/we_server.cpp index b5365fe0f..1e4415eac 100644 --- a/writeengine/server/we_server.cpp +++ b/writeengine/server/we_server.cpp @@ -51,6 +51,8 @@ using namespace oam; #include "utils_utf8.h" #include "dbrm.h" +#include "crashtrace.h" + namespace { void added_a_pm(int) @@ -96,6 +98,9 @@ int main(int argc, char** argv) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("WriteEngineServ"); + printf ("Locale is : %s\n", systemLang.c_str() ); //set BUSY_INIT state @@ -119,6 +124,12 @@ int main(int argc, char** argv) sigaction(SIGHUP, &sa, 0); sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, 0); + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = fatalHandler; + sigaction(SIGSEGV, &sa, 0); + sigaction(SIGABRT, &sa, 0); + sigaction(SIGFPE, &sa, 0); #endif // Init WriteEngine Wrapper (including Config Columnstore.xml cache) From 44989fbd74900ef045c07bd40b25d9a08a7a322f Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 10 Jan 2018 11:41:44 -0600 Subject: [PATCH 11/41] MCOL-1149 - add a main resume flag --- procmon/main.cpp | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/procmon/main.cpp b/procmon/main.cpp index 88deeeb77..f6a0d10dc 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -26,6 +26,8 @@ namespace bi=boost::interprocess; #include "IDBPolicy.h" +#include "crashtrace.h" + using namespace std; using namespace messageqcpp; using namespace processmonitor; @@ -55,9 +57,11 @@ void updateShareMemory(processStatusList* aPtr); bool runStandby = false; bool processInitComplete = false; bool rootUser = true; +bool mainResumeFlag; string USER = "root"; string PMwithUM = "n"; + //extern std::string gOAMParentModuleName; extern bool gOAMParentModuleFlag; @@ -75,6 +79,14 @@ int main(int argc, char **argv) setuid(0); // set effective ID to root; ignore return status #endif + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + if (argc > 1 && string(argv[1]) == "--daemon") { if (fork() != 0) return 0; @@ -435,13 +447,24 @@ int main(int argc, char **argv) unlink ("/var/log/mariadb/columnstore/activeAlarms"); } + //Clear mainResumeFlag + + mainResumeFlag = false; + //launch Status table control thread on 'pm' modules pthread_t statusThread; int ret = pthread_create (&statusThread, NULL, (void*(*)(void*)) &statusControlThread, NULL); if ( ret != 0 ) log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); - sleep(6); // give the Status thread time to fully initialize + //wait for flag to be set + + while(!mainResumeFlag) + { + log.writeLog(__LINE__, "WATING FOR mainResumeFlag to be set", LOG_TYPE_DEBUG); + + sleep(1); + } } SystemStatus systemstatus; @@ -723,6 +746,8 @@ int main(int argc, char **argv) } } + log.writeLog(__LINE__, "SYSTEM STATUS = " + oam.itoa(systemstatus.SystemOpState), LOG_TYPE_DEBUG); + if ( systemstatus.SystemOpState != MAN_OFFLINE && !DISABLED) { // Loop through the process list to check the process current state @@ -2026,6 +2051,10 @@ static void statusControlThread() log.writeLog(__LINE__, "Dbroot Status shared Memory allociated and Initialized", LOG_TYPE_DEBUG); } + //Set mainResumeFlag, to start up main thread + + mainResumeFlag = true; + string portName = "ProcStatusControl"; if (runStandby) { portName = "ProcStatusControlStandby"; From 1ffeda44d673075936f19123dfc41a626f1e13d5 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 10 Jan 2018 12:17:57 -0600 Subject: [PATCH 12/41] MCOL-962 Add mcsSystemReady(), mcsSystemReadOnly() and mcsWritesSuspended() --- dbcon/mysql/ha_calpont_impl.cpp | 124 ++++++++++++++++++++++++ dbcon/mysql/install_calpont_mysql.sh | 3 + oamapps/mcsadmin/mcsadmin.cpp | 9 +- oamapps/postConfigure/installer.cpp | 2 +- procmon/main.cpp | 2 +- utils/winport/win_setup_mysql_part2.sql | 3 + 6 files changed, 136 insertions(+), 7 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index d4669c8d5..3b3a5f00b 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1819,6 +1819,130 @@ void calsettrace_deinit(UDF_INIT* initid) { } +#ifdef _MSC_VER +__declspec(dllexport) +#endif +// Return 1 if system is ready for reads or 0 if not. +long long mcssystemready(UDF_INIT* initid, UDF_ARGS* args, + char* is_null, char* error) +{ + long long rtn = 0; + Oam oam; + DBRM dbrm(true); + SystemStatus systemstatus; + + try + { + oam.getSystemStatus(systemstatus); + if (systemstatus.SystemOpState == ACTIVE + && dbrm.getSystemReady() + && dbrm.getSystemQueryReady()) + { + return 1; + } + } + catch (...) + { + *error = 1; + } + return rtn; +} + +#ifdef _MSC_VER +__declspec(dllexport) +#endif +my_bool mcssystemready_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + return 0; +} + +#ifdef _MSC_VER +__declspec(dllexport) +#endif +void mcssystemready_deinit(UDF_INIT* initid) +{ +} + +#ifdef _MSC_VER +__declspec(dllexport) +#endif +// Return 1 if system is read only; 0 if writeable +long long mcssystemreadonly(UDF_INIT* initid, UDF_ARGS* args, + char* is_null, char* error) +{ + long long rtn = 0; + DBRM dbrm(true); + + try + { + if (dbrm.isReadWrite()) // Returns 0 for writable, 5 for read only + { + rtn = 1; + } + } + catch (...) + { + *error = 1; + rtn = 1; + } + return rtn; +} + +#ifdef _MSC_VER +__declspec(dllexport) +#endif +my_bool mcssystemreadonly_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + return 0; +} + +#ifdef _MSC_VER +__declspec(dllexport) +#endif +void mcssystemreadonly_deinit(UDF_INIT* initid) +{ +} + +#ifdef _MSC_VER +__declspec(dllexport) +#endif +// Return 1 if system is read only; 0 if writeable +long long mcswritessuspended(UDF_INIT* initid, UDF_ARGS* args, + char* is_null, char* error) +{ + long long rtn = 0; + DBRM dbrm(true); + + try + { + if (dbrm.getSystemSuspended()) + { + rtn = 1; + } + } + catch (...) + { + *error = 1; + rtn = 1; + } + return rtn; +} + +#ifdef _MSC_VER +__declspec(dllexport) +#endif +my_bool mcswritessuspended_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + return 0; +} + +#ifdef _MSC_VER +__declspec(dllexport) +#endif +void mcswritessuspended_deinit(UDF_INIT* initid) +{ +} + #define MAXSTRINGLENGTH 50 const char* PmSmallSideMaxMemory = "pmmaxmemorysmallside"; diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index aaa17473a..17c6e1817 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -82,6 +82,9 @@ CREATE FUNCTION idbextentmin RETURNS STRING soname 'libcalmysql.so'; CREATE FUNCTION idbextentmax RETURNS STRING soname 'libcalmysql.so'; CREATE FUNCTION idbpartition RETURNS STRING soname 'libcalmysql.so'; CREATE FUNCTION idblocalpm RETURNS INTEGER soname 'libcalmysql.so'; +CREATE FUNCTION mcssystemready RETURNS INTEGER soname 'libcalmysql.so'; +CREATE FUNCTION mcssystemreadonly RETURNS INTEGER soname 'libcalmysql.so'; +CREATE FUNCTION mcswritessuspended RETURNS INTEGER soname 'libcalmysql.so'; CREATE DATABASE IF NOT EXISTS infinidb_vtable; CREATE DATABASE IF NOT EXISTS infinidb_querystats; diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 7d8a6c153..795414eea 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -1878,7 +1878,7 @@ int processCommand(string* arguments) } string DataRedundancyConfig; - string DataRedundancyCopies; + int DataRedundancyCopies; string DataRedundancyStorageType; try { oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); @@ -3565,7 +3565,7 @@ int processCommand(string* arguments) } string DataRedundancyConfig; - string DataRedundancyCopies; + int DataRedundancyCopies; string DataRedundancyStorageType; try { oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); @@ -5552,7 +5552,7 @@ int processCommand(string* arguments) } } string command = startup::StartUp::installDir() + "/bin/remote_command.sh " + (*hostConfigIter).IPAddr + " " + password + " 'mkdir -p " + startup::StartUp::installDir() + "/gluster/brick" + oam.itoa(brickID) + "'"; - int status = system(command.c_str()); +// int status = system(command.c_str()); brickID++; } } @@ -5853,7 +5853,7 @@ int processCommand(string* arguments) } } - if ( DataRedundancyConfig == "y" && devicenetworklist.size() != DataRedundancyCopies) { + if ( DataRedundancyConfig == "y" && devicenetworklist.size() != (size_t)DataRedundancyCopies) { cout << endl << "**** removeModule Failed : Data Redundancy requires you to remove modules in groups equal to number of copies" << endl; quit = true; } @@ -6828,7 +6828,6 @@ int processCommand(string* arguments) { string DataRedundancyConfig = "n"; - int DataRedundancyCopies; try { oam.getSystemConfig( "DataRedundancyConfig", DataRedundancyConfig); } diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index bc91217f0..569b699fa 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -250,7 +250,7 @@ int main(int argc, char *argv[]) catch (...) {} //get memory stats - long long total = myinfo.totalram / 1024 / 1000; +// long long total = myinfo.totalram / 1024 / 1000; // adjust max memory, 25% of total memory string percent = "25%"; diff --git a/procmon/main.cpp b/procmon/main.cpp index cc6835f3a..10da06f69 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -165,7 +165,7 @@ int main(int argc, char **argv) //re-read local system info with updated Columnstore.xml sleep(1); - Config* sysConfig = Config::makeConfig(); +// Config* sysConfig = Config::makeConfig(); MonitorConfig config; //PMwithUM config diff --git a/utils/winport/win_setup_mysql_part2.sql b/utils/winport/win_setup_mysql_part2.sql index 894e1583d..0a2a972fb 100644 --- a/utils/winport/win_setup_mysql_part2.sql +++ b/utils/winport/win_setup_mysql_part2.sql @@ -9,6 +9,9 @@ CREATE FUNCTION calonlinealter RETURNS INTEGER SONAME 'libcalmysql.dll'; CREATE FUNCTION calviewtablelock RETURNS STRING SONAME 'libcalmysql.dll'; CREATE FUNCTION calcleartablelock RETURNS STRING SONAME 'libcalmysql.dll'; CREATE FUNCTION calgetsqlcount RETURNS STRING SONAME 'libcalmysql.dll'; +CREATE FUNCTION mcssystemready RETURNS INTEGER SONAME 'libcalmysql.dll'; +CREATE FUNCTION mcssystemreadonly RETURNS INTEGER SONAME 'libcalmysql.dll'; +CREATE FUNCTION mcswritessuspended RETURNS INTEGER SONAME 'libcalmysql.dll'; create database if not exists calpontsys; use calpontsys; From 504fcf07785ec10ead92ac31600b152f6763728c Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 12 Jan 2018 16:29:01 -0600 Subject: [PATCH 13/41] MCOL-1128 Allow exeMgrThreadPool to use unlimited threads. The number will be limited by the number of sessions allowed in server. --- dbcon/joblist/jobstep.cpp | 2 +- dbcon/joblist/resourcemanager.h | 7 +---- dbcon/mysql/ha_calpont_impl.cpp | 4 +-- dmlproc/dmlproc.cpp | 2 +- exemgr/femsghandler.cpp | 2 +- exemgr/main.cpp | 12 +++----- utils/threadpool/threadpool.cpp | 51 ++++++++++++++++++++++++--------- 7 files changed, 47 insertions(+), 33 deletions(-) diff --git a/dbcon/joblist/jobstep.cpp b/dbcon/joblist/jobstep.cpp index 94229871a..b24a0c0ea 100644 --- a/dbcon/joblist/jobstep.cpp +++ b/dbcon/joblist/jobstep.cpp @@ -56,7 +56,7 @@ namespace joblist { boost::mutex JobStep::fLogMutex; //=PTHREAD_MUTEX_INITIALIZER; -ThreadPool JobStep::jobstepThreadPool(defaultJLThreadPoolSize, 0); +ThreadPool JobStep::jobstepThreadPool(0, 0); ostream& operator<<(ostream& os, const JobStep* rhs) { diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index 42ee6e569..ffc2de639 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -93,8 +93,6 @@ namespace joblist const uint64_t defaultNumBuckets = 128; const uint64_t defaultMaxElementsPerBuckert = 16 * 1024 * 1024; - const int defaultEMServerThreads = 50; - const int defaultEMServerQueueSize = 100; const int defaultEMSecondsBetweenMemChecks = 1; const int defaultEMMaxPct = 95; const int defaultEMPriority = 21; // @Bug 3385 @@ -149,13 +147,10 @@ namespace joblist typedef std::map MemMap; - - int getEmServerThreads() const { return getUintVal(fExeMgrStr, "ServerThreads", defaultEMServerThreads); } - int getEmServerQueueSize() const { return getUintVal(fExeMgrStr, "ServerQueueSize", defaultEMServerQueueSize); } int getEmSecondsBetweenMemChecks() const { return getUintVal(fExeMgrStr, "SecondsBetweenMemChecks", defaultEMSecondsBetweenMemChecks); } int getEmMaxPct() const { return getUintVal(fExeMgrStr, "MaxPct", defaultEMMaxPct); } EXPORT int getEmPriority() const; - int getEmExecQueueSize() const { return getUintVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); } + int getEmExecQueueSize() const { return getIntVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); } int getHjMaxBuckets() const { return getUintVal(fHashJoinStr, "MaxBuckets", defaultHJMaxBuckets); } unsigned getHjNumThreads() const { return fHjNumThreads; } //getUintVal(fHashJoinStr, "NumThreads", defaultNumThreads); } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index d4669c8d5..3b80a25bd 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -3466,7 +3466,7 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) if (((thd->lex)->sql_command == SQLCOM_INSERT) && (rows > 0)) ci->useCpimport = 0; - if ((ci->useCpimport > 0) && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) //If autocommit on batch insert will use cpimport to load data + if ((ci->useCpimport > 0) && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) //If autocommit on batch insert will use cpimport to load data { //store table info to connection info CalpontSystemCatalog::TableName tableName; @@ -3923,7 +3923,7 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) if ( ( ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || (thd->lex)->sql_command == SQLCOM_INSERT_SELECT) && !ci->singleInsert ) { - //@Bug 2438. Only load dta infile calls last batch process + //@Bug 2438. Only load data infile calls last batch process /* if ( ci->isLoaddataInfile && ((thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) || (ci->useCpimport == 0))) { //@Bug 2829 Handle ctrl-C if ( thd->killed > 0 ) diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index c1575ff27..c8ad57b62 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -565,7 +565,7 @@ int main(int argc, char* argv[]) // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). // From the pools perspective, it has no idea if it is ExeMgr doing the // creation, so it has no idea which way to set the flag. So we set the max here. - JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); +// JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); JobStep::jobstepThreadPool.setName("DMLProcJobList"); // if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") diff --git a/exemgr/femsghandler.cpp b/exemgr/femsghandler.cpp index f1b36204e..b306e00ac 100644 --- a/exemgr/femsghandler.cpp +++ b/exemgr/femsghandler.cpp @@ -24,7 +24,7 @@ using namespace std; using namespace joblist; using namespace messageqcpp; -threadpool::ThreadPool FEMsgHandler::threadPool(50,100); +threadpool::ThreadPool FEMsgHandler::threadPool; namespace { diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 0b63388d3..690511198 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1436,7 +1436,7 @@ int main(int argc, char* argv[]) // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). // From the pools perspective, it has no idea if it is ExeMgr doing the // creation, so it has no idea which way to set the flag. So we set the max here. - JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); +// JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); JobStep::jobstepThreadPool.setName("ExeMgrJobList"); // if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") // { @@ -1444,14 +1444,10 @@ int main(int argc, char* argv[]) // JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); // } - int serverThreads = rm->getEmServerThreads(); - int serverQueueSize = rm->getEmServerQueueSize(); int maxPct = rm->getEmMaxPct(); int pauseSeconds = rm->getEmSecondsBetweenMemChecks(); int priority = rm->getEmPriority(); - FEMsgHandler::threadPool.setMaxThreads(serverThreads); - FEMsgHandler::threadPool.setQueueSize(serverQueueSize); FEMsgHandler::threadPool.setName("FEMsgHandler"); if (maxPct > 0) @@ -1472,8 +1468,7 @@ int main(int argc, char* argv[]) } } - cout << "Starting ExeMgr: st = " << serverThreads << ", sq = " << - serverQueueSize << ", qs = " << rm->getEmExecQueueSize() << ", mx = " << maxPct << ", cf = " << + cout << "Starting ExeMgr: qs = " << rm->getEmExecQueueSize() << ", mx = " << maxPct << ", cf = " << rm->getConfig()->configFile() << endl; //set ACTIVE state @@ -1488,8 +1483,9 @@ int main(int argc, char* argv[]) } } - threadpool::ThreadPool exeMgrThreadPool(serverThreads, serverQueueSize); + threadpool::ThreadPool exeMgrThreadPool; exeMgrThreadPool.setName("ExeMgrServer"); + exeMgrThreadPool.setDebug(true); for (;;) { IOSocket ios; diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index 1281e0b5f..8b0c0a848 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -196,6 +196,25 @@ uint64_t ThreadPool::invoke(const Functor_T &threadfunc) bAdded = true; } + if (fDebug) + { + ostringstream oss; + oss << "invoke thread " << " on " << fName + << " max " << fMaxThreads + << " queue " << fQueueSize + << " threads " << fThreadCount + << " running " << fIssued + << " waiting " << (waitingFunctorsSize - fIssued) + << " total " << waitingFunctorsSize; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format( args ); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage( message ); + } + // fQueueSize = 0 disables the queue and is an indicator to allow any number of threads to actually run. if (fThreadCount < fMaxThreads || fQueueSize == 0) { @@ -204,20 +223,6 @@ uint64_t ThreadPool::invoke(const Functor_T &threadfunc) lock1.unlock(); fThreads.create_thread(beginThreadFunc(*this)); - if (fDebug) - { - ostringstream oss; - oss << "invoke: Starting thread " << fThreadCount << " max " << fMaxThreads - << " queue " << fQueueSize; - logging::Message::Args args; - logging::Message message(0); - args.add(oss.str()); - message.format( args ); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logWarningMessage( message ); - } - if (bAdded) break; @@ -301,6 +306,24 @@ void ThreadPool::beginThread() throw() { Container_T::iterator todo = fNextFunctor++; ++fIssued; + if (fDebug) + { + ostringstream oss; + oss << "starting thread " << " on " << fName + << " max " << fMaxThreads + << " queue " << fQueueSize + << " threads " << fThreadCount + << " running " << fIssued + << " waiting " << (waitingFunctorsSize - fIssued) + << " total " << waitingFunctorsSize; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format( args ); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage( message ); + } lock1.unlock(); try { From 016fb96fea0c558b4e2f8d9390f17d658119ec9b Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 12 Jan 2018 16:32:47 -0600 Subject: [PATCH 14/41] MCOL-1128 remove debug --- exemgr/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 690511198..6967b2ca9 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1485,7 +1485,7 @@ int main(int argc, char* argv[]) threadpool::ThreadPool exeMgrThreadPool; exeMgrThreadPool.setName("ExeMgrServer"); - exeMgrThreadPool.setDebug(true); + for (;;) { IOSocket ios; From 2d0cea5542e178794cb4855757b80ffa8ab85bd0 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 16 Jan 2018 10:18:29 -0600 Subject: [PATCH 15/41] MCOL-1167 - fixed -c option --- oamapps/postConfigure/postConfigure.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9f7505f6b..4016688c2 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -166,7 +166,7 @@ bool thread_remote_installer = true; string singleServerInstall = "1"; string reuseConfig ="n"; -string oldFileName; +string oldFileName = oam::UnassignedName; string glusterCopies; string glusterInstalled = "n"; string hadoopInstalled = "n"; @@ -370,7 +370,8 @@ int main(int argc, char *argv[]) exit(1); } - oldFileName = installDir + "/etc/Columnstore.xml.rpmsave"; + if ( oldFileName == oam::UnassignedName ) + oldFileName = installDir + "/etc/Columnstore.xml.rpmsave"; cout << endl; cout << "This is the MariaDB ColumnStore System Configuration and Installation tool." << endl; From ce1f9c2ddab0ff33d6a81112f12782ad6012f63d Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 16 Jan 2018 10:33:13 -0600 Subject: [PATCH 16/41] MCOL-1135 - change rc-local service start command --- oam/install_scripts/post-install | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index bd3d3fb55..fc60e32e7 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -190,12 +190,13 @@ if [ $user = "root" ]; then else $SUDO chmod 777 $RCFILE printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILEl > /dev/null 2>&1 + + if [ -n "$systemctl" ]; then + $SUDO systemctl start rc-local >/dev/null 2>&1 + $SUDO systemctl enable rc-local >/dev/null 2>&1 + fi fi -if [ -n "$systemctl" ]; then - $SUDO systemctl restart rc-local >/dev/null 2>&1 - $SUDO systemctl enable rc-local >/dev/null 2>&1 -fi #setup the columnstore service script rm -f /etc/init.d/columnstore >/dev/null 2>&1 From b3bbd89d008a6e8d62511c3ff81938b3d4e1780c Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 16 Jan 2018 12:29:46 -0600 Subject: [PATCH 17/41] MCOL-962 Can't comment out import code. Do something else to remove warning. --- oamapps/mcsadmin/mcsadmin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 795414eea..f7a6e2a38 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -5552,7 +5552,7 @@ int processCommand(string* arguments) } } string command = startup::StartUp::installDir() + "/bin/remote_command.sh " + (*hostConfigIter).IPAddr + " " + password + " 'mkdir -p " + startup::StartUp::installDir() + "/gluster/brick" + oam.itoa(brickID) + "'"; -// int status = system(command.c_str()); + system(command.c_str()); brickID++; } } From a9c8f4821b4a3734b3413e5f2834ddcda4c39fee Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 16 Jan 2018 12:41:41 -0600 Subject: [PATCH 18/41] MCOL-1061 - fixed alais name --- oamapps/postConfigure/installer.cpp | 2 +- oamapps/postConfigure/postConfigure.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index bc91217f0..ffae8d34c 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -812,7 +812,7 @@ int main(int argc, char *argv[]) cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; - cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias" << endl << endl; + cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias.sh" << endl << endl; } else { diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9f7505f6b..3af0dcd90 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3336,7 +3336,7 @@ int main(int argc, char *argv[]) cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; - cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias" << endl << endl; + cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias.sh" << endl << endl; } else { From 9749d825b509750911b136f69f2c25b4b4597121 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 16 Jan 2018 13:26:44 -0600 Subject: [PATCH 19/41] MCOL-1165 use the threadpool's idle down feature --- dbcon/joblist/jobstep.cpp | 2 +- dbcon/joblist/resourcemanager.h | 16 +++++++++++----- dmlproc/dmlproc.cpp | 12 ++++++------ exemgr/main.cpp | 27 ++++++++++++++++++--------- utils/threadpool/threadpool.cpp | 2 +- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/dbcon/joblist/jobstep.cpp b/dbcon/joblist/jobstep.cpp index b24a0c0ea..94229871a 100644 --- a/dbcon/joblist/jobstep.cpp +++ b/dbcon/joblist/jobstep.cpp @@ -56,7 +56,7 @@ namespace joblist { boost::mutex JobStep::fLogMutex; //=PTHREAD_MUTEX_INITIALIZER; -ThreadPool JobStep::jobstepThreadPool(0, 0); +ThreadPool JobStep::jobstepThreadPool(defaultJLThreadPoolSize, 0); ostream& operator<<(ostream& os, const JobStep* rhs) { diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index ffc2de639..73de7ce75 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -93,6 +93,7 @@ namespace joblist const uint64_t defaultNumBuckets = 128; const uint64_t defaultMaxElementsPerBuckert = 16 * 1024 * 1024; + const int defaultEMServerThreads = 50; const int defaultEMSecondsBetweenMemChecks = 1; const int defaultEMMaxPct = 95; const int defaultEMPriority = 21; // @Bug 3385 @@ -147,10 +148,14 @@ namespace joblist typedef std::map MemMap; - int getEmSecondsBetweenMemChecks() const { return getUintVal(fExeMgrStr, "SecondsBetweenMemChecks", defaultEMSecondsBetweenMemChecks); } - int getEmMaxPct() const { return getUintVal(fExeMgrStr, "MaxPct", defaultEMMaxPct); } - EXPORT int getEmPriority() const; - int getEmExecQueueSize() const { return getIntVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); } + // @MCOL-513 - Added threadpool to ExeMgr + int getEmServerThreads() const { return getIntVal(fExeMgrStr, "ThreadPoolSize", defaultEMServerThreads); } + std::string getExeMgrThreadPoolDebug() const { return getStringVal(fExeMgrStr, "ThreadPoolDebug", "N"); } + + int getEmSecondsBetweenMemChecks() const { return getUintVal(fExeMgrStr, "SecondsBetweenMemChecks", defaultEMSecondsBetweenMemChecks); } + int getEmMaxPct() const { return getUintVal(fExeMgrStr, "MaxPct", defaultEMMaxPct); } + EXPORT int getEmPriority() const; + int getEmExecQueueSize() const { return getIntVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); } int getHjMaxBuckets() const { return getUintVal(fHashJoinStr, "MaxBuckets", defaultHJMaxBuckets); } unsigned getHjNumThreads() const { return fHjNumThreads; } //getUintVal(fHashJoinStr, "NumThreads", defaultNumThreads); } @@ -165,8 +170,9 @@ namespace joblist uint32_t getJlScanLbidReqThreshold() const { return getUintVal(fJobListStr,"ScanLbidReqThreshold", defaultScanLbidReqThreshold); } // @MCOL-513 - Added threadpool to JobSteps - uint32_t getJLThreadPoolSize() const { return getUintVal(fJobListStr, "ThreadPoolSize", defaultJLThreadPoolSize); } + int getJLThreadPoolSize() const { return getIntVal(fJobListStr, "ThreadPoolSize", defaultJLThreadPoolSize); } std::string getJlThreadPoolDebug() const { return getStringVal(fJobListStr, "ThreadPoolDebug", "N"); } + std::string getDMLJlThreadPoolDebug() const { return getStringVal(fJobListStr, "DMLThreadPoolDebug", "N"); } // @bug 1264 - Added LogicalBlocksPerScan configurable which determines the number of blocks contained in each BPS scan request. uint32_t getJlLogicalBlocksPerScan() const { return getUintVal(fJobListStr,"LogicalBlocksPerScan", defaultLogicalBlocksPerScan); } diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index c8ad57b62..0db77f23a 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -565,14 +565,14 @@ int main(int argc, char* argv[]) // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). // From the pools perspective, it has no idea if it is ExeMgr doing the // creation, so it has no idea which way to set the flag. So we set the max here. -// JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); + JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); JobStep::jobstepThreadPool.setName("DMLProcJobList"); -// if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") -// { -// JobStep::jobstepThreadPool.setDebug(true); -// JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); -// } + if (rm->getDMLJlThreadPoolDebug() == "Y" || rm->getDMLJlThreadPoolDebug() == "y") + { + JobStep::jobstepThreadPool.setDebug(true); + JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); + } //set ACTIVE state try diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 6967b2ca9..5abed0144 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1436,18 +1436,20 @@ int main(int argc, char* argv[]) // because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton). // From the pools perspective, it has no idea if it is ExeMgr doing the // creation, so it has no idea which way to set the flag. So we set the max here. -// JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); + JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); JobStep::jobstepThreadPool.setName("ExeMgrJobList"); -// if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") -// { -// JobStep::jobstepThreadPool.setDebug(true); -// JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); -// } + if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") + { + JobStep::jobstepThreadPool.setDebug(true); + JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); + } + int serverThreads = rm->getEmServerThreads(); int maxPct = rm->getEmMaxPct(); int pauseSeconds = rm->getEmSecondsBetweenMemChecks(); int priority = rm->getEmPriority(); + FEMsgHandler::threadPool.setMaxThreads(serverThreads); FEMsgHandler::threadPool.setName("FEMsgHandler"); if (maxPct > 0) @@ -1468,7 +1470,8 @@ int main(int argc, char* argv[]) } } - cout << "Starting ExeMgr: qs = " << rm->getEmExecQueueSize() << ", mx = " << maxPct << ", cf = " << + cout << "Starting ExeMgr: st = " << serverThreads << + ", qs = " << rm->getEmExecQueueSize() << ", mx = " << maxPct << ", cf = " << rm->getConfig()->configFile() << endl; //set ACTIVE state @@ -1483,10 +1486,16 @@ int main(int argc, char* argv[]) } } - threadpool::ThreadPool exeMgrThreadPool; + threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); exeMgrThreadPool.setName("ExeMgrServer"); - for (;;) + if (rm->getExeMgrThreadPoolDebug() == "Y" || rm->getExeMgrThreadPoolDebug() == "y") + { + exeMgrThreadPool.setDebug(true); + exeMgrThreadPool.invoke(ThreadPoolMonitor(&exeMgrThreadPool)); + } + + for (;;) { IOSocket ios; ios = mqs->accept(); diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index 8b0c0a848..197bdedd9 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -470,7 +470,7 @@ void ThreadPoolMonitor::operator()() << setw(4) << tv.tv_usec/100 << " Name " << fPool->fName << " Active " << fPool->waitingFunctorsSize - << " Most " << fPool->fThreadCount + << " ThdCnt " << fPool->fThreadCount << " Max " << fPool->fMaxThreads << " Q " << fPool->fQueueSize << endl; From 7bc2e2476954a8e7fe1979a08ebbd4ec4561d043 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 16 Jan 2018 14:06:24 -0600 Subject: [PATCH 20/41] MCOL-1066 - changed direcory path for non-root install local disk --- oamapps/serverMonitor/diskMonitor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oamapps/serverMonitor/diskMonitor.cpp b/oamapps/serverMonitor/diskMonitor.cpp index 2edcaa2f6..bef1df560 100644 --- a/oamapps/serverMonitor/diskMonitor.cpp +++ b/oamapps/serverMonitor/diskMonitor.cpp @@ -23,6 +23,7 @@ ***************************************************************************/ #include "serverMonitor.h" +#include "installdir.h" using namespace std; using namespace oam; @@ -223,7 +224,7 @@ void diskMonitor() string fileName; // check local if ( deviceName == "/") { - fileName = deviceName + "usr/local/mariadb/columnstore/"; + fileName = deviceName + startup::StartUp::installDir(); } else { From 3ed23512e1d7a7d8c631ff687ecf71b7b752b330 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 17 Jan 2018 11:18:39 -0600 Subject: [PATCH 21/41] MCOL-1058 - added check for maradin password and mariadb-libs paclage --- oam/install_scripts/mariadb-command-line.sh | 7 + oam/install_scripts/remote_command.sh | 2 +- .../clusterTester/columnstoreClusterTester.sh | 285 ++++++++++++++++++ 3 files changed, 293 insertions(+), 1 deletion(-) mode change 100644 => 100755 oam/install_scripts/mariadb-command-line.sh diff --git a/oam/install_scripts/mariadb-command-line.sh b/oam/install_scripts/mariadb-command-line.sh old mode 100644 new mode 100755 index bb2750581..efdbbddd5 --- a/oam/install_scripts/mariadb-command-line.sh +++ b/oam/install_scripts/mariadb-command-line.sh @@ -8,6 +8,13 @@ # check log for error checkForError() { + grep "ERROR 1045" /tmp/mariadb-command-line.log > /tmp/error.check + if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then + echo "ERROR - PASSWORD: check log file: /tmp/mariadb-command-line.log" + rm -f /tmp/error.check + exit 2 + fi + grep ERROR /tmp/mariadb-command-line.log > /tmp/error.check if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then echo "ERROR: check log file: /tmp/mariadb-command-line.log" diff --git a/oam/install_scripts/remote_command.sh b/oam/install_scripts/remote_command.sh index 9d0f7c2ed..75095632e 100755 --- a/oam/install_scripts/remote_command.sh +++ b/oam/install_scripts/remote_command.sh @@ -10,7 +10,7 @@ # Argument 5 - Remote user name (optional) # Argument 6 - Force a tty to be allocated (optional) set stty_init {cols 512 -opost}; -set timeout 10 +set timeout 30 set SERVER [lindex $argv 0] set PASSWORD [lindex $argv 1] set COMMAND [lindex $argv 2] diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 78fa3f889..da0f83440 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -711,6 +711,49 @@ checkTime() fi } +checkMysqlPassword() +{ + # Locale check + # + echo "" + echo "** Run MariaDB Console Password check" + echo "" + + #get MariaDB password + pass=true + `$COLUMNSTORE_INSTALL_DIR/mysql/mysql-Columnstore start > /dev/null 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/mariadb-command-line.sh > /dev/null 2>&1` + if [ "$?" -eq 2 ]; then + echo "${bold}Failed${normal}, Local Node MariaDB login failed with missing password file, /root/.my.cnf" + fi + + if [ "$IPADDRESSES" != "" ]; then + `/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/mariadb-command-line.sh /tmp/.` + + for ipadd in "${NODE_IPADDRESS[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD $COLUMNSTORE_INSTALL_DIR/mysql/mysql-Columnstore start > /dev/null 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD /tmp/mariadb-command-line.sh 1 > /tmp/remote_scp_put_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "Error running remote_scp_put.sh to $ipadd Node, check /tmp/remote_scp_put_check" + exit 1 + else + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD /tmp/mariadb-command-line.sh 1 > /tmp/remote_command_check` + `cat /tmp/remote_command_check | grep "ERROR - PASSWORD" > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node MariaDB login failed with missing password file, /root/.my.cnf" + pass=false + fi + fi + done + fi + + if ! $pass; then + checkContinue + else + echo "Passed, no problems detected with a MariaDB password being set without an associated /root/.my.cnf" + fi +} + checkPackages() { # @@ -722,6 +765,7 @@ checkPackages() echo "" declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "perl-DBD-MySQL") + declare -a CENTOS_PKG_NOT=("mariadb-libs") if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then if [ ! `which yum 2>/dev/null` ] ; then @@ -757,6 +801,24 @@ checkPackages() checkContinue fi + #check for package that shouldnt be installed + pass=true + for PKG in "${CENTOS_PKG_NOT[@]}"; do + `yum list installed "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep Installed > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if [ $pass == true ] ; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi + echo "" pass=true if [ "$IPADDRESSES" != "" ]; then @@ -792,11 +854,37 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${CENTOS_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + rc="$?" + if [ $rc -eq 2 ] ; then + echo "${bold}Failed${normal}, $ipadd Node, 'yum' not installed" + pass=false + REPORTPASS=false + break + elif [ $rc -ne 1 ] ; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "perl-DBD-mysql") + declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18") if [ "$OS" == "suse12" ]; then if [ ! `which rpm 2>/dev/null` ] ; then @@ -821,6 +909,24 @@ checkPackages() else checkContinue fi + + #check for package that shouldnt be installed + pass=true + for PKG in "${SUSE_PKG_NOT[@]}"; do + `rpm -qi "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep "not installed" > /dev/null 2>&1` + if [ "$?" -ne 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi fi echo "" @@ -844,11 +950,32 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${SUSE_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + rc="$?" + if [ $rc -eq 0 ] ; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libdbd-mysql-perl") + declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") if [ "$OS" == "ubuntu16" ] ; then if [ ! `which dpkg 2>/dev/null` ] ; then @@ -873,6 +1000,24 @@ checkPackages() else checkContinue fi + + #check for package that shouldnt be installed + pass=true + for PKG in "${UBUNTU_PKG_NOT[@]}"; do + `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi fi echo "" @@ -909,11 +1054,45 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${UBUNTU_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + else + `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" + pass=false + break + else + `cat pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + fi + + `rm -f pkg_check` + fi + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libdbd-mysql-perl") + declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian8" ]; then if [ ! `which dpkg 2>/dev/null` ] ; then @@ -938,6 +1117,24 @@ checkPackages() else checkContinue fi + + #check for package that shouldnt be installed + pass=true + for PKG in "${DEBIAN_PKG_NOT[@]}"; do + `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi fi echo "" @@ -974,11 +1171,45 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${DEBIAN_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + else + `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" + pass=false + break + else + `cat pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + fi + + `rm -f pkg_check` + fi + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") + declare -a DEBIAN9_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian9" ]; then if [ ! `which dpkg 2>/dev/null` ] ; then @@ -1003,6 +1234,25 @@ checkPackages() else checkContinue fi + + #check for package that shouldnt be installed + pass=true + for PKG in "${DEBIAN9_PKG_NOT[@]}"; do + `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi + fi echo "" @@ -1039,6 +1289,39 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${DEBIAN9_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + else + `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" + pass=false + break + else + `cat pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + fi + + `rm -f pkg_check` + fi + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi @@ -1063,6 +1346,8 @@ if [ "$IPADDRESSES" != "" ]; then checkPorts checkTime fi + +checkMysqlPassword checkPackages if [ $REPORTPASS == true ] ; then From 8a78fbacd2e1c3c3124b5e401865401443d43d0d Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 17 Jan 2018 15:41:10 -0600 Subject: [PATCH 22/41] MCOL-1060 - fix spelling/grammer issues --- utils/clusterTester/columnstoreClusterTester.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index da0f83440..8fce62b7b 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -43,7 +43,7 @@ checkContinue() { helpPrint () { ################################################################################ echo "" - echo "This is the MariaDB ColumnStore Cluster System Test tool." + echo "This is the MariaDB ColumnStore Cluster System Test Tool." echo "" echo "It will run a set of test to validate the setup of the MariaDB Columnstore system." echo "This can be run prior to the install of MariaDB ColumnStore to make sure the" @@ -1352,7 +1352,7 @@ checkPackages if [ $REPORTPASS == true ] ; then echo "" - echo "*** Finished Validation of the Cluster, all Test Passed ***" + echo "*** Finished Validation of the Cluster, all Tests Passed ***" echo "" exit 0 else From c56555e9fe88a2b8df2b20eb1ee7692e7437e8bd Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 18 Jan 2018 17:14:14 -0600 Subject: [PATCH 23/41] MCOL-1137 - fixed master/slave setup after failvover --- .../disable-rep-columnstore.sh | 2 +- procmgr/main.cpp | 130 ++++++++++-------- procmgr/processmanager.cpp | 41 +++++- procmon/processmonitor.cpp | 10 ++ 4 files changed, 120 insertions(+), 63 deletions(-) diff --git a/oam/install_scripts/disable-rep-columnstore.sh b/oam/install_scripts/disable-rep-columnstore.sh index 0a8c9b17f..1c730da4c 100644 --- a/oam/install_scripts/disable-rep-columnstore.sh +++ b/oam/install_scripts/disable-rep-columnstore.sh @@ -56,7 +56,7 @@ checkForError # # Run reset slave command # -echo "Run start slave command" >>/tmp/disable-rep-status.log +echo "Run reset slave command" >>/tmp/disable-rep-status.log cat >/tmp/idb_disable-rep.sql < Date: Fri, 19 Jan 2018 03:43:00 -0600 Subject: [PATCH 24/41] MCOL-1042: add full list of dependencies. --- cpackEngineDEB.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 461714ced..7631c08b1 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -65,9 +65,9 @@ if (EXISTS "/etc/debian_version") set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") endif () if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs, libsnappy1") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, libsnappy1") else () - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs, libsnappy1v5") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, libsnappy1v5") endif () SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") From 5e30895b9c16af157dd849e0cdd233c61d2e372d Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 19 Jan 2018 10:56:35 -0600 Subject: [PATCH 25/41] MCOL-1137 - fixed procmgr host_standby --- procmgr/processmanager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 10d52846c..9f359e46a 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -9718,7 +9718,7 @@ std::string ProcessManager::getStandbyModule() { // Found a ProcessManager in a COLD_STANDBY state newStandbyModule = systemprocessstatus.processstatus[i].Module; - break; + continue; } if ( systemprocessstatus.processstatus[i].ProcessName == "ProcessManager" && From a002d33d5d451f3a8f78c5e782f995c239b66bb4 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Fri, 19 Jan 2018 13:38:15 -0600 Subject: [PATCH 26/41] Merge deleted change to include columnstoreversion.h --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 51d2e7a8a..482e8d19e 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,8 +76,7 @@ ENDIF("${isSystemDir}" STREQUAL "-1") INCLUDE (configureEngine.cmake) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum @ONLY IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL_ENGINE} COMPONENT platform) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) INCLUDE(bison.cmake) From c1035dae73c8cedbcfd521c0026e1c10c924825f Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 19 Jan 2018 15:21:28 -0600 Subject: [PATCH 27/41] MCOL-1060 - test tool change #2 --- utils/clusterTester/columnstoreClusterTester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 8fce62b7b..b512eeb20 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -1330,7 +1330,7 @@ checkPackages() } echo "" -echo "*** This is the MariaDB Columnstore Cluster System test tool ***" +echo "*** This is the MariaDB Columnstore Cluster System Test Tool ***" echo "" checkLocalOS From 403e0ef1dca773c9e7b3ce9e1dc97007b93d0e73 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Fri, 19 Jan 2018 16:02:59 -0600 Subject: [PATCH 28/41] MCOL-1114: Change cmake minimum versions. --- CMakeLists.txt | 3 ++- cpackEngineDEB.cmake | 2 ++ cpackEngineRPM.cmake | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b40c663a3..0932307ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12) + # Avoid warnings in higher versions if("${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" GREATER 2.6) CMAKE_POLICY(VERSION 2.8) diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 3c80e3ccd..30d3d0ab9 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -1,5 +1,7 @@ IF(DEB) +CMAKE_MINIMUM_REQUIRED(VERSION 3.4) + SET(CMAKE_INSTALL_PREFIX ${INSTALL_ENGINE}) SET(CPACK_GENERATOR "DEB") diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 7ad235dc8..98974325d 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -5,7 +5,6 @@ SET(CMAKE_INSTALL_PREFIX ${INSTALL_ENGINE}) SET(CPACK_GENERATOR "RPM") SET(CPACK_RPM_PACKAGE_DEBUG 1) SET(CPACK_PACKAGING_INSTALL_PREFIX ${INSTALL_ENGINE}) -CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7) SET(CPACK_RPM_COMPONENT_INSTALL ON) From 63adbd0f9921807bb480a570fcd8c9bc336fd5f4 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 22 Jan 2018 15:15:03 +0000 Subject: [PATCH 29/41] Fix missing compiler flag from 1.0 -> 1.1 merge --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 482e8d19e..c36bdcfaa 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,8 +107,8 @@ endif() INCLUDE(check_compiler_flag.cmake) -MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) -MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) +MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) +MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) # enable security hardening features, like most distributions do # in our benchmarks that costs about ~1% of performance, depending on the load From def46ca31b3df0e8c916a133c8d60c3e9627ad36 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 22 Jan 2018 14:13:42 -0600 Subject: [PATCH 30/41] update version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0a349c477..7053d9518 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=12 +COLUMNSTORE_VERSION_PATCH=13 COLUMNSTORE_VERSION_RELEASE=1 From 1a16847cd5041a67300dbb59303c4a0e41a37325 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 22 Jan 2018 14:19:12 -0600 Subject: [PATCH 31/41] MCOL-1138 - fix issue where no HOT_STANDBY procmgr existed after pm1 outage --- procmgr/main.cpp | 11 ++++++++++- procmgr/processmanager.cpp | 5 +++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 5ad80014c..81e1593af 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1828,9 +1828,18 @@ void pingDeviceThread() break; // if disabled and not amazon, skip - if (opState == oam::AUTO_DISABLED && !amazon) + if ( (opState == oam::AUTO_DISABLED) && !amazon) break; + // if disabled, amazon,and NOT terminated skip + if ( (opState == oam::AUTO_DISABLED) && amazon) + { + // return values = 'ip address' for running or rebooting, stopped or terminated + string currentIPAddr = oam.getEC2InstanceIpAddress(hostName); + if ( currentIPAddr != "terminated") + break; + } + log.writeLog(__LINE__, "module failed to respond to pings: " + moduleName, LOG_TYPE_WARNING); //bump module ping failure counter diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 9f359e46a..5a2739358 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -9455,7 +9455,7 @@ int ProcessManager::OAMParentModuleChange() //restart/reinit processes to force their release of the controller node port if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) && - ( moduleNameList.size() <= 1 && config.moduleType() == "pm") ) + ( moduleNameList.size() <= 0 && config.moduleType() == "pm") ) { status = 0; } @@ -9710,7 +9710,8 @@ std::string ProcessManager::getStandbyModule() //already have a hot-standby return ""; - if ( backupStandbyModule != "NONE" ) + if ( ( backupStandbyModule != "NONE" ) || + ( newStandbyModule != "NONE" ) ) continue; if ( systemprocessstatus.processstatus[i].ProcessName == "ProcessManager" && From 8e0a5e65b6572fc139492f432740a15c09a3f6e6 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 23 Jan 2018 10:47:16 -0600 Subject: [PATCH 32/41] MCOL-1042: edit list of debian package depends --- cpackEngineDEB.cmake | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 7631c08b1..d850d04db 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -65,9 +65,11 @@ if (EXISTS "/etc/debian_version") set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") endif () if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, libsnappy1") -else () - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, libsnappy1v5") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1") +elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5") +else() + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, snappy, net-tools") endif () SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") From 6211372f61bbd24b5746660182088a7e8fea4eca Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 23 Jan 2018 21:17:33 +0000 Subject: [PATCH 33/41] MCOL-1160 Track and flush dictionary blocks When bulk write API is used dictionary blocks that are written to needs flushing in PrimProc. This patch tracks the blocks and flushes them. --- writeengine/server/we_dmlcommandproc.cpp | 16 ++++++++++ writeengine/wrapper/writeengine.cpp | 38 ++++++++++++++++++++++++ writeengine/wrapper/writeengine.h | 12 ++++++++ 3 files changed, 66 insertions(+) diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 9015decff..0fda76182 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2027,6 +2027,22 @@ uint8_t WE_DMLCommandProc::commitBatchAutoOn(messageqcpp::ByteStream& bs, std::s if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size()>0) ) cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); TableMetaData::removeTableMetaData(tableOid); + + // MCOL-1160 For API bulk insert flush the PrimProc cached dictionary + // blocks tounched + std::tr1::unordered_map::iterator mapIter; + mapIter = fWEWrapper.getDictMap().find(txnID); + if (mapIter != fWEWrapper.getDictMap().end()) + { + std::set::iterator lbidIter; + std::vector dictFlushBlks; + for (lbidIter = (*mapIter).second.begin(); lbidIter != (*mapIter).second.end(); lbidIter++) + { + dictFlushBlks.push_back((*lbidIter)); + } + cacheutils::flushPrimProcAllverBlocks(dictFlushBlks); + fWEWrapper.getDictMap().erase(txnID); + } return rc; } diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 8da33caad..e2d785021 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1485,6 +1485,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, bool newExtent = false; RIDList ridList; ColumnOp* colOp = NULL; + std::vector dictLbids; // Set tmp file suffix to modify HDFS db file bool useTmpSuffix = false; @@ -1898,6 +1899,7 @@ timer.start("tokenize"); timer.stop("tokenize"); #endif memcpy(colValPtr, &dctTuple.token, 8); + dictLbids.push_back(dctTuple.token.fbo); } dctStr_iter++; @@ -1946,6 +1948,7 @@ timer.start("tokenize"); timer.stop("tokenize"); #endif memcpy(colValPtr, &dctTuple.token, 8); + dictLbids.push_back(dctTuple.token.fbo); } dctStr_iter++; } @@ -2100,6 +2103,7 @@ timer.start("writeColumnRec"); // Write row(s) to database file(s) //---------------------------------------------------------------------- bool versioning = !(isAutoCommitOn && insertSelect); + AddDictToList(txnid, dictLbids); rc = writeColumnRecBinary(txnid, colStructList, colValueList, rowIdArray, newColStructList, tableOid, useTmpSuffix, versioning); // @bug 5572 HDFS tmp file } return rc; @@ -5177,6 +5181,7 @@ int WriteEngineWrapper::rollbackCommon(const TxnID& txnid, int sessionId) // BUG 4312 RemoveTxnFromLBIDMap(txnid); + RemoveTxnFromDictMap(txnid); config::Config *config = config::Config::makeConfig(); prefix = config->getConfig("SystemConfig", "DBRMRoot"); @@ -5401,6 +5406,7 @@ int WriteEngineWrapper::rollbackVersion(const TxnID& txnid, int sessionId) { // BUG 4312 RemoveTxnFromLBIDMap(txnid); + RemoveTxnFromDictMap(txnid); return BRMWrapper::getInstance()->rollBackVersion(txnid, sessionId); } @@ -5492,6 +5498,7 @@ int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, int WriteEngineWrapper::flushDataFiles(int rc, const TxnID txnId, std::map & columnOids) { RemoveTxnFromLBIDMap(txnId); + RemoveTxnFromDictMap(txnId); for (int i = 0; i < TOTAL_COMPRESS_OP; i++) { @@ -5506,6 +5513,27 @@ int WriteEngineWrapper::flushDataFiles(int rc, const TxnID txnId, std::map& lbids) +{ + std::tr1::unordered_map::iterator mapIter; + + mapIter = m_dictLBIDMap.find(txnid); + if (mapIter == m_dictLBIDMap.end()) + { + dictLBIDRec_t tempRecord; + tempRecord.insert(lbids.begin(), lbids.end()); + m_dictLBIDMap[txnid] = tempRecord; + return; + } + else + { + dictLBIDRec_t &txnRecord = mapIter->second; + txnRecord.insert(lbids.begin(), lbids.end()); + } + +} + /*********************************************************** * DESCRIPTION: * Add an lbid to a list of lbids for sending to markExtentsInvalid. @@ -5598,6 +5626,16 @@ int WriteEngineWrapper::AddLBIDtoList(const TxnID txnid, return rtn; } +void WriteEngineWrapper::RemoveTxnFromDictMap(const TxnID txnid) +{ + std::tr1::unordered_map::iterator mapIter; + + mapIter = m_dictLBIDMap.find(txnid); + if (mapIter != m_dictLBIDMap.end()) + { + m_dictLBIDMap.erase(txnid); + } +} /*********************************************************** * DESCRIPTION: diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index 7c7862d5f..f1783ac8e 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -85,6 +85,7 @@ struct TxnLBIDRec }; typedef boost::shared_ptr SP_TxnLBIDRec_t; +typedef std::set dictLBIDRec_t; /** Class WriteEngineWrapper */ class WriteEngineWrapper : public WEObj @@ -408,6 +409,10 @@ public: return m_txnLBIDMap; }; + std::tr1::unordered_map& getDictMap() + { + return m_dictLBIDMap; + }; /** * @brief Flush the ChunkManagers. */ @@ -686,6 +691,9 @@ private: const RID filesPerColumnPartition, const RID extentsPerSegmentFile, const RID extentRows, uint16_t startDBRoot, unsigned dbrootCnt); + void AddDictToList(const TxnID txnid, std::vector& lbids); + void RemoveTxnFromDictMap(const TxnID txnid); + // Bug 4312: We use a hash set to hold the set of starting LBIDS for a given // txn so that we don't waste time marking the same extent as invalid. This // list should be trimmed if it gets too big. @@ -703,6 +711,10 @@ private: // This is a Map of sets of LBIDS for each transaction. A Transaction's list will be removed upon commit or rollback. std::tr1::unordered_map m_txnLBIDMap; + // MCOL-1160: We need to track dictionary LBIDs so we can tell PrimProc + // to flush the blocks after an API bulk-write. + std::tr1::unordered_map m_dictLBIDMap; + ColumnOp* m_colOp[TOTAL_COMPRESS_OP]; // column operations Dctnry* m_dctnry[TOTAL_COMPRESS_OP]; // dictionary operations OpType m_opType; // operation type From dfb5aa41c65a7cec766fb9c090c67c1dbad47174 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 24 Jan 2018 16:39:24 -0600 Subject: [PATCH 34/41] MCOL-1137-fixed issue with no slaves --- procmon/processmonitor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 51888329c..3a6d69f24 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -4951,8 +4951,8 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master string moduleName = (*pt).DeviceName; //skip if local module or module is not ACTIVE - if ( moduleName == config.moduleName() ) - continue; +// if ( moduleName == config.moduleName() ) +// continue; int opState = oam::ACTIVE; bool degraded; From bd5daf240b7ab5664bb81ba9d10cdce914e642ac Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 24 Jan 2018 16:43:23 -0600 Subject: [PATCH 35/41] MCOL-1137-fixed issue with no slaves --- procmon/processmonitor.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 3a6d69f24..8f51cccf1 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -4950,9 +4950,7 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master { string moduleName = (*pt).DeviceName; - //skip if local module or module is not ACTIVE -// if ( moduleName == config.moduleName() ) -// continue; + //skip if module is not ACTIVE int opState = oam::ACTIVE; bool degraded; From 9454f6e7f619a8eb42e65cf2f3026e0d1f70c259 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 26 Jan 2018 14:54:48 -0600 Subject: [PATCH 36/41] MCOL-1034 - incorporated Bens changes into latest branch --- procmgr/main.cpp | 136 ++++++++----------------------------- procmgr/processmanager.cpp | 56 ++++++--------- 2 files changed, 50 insertions(+), 142 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 81e1593af..a5ebb9183 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1588,14 +1588,17 @@ void pingDeviceThread() if ( moduleName.substr(0,MAX_MODULE_TYPE_SIZE) == "pm" ) processManager.setStandbyModule(moduleName); } - DBRootConfigList::iterator pt = dbrootConfigList.begin(); - if (( DBRootStorageType == "DataRedundancy") && (*pt == 1)) + if ((moduleName.find("pm") == 0) && (dbrootConfigList.size() > 0)) { - log.writeLog(__LINE__, "stopModule, " + config.moduleName(), LOG_TYPE_DEBUG); - processManager.stopModule(config.moduleName(), oam::FORCEFUL, false); - processManager.switchParentOAMModule(moduleName); - processManager.stopProcess(config.moduleName(), "ProcessManager", oam::FORCEFUL, true); - break; + DBRootConfigList::iterator pt = dbrootConfigList.begin(); + if (( DBRootStorageType == "DataRedundancy") && (*pt == 1)) + { + log.writeLog(__LINE__, "stopModule, " + config.moduleName(), LOG_TYPE_DEBUG); + processManager.stopModule(config.moduleName(), oam::FORCEFUL, false); + processManager.switchParentOAMModule(moduleName); + processManager.stopProcess(config.moduleName(), "ProcessManager", oam::FORCEFUL, true); + break; + } } } else { @@ -1627,22 +1630,13 @@ void pingDeviceThread() oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); + //set recycle process + processManager.recycleProcess(moduleName); + //distribute config file processManager.distributeConfigFile("system"); sleep(1); - // if a PM module was started successfully, restart ACTIVE DBRM(s), ExeMgr(s) / mysqld - if( moduleName.find("pm") == 0 ) { - processManager.restartProcessType("DBRMControllerNode", moduleName); - processManager.restartProcessType("DBRMWorkerNode"); - processManager.stopProcessType("DDLProc"); - processManager.stopProcessType("DMLProc"); - processManager.stopProcessType("ExeMgr"); - processManager.restartProcessType("PrimProc"); - sleep(1); - processManager.restartProcessType("ExeMgr"); - } - string moduleType = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); if ( MySQLRep == "y" ) { @@ -1660,22 +1654,6 @@ void pingDeviceThread() processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true); } } - else - { - if( moduleName.find("pm") == 0 ) { - processManager.restartProcessType("mysql", moduleName); - sleep(1); - } - } - - // if a PM module was started successfully, DMLProc/DDLProc - if( moduleName.find("pm") == 0 ) { - processManager.restartProcessType("WriteEngineServer"); - sleep(1); - processManager.restartProcessType("DDLProc"); - sleep(1); - processManager.restartProcessType("DMLProc"); - } //enable query stats dbrm.setSystemQueryReady(true); @@ -1684,79 +1662,6 @@ void pingDeviceThread() processManager.setQuerySystemState(true); processManager.setSystemState(oam::ACTIVE); - - //reset standby module - string newStandbyModule = processManager.getStandbyModule(); - - //send message to start new Standby Process-Manager, if needed - if ( !newStandbyModule.empty() && newStandbyModule != "NONE") { - processManager.setStandbyModule(newStandbyModule); - } - else - { - Config* sysConfig = Config::makeConfig(); - - // clear Standby OAM Module - sysConfig->setConfig("SystemConfig", "StandbyOAMModuleName", oam::UnassignedName); - sysConfig->setConfig("ProcStatusControlStandby", "IPAddr", oam::UnassignedIpAddr); - - //update Calpont Config table - try { - sysConfig->write(); - } - catch(...) - { - log.writeLog(__LINE__, "ERROR: sysConfig->write", LOG_TYPE_ERROR); - } - } - - if ( moduletypeconfig.RunType == SIMPLEX ) { - //start SIMPLEX runtype processes on a SIMPLEX runtype module - string moduletype = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); - DeviceNetworkList::iterator pt = moduletypeconfig.ModuleNetworkList.begin(); - for( ; pt != moduletypeconfig.ModuleNetworkList.end() ; pt++) - { - string launchModuleName = (*pt).DeviceName; - string launchModuletype = launchModuleName.substr(0,MAX_MODULE_TYPE_SIZE); - if ( moduletype != launchModuletype ) - continue; - - //skip if active pm module (local module) - if ( launchModuleName == config.moduleName() ) - continue; - - //check if module is active before starting any SIMPLEX STANDBY apps - try{ - int launchopState = oam::ACTIVE; - bool degraded; - oam.getModuleStatus(launchModuleName, launchopState, degraded); - - if (launchopState != oam::ACTIVE && launchopState != oam::STANDBY ) { - continue; - } - } - catch (exception& ex) - { -// string error = ex.what(); -// log.writeLog(__LINE__, "EXCEPTION ERROR on : " + error, LOG_TYPE_ERROR); - } - catch(...) - { -// log.writeLog(__LINE__, "EXCEPTION ERROR on getModuleStatus on module " + moduleName + ": Caught unknown exception!", LOG_TYPE_ERROR); - } - - int status; - log.writeLog(__LINE__, "Starting up STANDBY process on module " + launchModuleName, LOG_TYPE_DEBUG); - for ( int j = 0 ; j < 20 ; j ++ ) - { - status = processManager.startModule(launchModuleName, oam::FORCEFUL, oam::AUTO_OFFLINE); - if ( status == API_SUCCESS) - break; - } - log.writeLog(__LINE__, "pingDeviceThread: ACK received from '" + launchModuleName + "' Process-Monitor, return status = " + oam.itoa(status), LOG_TYPE_DEBUG); - } - } - //clear count moduleInfoList[moduleName] = 0; } @@ -2198,6 +2103,21 @@ void pingDeviceThread() //start SIMPLEX runtype processes on a SIMPLEX runtype module string moduletype = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); + if ( MySQLRep == "y" ) { + if ( moduletype == "um" || + ( moduletype == "pm" && config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) || + ( moduletype == "pm" && PMwithUM == "y") ) { + + //setup MySQL Replication for started modules + log.writeLog(__LINE__, "Setup MySQL Replication for module recovering from outage on " + moduleName, LOG_TYPE_DEBUG); + DeviceNetworkList devicenetworklist; + DeviceNetworkConfig devicenetworkconfig; + devicenetworkconfig.DeviceName = moduleName; + devicenetworklist.push_back(devicenetworkconfig); + processManager.setMySQLReplication(devicenetworklist); + } + } + try{ oam.getSystemConfig(moduletype, moduletypeconfig); } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 5a2739358..3eed6727f 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3438,40 +3438,27 @@ void ProcessManager::recycleProcess(string module) } catch(...) {} + //restart ExeMgrs/mysql if module is a pm - if ( moduleType == "pm" ) { - restartProcessType("DBRMControllerNode", module); - restartProcessType("DBRMWorkerNode"); + restartProcessType("DBRMControllerNode", module); + restartProcessType("DBRMWorkerNode"); + if( PrimaryUMModuleName == module) + { stopProcessType("DDLProc"); stopProcessType("DMLProc"); - stopProcessType("ExeMgr"); - restartProcessType("PrimProc"); - sleep(1); - restartProcessType("ExeMgr"); - sleep(1); - restartProcessType("mysql"); - } - else - { - restartProcessType("DBRMControllerNode", module); - restartProcessType("DBRMWorkerNode"); - restartProcessType("ExeMgr"); - } - if ( PrimaryUMModuleName == module ) - { - restartProcessType("DDLProc", module); - sleep(1); - restartProcessType("DMLProc", module); } + stopProcessType("ExeMgr"); + restartProcessType("PrimProc"); + sleep(1); + restartProcessType("ExeMgr"); + sleep(1); + restartProcessType("mysql"); - if( moduleType == "pm" && PrimaryUMModuleName != module) - { - restartProcessType("WriteEngineServer"); - sleep(1); - restartProcessType("DDLProc"); - sleep(1); - restartProcessType("DMLProc", module); - } + restartProcessType("WriteEngineServer"); + sleep(1); + restartProcessType("DDLProc",module); + sleep(1); + restartProcessType("DMLProc",module); return; } @@ -4263,7 +4250,6 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski SystemProcessStatus systemprocessstatus; ProcessStatus processstatus; int retStatus = API_SUCCESS; - bool setPMProcIPs = true; log.writeLog(__LINE__, "restartProcessType: Restart all " + processName, LOG_TYPE_DEBUG); @@ -4311,7 +4297,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) ) continue; - if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) && setPMProcIPs ) + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { string procModuleType = systemprocessstatus.processstatus[i].Module.substr(0,MAX_MODULE_TYPE_SIZE); if ( procModuleType == "pm" && PMwithUM == "y" ) @@ -4339,11 +4325,13 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski // if DDL or DMLProc, change IP Address if ( retStatus == oam::API_SUCCESS ) { - if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) && setPMProcIPs ) + sleep(5); + ProcessStatus procstat; + oam.getProcessStatus(processName, systemprocessstatus.processstatus[i].Module, procstat); + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { processManager.setPMProcIPs(systemprocessstatus.processstatus[i].Module, processName); - setPMProcIPs = false; - continue; + break; } } } From 63f8e1ce7147fef2804b44587817bcafcfd35164 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 30 Jan 2018 11:46:47 +0000 Subject: [PATCH 37/41] MCOL-1176 Fix API extent rollover When the API inserts data into ColumnStore which will roll over into a new extent that data wasn't being put into the new extent and corruption occured. This patch now tracks the additional data and inserts it into the new extent. It also makes sure the LBIDs are stored so that they are correctly committed. --- writeengine/wrapper/writeengine.cpp | 187 ++++++++++++++++++++++++++-- writeengine/wrapper/writeengine.h | 1 + 2 files changed, 179 insertions(+), 9 deletions(-) diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 8da33caad..dc863b823 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1475,6 +1475,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, Column curCol; ColStruct curColStruct; ColStructList newColStructList; + std::vector colNewValueList; DctnryStructList newDctnryStructList; HWM hwm = 0; HWM oldHwm = 0; @@ -2055,6 +2056,19 @@ timer.stop("tokenize"); tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); } + //-------------------------------------------------------------------------- + //Prepare the valuelist for the new extent + //-------------------------------------------------------------------------- + + for (unsigned i=1; i <= totalColumns; i++) + { + // Copy values to second value list + for (uint64_t j=rowsLeft; j > 0; j--) + { + colNewValueList.push_back(colValueList[(totalRow*i)-j]); + } + } + // end of allocate row id #ifdef PROFILE @@ -2091,6 +2105,22 @@ timer.start("writeColumnRec"); } } } + // If we create a new extent for this batch + for (unsigned i = 0; i < newColStructList.size(); i++) + { + colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + width = newColStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRidNew , BYTE_PER_BLOCK/width, width, curFbo, curBio); + if (successFlag) { + if (curFbo != lastFbo) { + RETURN_ON_ERROR(AddLBIDtoList(txnid, + lbids, + colDataTypes, + newColStructList[i], + curFbo)); + } + } + } } if (lbids.size() > 0) @@ -2100,7 +2130,7 @@ timer.start("writeColumnRec"); // Write row(s) to database file(s) //---------------------------------------------------------------------- bool versioning = !(isAutoCommitOn && insertSelect); - rc = writeColumnRecBinary(txnid, colStructList, colValueList, rowIdArray, newColStructList, tableOid, useTmpSuffix, versioning); // @bug 5572 HDFS tmp file + rc = writeColumnRecBinary(txnid, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, useTmpSuffix, versioning); // @bug 5572 HDFS tmp file } return rc; } @@ -4564,6 +4594,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, std::vector& colValueList, RID* rowIdArray, const ColStructList& newColStructList, + std::vector& newColValueList, const int32_t tableOid, bool useTmpSuffix, bool versioning) @@ -4574,7 +4605,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, Column curCol; ColStructList::size_type totalColumn; ColStructList::size_type i; - size_t totalRow; + size_t totalRow1, totalRow2; setTransId(txnid); @@ -4582,11 +4613,21 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, #ifdef PROFILE StopWatch timer; #endif - totalRow = colValueList.size() / totalColumn; - valArray = malloc(sizeof(uint64_t) * totalRow); + totalRow1 = colValueList.size() / totalColumn; + if (newColValueList.size() > 0) + { + totalRow2 = newColValueList.size() / newColStructList.size(); + totalRow1 -= totalRow2; + } + else + { + totalRow2 = 0; + } - if (totalRow == 0) + valArray = malloc(sizeof(uint64_t) * totalRow1); + + if (totalRow1 == 0) return rc; TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); @@ -4634,7 +4675,7 @@ StopWatch timer; if (versioning) { rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], - colStructList[i].colWidth, totalRow, firstPart, rangeList); + colStructList[i].colWidth, totalRow1, firstPart, rangeList); if (rc != NO_ERROR) { if (colStructList[i].fCompressionType == 0) { @@ -4652,9 +4693,9 @@ StopWatch timer; uint8_t tmp8; uint16_t tmp16; uint32_t tmp32; - for (size_t j = 0; j < totalRow; j++) + for (size_t j = 0; j < totalRow1; j++) { - uint64_t curValue = colValueList[(totalRow*i) + j]; + uint64_t curValue = colValueList[((totalRow1 + totalRow2)*i) + j]; switch (colStructList[i].colType) { case WriteEngine::WR_VARBINARY : // treat same as char for now @@ -4692,7 +4733,7 @@ StopWatch timer; #ifdef PROFILE timer.start("writeRow "); #endif - rc = colOp->writeRow(curCol, totalRow, firstPart, valArray); + rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); #ifdef PROFILE timer.stop("writeRow "); #endif @@ -4707,7 +4748,135 @@ timer.stop("writeRow "); } // end of for (i = 0 if (valArray != NULL) + { free(valArray); + valArray = NULL; + } + + // MCOL-1176 - Write second extent + if (totalRow2) + { + valArray = malloc(sizeof(uint64_t) * totalRow2); + for (i = 0; i < newColStructList.size(); i++) + { + //@Bug 2205 Check if all rows go to the new extent + //Write the first batch + RID * secondPart = rowIdArray + totalRow1; + ColumnOp* colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, newColStructList[i].colWidth, + newColStructList[i].colDataType, newColStructList[i].colType, newColStructList[i].dataOid, + newColStructList[i].fCompressionType, newColStructList[i].fColDbRoot, + newColStructList[i].fColPartition, newColStructList[i].fColSegment); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(newColStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) + break; + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = newColStructList[i].fColDbRoot; + aExt.partNum = newColStructList[i].fColPartition; + aExt.segNum = newColStructList[i].fColSegment; + aExt.compType = newColStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); + } + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + if (rc != NO_ERROR) + break; + + // handling versioning + vector rangeList; + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, newColStructList[i], + newColStructList[i].colWidth, totalRow2, secondPart, rangeList); + if (rc != NO_ERROR) { + if (newColStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + //totalRow1 -= totalRow2; + // have to init the size here + // nullArray = (bool*) malloc(sizeof(bool) * totalRow); + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + for (size_t j = 0; j < totalRow2; j++) + { + uint64_t curValue = newColValueList[(totalRow2*i) + j]; + switch (newColStructList[i].colType) + { + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + ((uint64_t*)valArray)[j] = curValue; + break; + case WriteEngine::WR_INT: + case WriteEngine::WR_UINT: + case WriteEngine::WR_FLOAT: + tmp32 = curValue; + ((uint32_t*)valArray)[j] = tmp32; + break; + case WriteEngine::WR_ULONGLONG: + case WriteEngine::WR_LONGLONG: + case WriteEngine::WR_DOUBLE: + case WriteEngine::WR_TOKEN: + ((uint64_t*)valArray)[j] = curValue; + break; + case WriteEngine::WR_BYTE: + case WriteEngine::WR_UBYTE: + tmp8 = curValue; + ((uint8_t*)valArray)[j] = tmp8; + break; + case WriteEngine::WR_SHORT: + case WriteEngine::WR_USHORT: + tmp16 = curValue; + ((uint16_t*)valArray)[j] = tmp16; + break; + } + } + + + #ifdef PROFILE + timer.start("writeRow "); + #endif + rc = colOp->writeRow(curCol, totalRow2, secondPart, valArray); + #ifdef PROFILE + timer.stop("writeRow "); + #endif + colOp->closeColumnFile(curCol); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + // check error + if (rc != NO_ERROR) + break; + + } // end of for (i = 0 + } + if (valArray != NULL) + free(valArray); + #ifdef PROFILE timer.finish(); diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index 7c7862d5f..f254a28cd 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -663,6 +663,7 @@ private: int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, std::vector& colValueList, RID* rowIdArray, const ColStructList& newColStructList, + std::vector& newColValueList, const int32_t tableOid, bool useTmpSuffix, bool versioning = true); From da4c10dc0e2689df9b101a9fd2ddc46440ad73f0 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 30 Jan 2018 13:04:04 -0600 Subject: [PATCH 38/41] MCOL-1034 - fixed ddl/dml running double and dbrm read-only during failover --- procmgr/main.cpp | 70 +++++++++++++--------------------- procmgr/processmanager.cpp | 77 +++++++++++++++++++++++--------------- 2 files changed, 74 insertions(+), 73 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index a5ebb9183..495432bcd 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -2061,37 +2061,36 @@ void pingDeviceThread() //set query system state ready processManager.setQuerySystemState(true); + } - sleep(2); - //check if down module was Standby OAM, if so find another one - if ( moduleName == config.OAMStandbyName() ) { - - //set down module ProcessManager to AOS - processManager.setProcessState(moduleName, "ProcessManager", oam::AUTO_OFFLINE, 0); - - //get another standby OAM module - string newStandbyModule = processManager.getStandbyModule(); - - //send message to start new Standby Process-Manager, if needed - if ( !newStandbyModule.empty() && newStandbyModule != "NONE") { - processManager.setStandbyModule(newStandbyModule); + //check if down module was Standby OAM, if so find another one + if ( moduleName == config.OAMStandbyName() ) { + + //set down module ProcessManager to AOS + processManager.setProcessState(moduleName, "ProcessManager", oam::AUTO_OFFLINE, 0); + + //get another standby OAM module + string newStandbyModule = processManager.getStandbyModule(); + + //send message to start new Standby Process-Manager, if needed + if ( !newStandbyModule.empty() && newStandbyModule != "NONE") { + processManager.setStandbyModule(newStandbyModule); + } + else + { + Config* sysConfig = Config::makeConfig(); + + // clear Standby OAM Module + sysConfig->setConfig("SystemConfig", "StandbyOAMModuleName", oam::UnassignedName); + sysConfig->setConfig("ProcStatusControlStandby", "IPAddr", oam::UnassignedIpAddr); + + //update Calpont Config table + try { + sysConfig->write(); } - else + catch(...) { - Config* sysConfig = Config::makeConfig(); - - // clear Standby OAM Module - sysConfig->setConfig("SystemConfig", "StandbyOAMModuleName", oam::UnassignedName); - sysConfig->setConfig("ProcStatusControlStandby", "IPAddr", oam::UnassignedIpAddr); - - //update Calpont Config table - try { - sysConfig->write(); - } - catch(...) - { - log.writeLog(__LINE__, "ERROR: sysConfig->write", LOG_TYPE_ERROR); - } + log.writeLog(__LINE__, "ERROR: sysConfig->write", LOG_TYPE_ERROR); } } } @@ -2103,21 +2102,6 @@ void pingDeviceThread() //start SIMPLEX runtype processes on a SIMPLEX runtype module string moduletype = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); - if ( MySQLRep == "y" ) { - if ( moduletype == "um" || - ( moduletype == "pm" && config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) || - ( moduletype == "pm" && PMwithUM == "y") ) { - - //setup MySQL Replication for started modules - log.writeLog(__LINE__, "Setup MySQL Replication for module recovering from outage on " + moduleName, LOG_TYPE_DEBUG); - DeviceNetworkList devicenetworklist; - DeviceNetworkConfig devicenetworkconfig; - devicenetworkconfig.DeviceName = moduleName; - devicenetworklist.push_back(devicenetworkconfig); - processManager.setMySQLReplication(devicenetworklist); - } - } - try{ oam.getSystemConfig(moduletype, moduletypeconfig); } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 3eed6727f..339e19dfd 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3431,33 +3431,37 @@ void ProcessManager::recycleProcess(string module) string moduleType = module.substr(0,MAX_MODULE_TYPE_SIZE); - // if a UM module, send a restart on DMLProc/DDLProc to get started on another UM, if needed string PrimaryUMModuleName; try { oam.getSystemConfig("PrimaryUMModuleName", PrimaryUMModuleName); } catch(...) {} - - //restart ExeMgrs/mysql if module is a pm restartProcessType("DBRMControllerNode", module); restartProcessType("DBRMWorkerNode"); + if( PrimaryUMModuleName == module) { stopProcessType("DDLProc"); stopProcessType("DMLProc"); } + stopProcessType("ExeMgr"); + restartProcessType("PrimProc"); sleep(1); + restartProcessType("ExeMgr"); sleep(1); + restartProcessType("mysql"); restartProcessType("WriteEngineServer"); sleep(1); + restartProcessType("DDLProc",module); sleep(1); + restartProcessType("DMLProc",module); return; @@ -4325,7 +4329,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski // if DDL or DMLProc, change IP Address if ( retStatus == oam::API_SUCCESS ) { - sleep(5); +// sleep(5); ProcessStatus procstat; oam.getProcessStatus(processName, systemprocessstatus.processstatus[i].Module, procstat); if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) @@ -9388,20 +9392,6 @@ int ProcessManager::OAMParentModuleChange() sleep(1); } - //set status to BUSY_INIT while failover is in progress - processManager.setSystemState(oam::BUSY_INIT); - - // graceful start snmptrap-daemon - string EnableSNMP = "y"; - try { - oam.getSystemConfig("EnableSNMP", EnableSNMP); - } - catch(...) - {} - - if ( EnableSNMP == "y" ) - startProcess(config.moduleName(), "SNMPTrapDaemon", oam::GRACEFUL); - // set alarm ALARMManager aManager; aManager.sendAlarmReport(config.moduleName().c_str(), MODULE_SWITCH_ACTIVE, SET); @@ -9428,7 +9418,6 @@ int ProcessManager::OAMParentModuleChange() processManager.stopModule(config.moduleName(), oam::FORCEFUL, true); string localModule = config.moduleName(); -// processManager.setModuleState(localModule, oam::AUTO_INIT); pthread_t startmodulethread; int status = pthread_create (&startmodulethread, NULL, (void*(*)(void*)) &startModuleThread, &localModule); @@ -9441,19 +9430,47 @@ int ProcessManager::OAMParentModuleChange() status = startsystemthreadStatus; } + // waiting until dml are ACTIVE + while (true) + { + ProcessStatus DMLprocessstatus; + try { + oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); + } + catch (exception& ex) + {} + catch(...) + {} + + if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) + log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback" , LOG_TYPE_DEBUG); + + if (DMLprocessstatus.ProcessOpState == oam::ACTIVE) + break; + + if (DMLprocessstatus.ProcessOpState == oam::FAILED) + break; + + // wait some more + sleep(2); + } + + //set recycle process + processManager.recycleProcess(downOAMParentName); + //restart/reinit processes to force their release of the controller node port if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) && ( moduleNameList.size() <= 0 && config.moduleType() == "pm") ) { - status = 0; + int status = 0; } else { - processManager.restartProcessType("mysql", localModule); - processManager.restartProcessType("ExeMgr", localModule); - processManager.restartProcessType("WriteEngineServer", localModule); +// processManager.restartProcessType("mysql", localModule); +// processManager.restartProcessType("ExeMgr", localModule); +// processManager.restartProcessType("WriteEngineServer", localModule); - processManager.reinitProcessType("DBRMWorkerNode"); +// processManager.reinitProcessType("DBRMWorkerNode"); //send message to start new Standby Process-Manager, if needed newStandbyModule = getStandbyModule(); @@ -9527,12 +9544,12 @@ int ProcessManager::OAMParentModuleChange() //restart DDLProc/DMLProc to perform any rollbacks, if needed //dont rollback in amazon, wait until down pm recovers - if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) - && !amazon ) { - processManager.restartProcessType("DDLProc", config.moduleName()); - sleep(1); - processManager.restartProcessType("DMLProc", config.moduleName()); - } +// if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) +// && !amazon ) { +// processManager.restartProcessType("DDLProc", config.moduleName()); +// sleep(1); +// processManager.restartProcessType("DMLProc", config.moduleName()); +// } if ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) { From 8c38917423e61310809386b748f61ab4258c9888 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 31 Jan 2018 15:41:55 -0600 Subject: [PATCH 39/41] MCOL-1034 --- procmgr/main.cpp | 25 +++++++++++++++++++++++-- procmgr/processmanager.cpp | 16 +++++++++++++--- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 495432bcd..6fa8fd5a8 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1758,7 +1758,18 @@ void pingDeviceThread() // if LAN OUTAGE ACTIVE,skip module checks if (LANOUTAGEACTIVE) break; - + + //check if down module is PrimaryUMModuleName + bool downPrimaryUM = false; + string PrimaryUMModuleName; + try { + oam.getSystemConfig("PrimaryUMModuleName", PrimaryUMModuleName); + } + catch(...) {} + + if ( PrimaryUMModuleName == moduleName ) + downPrimaryUM = true; + // if not disabled and amazon, skip if (opState != oam::AUTO_DISABLED ) { @@ -2101,7 +2112,17 @@ void pingDeviceThread() //start SIMPLEX runtype processes on a SIMPLEX runtype module string moduletype = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); - + + // reset up mysql rep slaves is master changed + if ( downPrimaryUM && + ( MySQLRep == "y" ) ) + { + //setup MySQL Replication for started modules + log.writeLog(__LINE__, "Setup MySQL Replication for module outage on " + moduleName, LOG_TYPE_DEBUG); + DeviceNetworkList devicenetworklist; + processManager.setMySQLReplication(devicenetworklist); + } + try{ oam.getSystemConfig(moduletype, moduletypeconfig); } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 339e19dfd..c4abe6793 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3437,9 +3437,18 @@ void ProcessManager::recycleProcess(string module) } catch(...) {} + //recycle DBRM processes in all cases restartProcessType("DBRMControllerNode", module); restartProcessType("DBRMWorkerNode"); + // only recycle dmlproc, if down/up module is non-parent UM + if ( ( moduleType == "um" ) && + ( PrimaryUMModuleName != module) ) + { + restartProcessType("DMLProc",module); + return; + } + if( PrimaryUMModuleName == module) { stopProcessType("DDLProc"); @@ -10498,8 +10507,7 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist for( ; listPT != devicenetworklist.end() ; listPT++) { string remoteModuleName = (*listPT).DeviceName; - log.writeLog(__LINE__, "Setup SlavMySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); - + //skip master if ( remoteModuleName == masterModule ) continue; @@ -10516,7 +10524,9 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) continue; - ByteStream msg1; + log.writeLog(__LINE__, "Setup Slave MySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); + + ByteStream msg1; ByteStream::byte requestID = oam::SLAVEREP; if ( !enable ) { requestID = oam::DISABLEREP; From 253c23d359e7f18fdeddd92f661c36d9ce2bb5ac Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Wed, 31 Jan 2018 16:29:39 -0600 Subject: [PATCH 40/41] MCOL-1152: Change the debian package names. --- cpackEngineDEB.cmake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 4a5e24b1f..42c1f6974 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -7,8 +7,6 @@ SET(CMAKE_INSTALL_PREFIX ${INSTALL_ENGINE}) SET(CPACK_GENERATOR "DEB") SET(CPACK_DEBIAN_PACKAGE_DEBUG 1) SET(CPACK_PACKAGING_INSTALL_PREFIX ${INSTALL_ENGINE}) -CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7) - # Note that this variable is DEB not DEBIAN! http://public.kitware.com/pipermail/cmake/2014-July/058030.html SET(CPACK_DEB_COMPONENT_INSTALL ON) @@ -16,7 +14,7 @@ SET(CPACK_DEB_COMPONENT_INSTALL ON) SET(CPACK_COMPONENTS_ALL platform libs storage-engine) SET(CPACK_PACKAGE_NAME "mariadb-columnstore") -SET(ENGINE_ARCH "x86_64") +SET(ENGINE_ARCH "amd64") IF (NOT CPACK_DEBIAN_PACKAGE_VERSION) SET (CPACK_DEBIAN_PACKAGE_VERSION ${PACKAGE_VERSION}) @@ -26,7 +24,7 @@ IF (NOT CPACK_DEBIAN_PACKAGE_RELEASE) ENDIF() SET(CPACK_DEBIAN_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) -SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_DEBIAN_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_RELEASE}-${ENGINE_ARCH}-${DEB}") +SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_DEBIAN_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_RELEASE}-${ENGINE_ARCH}") SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MariaDB Columnstore: a very fast and robust SQL database server") SET(CPACK_PACKAGE_URL "http://mariadb.org") From 01bd50ad09d0a6768fb49be4d3ad6945d243ff27 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 31 Jan 2018 16:33:57 -0600 Subject: [PATCH 41/41] MCOL-1034 --- procmgr/main.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 6fa8fd5a8..01153d11f 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -2106,13 +2106,6 @@ void pingDeviceThread() } } - // if disabled and amazon, break out - if ( (opState == oam::AUTO_DISABLED ) && amazon ) - break; - - //start SIMPLEX runtype processes on a SIMPLEX runtype module - string moduletype = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); - // reset up mysql rep slaves is master changed if ( downPrimaryUM && ( MySQLRep == "y" ) ) @@ -2123,6 +2116,13 @@ void pingDeviceThread() processManager.setMySQLReplication(devicenetworklist); } + // if disabled and amazon, break out + if ( (opState == oam::AUTO_DISABLED ) && amazon ) + break; + + //start SIMPLEX runtype processes on a SIMPLEX runtype module + string moduletype = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); + try{ oam.getSystemConfig(moduletype, moduletypeconfig); }