diff --git a/.gitignore b/.gitignore index ecb4702e8..f8246dc9f 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,5 @@ install_manifest_libs.txt install_manifest_platform.txt install_manifest_storage-engine.txt _CPack_Packages +columnstoreversion.h + diff --git a/CMakeLists.txt b/CMakeLists.txt index c075cc84e..2ff14cad0 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12) + # Avoid warnings in higher versions if("${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" GREATER 2.6) CMAKE_POLICY(VERSION 2.8) @@ -76,8 +77,7 @@ ENDIF("${isSystemDir}" STREQUAL "-1") INCLUDE (configureEngine.cmake) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum @ONLY IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL_ENGINE} COMPONENT platform) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) INCLUDE(bison.cmake) @@ -108,8 +108,8 @@ endif() INCLUDE(check_compiler_flag.cmake) -MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) -MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) +MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) +MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) # enable security hardening features, like most distributions do # in our benchmarks that costs about ~1% of performance, depending on the load diff --git a/build/BuildCalpontVersion.bat b/build/BuildCalpontVersion.bat deleted file mode 100644 index b2e8c62dd..000000000 --- a/build/BuildCalpontVersion.bat +++ /dev/null @@ -1,18 +0,0 @@ -for /f "delims= skip=1" %%x in (releasenum) do (set "%%x") -set packager="%USERNAME% <%USERNAME%@calpont.com>" -echo %packager% -set builddate="%DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2% %TIME:~0,2%:%TIME:~3,2%:%TIME:~6,2%" -set buildmachine=%computername% -copy /Y .\CalpontVersion.txt.in ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@PACKAGER@@/%packager%/ ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@BUILDDATE@@/%builddate%/ ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@VERSION@@/%version%/ ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@RELEASE@@/%release%/ ..\utils\winport\CalpontVersion.txt -sed -i -e s/@@BUILDMACHINE@@/%buildmachine%/ ..\utils\winport\CalpontVersion.txt -:: For some unknown reason, sed removes permissions. Put them back. -icacls ..\utils\winport\CalpontVersion.txt /grant SYSTEM:F administrators:F -cd .. -git status | grep "On branch" >> utils\winport\CalpontVersion.txt -git log -1 | head -3 | egrep "^(commit|Date)" >> utils\winport\CalpontVersion.txt - -cd build diff --git a/build/bootstrap b/build/bootstrap deleted file mode 100755 index e7dfc1f97..000000000 --- a/build/bootstrap +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# $Id: bootstrap 1333 2011-01-18 17:25:26Z rdempsey $ -# -# This script is responsible for setting up a fresh development tree -# following a git checkout. It copies static files and shared include -# files to the export tree prior to building the software. - -set -e - -mkdir -p export/mysql -#FIXME rename this -cp dbcon/mysql/install_calpont_mysql.sh export/mysql/ - -#TODO: see if this my.cnf is necessary -cp dbcon/mysql/my.cnf export/mysql/. - -#TODO: the init.d service file should not be mixed case -#TODO: the init.d service file should not contain calpont -cp dbcon/mysql/mysql-Columnstore export/mysql/ && \ -cp build/files.lst export/mysql/ && \ -cp dbcon/mysql/install_calpont_mysql.sh export/mysql/ && \ -cp dbcon/mysql/my.cnf export/mysql/ && \ -cp dbcon/mysql/mysql-Columnstore export/mysql/ - -#FIXME: -#This is not going to update the version in configure.ac -echo Generating version header -./build/genVersion.sh -if [ $? -ne 0 ]; then - echo Could not generate version header - exit $? -fi - -echo Running GNU autotools to generate build environment -aclocal && \ -autoconf && \ -autoheader && \ -libtoolize && \ -automake --add-missing && \ -autoreconf - -if [ $? -ne 0 ]; then - echo GNU autotools generation failed - exit $? -fi - - -echo GNU autotools environment generation completed successfully - -#cd net-snmp/net-snmp -# ./configure --prefix=$PREFIX --with-mib-modules=disman/event-mib --with-cc=gcc \ -# --with-logfile=$PREFIX/log/snmpd.log \ -# --with-sys-location=Unknown \ -# --with-sys-contact=root@localhost.localdomain \ -# --with-default-snmp-version=3 \ -# --with-persistent-directory=$PREFIX/var/net-snmp \ -# --without-openssl \ -# --with-ldflags="-Wl,-rpath -Wl,$PREFIX/lib" - diff --git a/build/bootstrap.4.5 b/build/bootstrap.4.5 index b8be8a899..ac400b225 100755 --- a/build/bootstrap.4.5 +++ b/build/bootstrap.4.5 @@ -49,7 +49,7 @@ for lib in libnetsnmpagent.so libnetsnmphelpers.so libnetsnmpmibs.so libnetsnmp. done popd >/dev/null -./build/genVersion.sh +#./build/genVersion.sh for x in \ $(find . -name '[mM]akefile' \ diff --git a/build/build_rpms b/build/build_rpms deleted file mode 100755 index 7af7e689b..000000000 --- a/build/build_rpms +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash -# -# $Id: build_rpms 1734 2012-10-25 18:07:16Z dlhill $ -# - -verbose=0 - -prefix=/usr/local/mariadb/columnstore -#mysqldir=$prefix/mariadb-columnstore/mysql -columnstoredir=$prefix - -# set default as community -community=true - -usage() -{ -cat <&2 - fi -done - -#try to find project root -while [ ! -d dbcon ]; do - cd .. - curdir=$(pwd) - if [ $curdir = / -o $curdir = $HOME ]; then - echo "I could not find the project root directory: I can't continue!" - exit 1 - fi -done - -if [ ! -f ./build/releasenum ]; then - echo "I could not find the file 'releasesum' in the build directory: I can't continue!" - exit 1 -fi - -. ./build/releasenum - -# make the install package -buildroot=$HOME/rpm/BUILD/mariadb-columnstore-$version.$release - -sudo rm -rf $buildroot -mkdir -p $buildroot -mkdir -p $buildroot/mariadb/columnstore -echo -n "Writing binary files to $buildroot..." -sudo chown -R builder:builder $columnstoredir -/bin/cp -f -r $columnstoredir/* $buildroot/mariadb/columnstore/. - -/bin/cp -r build/releasenum $buildroot/mariadb/columnstore/. - -#get mysql server files -#pushd $mysqldir > /dev/null -#mkdir -p $buildroot/mariadb/columnstore/mysql -#cpio -pdmu --quiet $buildroot/mariadb/columnstore/mysql < files.lst -#popd >/dev/null - -#clean up some files from buildroot -rm -f $buildroot/mariadb/columnstore/etc/Doxyfile -for file in $buildroot/mariadb/columnstore/lib/*; do - if [ -h $file ]; then - rm -f $file - fi -done - -#temporarily clean up these BRM thingies -for file in dbrm script-fcns; do - rm -f $buildroot/mariadb/columnstore/bin/$file -done - -#clean up these bulkload scripts -for file in bulkload.sh cleanup.sh dbbuilder.sh dbload.sh cpimport.sh bulkloadp.sh; do - rm -f $buildroot/mariadb/columnstore/bin/$file -done - -rm -f $buildroot/mariadb/columnstore/lib/libidbboot.a -rm -f $buildroot/mariadb/columnstore/lib/libiberty.a - -mkdir -p $HOME/rpm/BUILD $HOME/rpm/RPMS $HOME/rpm/SRPMS $HOME/rpm/SPECS $HOME/rpm/tmp - -if [ ! -f $HOME/.rpmmacros ]; then - cat > $HOME/.rpmmacros < /tmp/rpmbuild.out.$$ 2>&1 - -rc=$? -echo done. -if [ $verbose -gt 0 -o $rc -ne 0 ]; then - cat /tmp/rpmbuild.out.$$ -fi - -rm -f /tmp/*.$$ - -exit $rc - diff --git a/build/genVersion.sh b/build/genVersion.sh deleted file mode 100755 index 4d6c2eefd..000000000 --- a/build/genVersion.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# $Id: genVersion.sh 1019 2009-12-18 14:42:35Z rdempsey $ -# - -#prefix=export -#for arg in "$@"; do -# if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then -# prefix="`echo $arg | awk -F= '{print $2}'`" -# else -# echo "ignoring unknown argument: $arg" 1>&2 -# fi -#done -# -##try to find project root -#while [ ! -d dbcon ]; do -# cd .. -# curdir=$(pwd) -# if [ $curdir = / -o $curdir = $HOME ]; then -# echo "I could not find the project root directory: I can't continue!" -# exit 1 -# fi -#done -# -#if [ ! -f ./build/releasenum ]; then -# echo "I could not find the file 'releasesum' in the build directory: I can't continue!" -# exit 1 -#fi - -. ./build/releasenum - -#mkdir -p ${prefix}/include - -echo " -#ifndef VERSIONNUMBER_H_ -#define VERSIONNUMBER_H_ -#include -const std::string idb_version(\"$version\"); -const std::string idb_release(\"$release\"); -#endif -" > dbcon/mysql/versionnumber.h.tmp - -diff -bBq dbcon/mysql/versionnumber.h.tmp dbcon/mysql/versionnumber.h >/dev/null 2>&1 -if [ $? -ne 0 ]; then - cp dbcon/mysql/versionnumber.h.tmp dbcon/mysql/versionnumber.h -fi - -rm -f dbcon/mysql/versionnumber.h.tmp - diff --git a/build/make_src_tar b/build/make_src_tar deleted file mode 100755 index 098d75ff7..000000000 --- a/build/make_src_tar +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash -# -# $Id: make_src_tar 1932 2013-06-20 18:26:27Z dhill $ -# - -# basedir is the working dir for all the activity -basedir=$HOME -# vname is the name of the top-level dir in the tar file -vname= -# ofile is the output tar file name and is relative to a hidden dir, so it should be absolute -ofile= -# branch is the branch to checkout (develop by default) -branch= -for arg in "$@"; do - if [ $(expr -- "$arg" : '--basedir=') -eq 10 ]; then - basedir="$(echo $arg | awk -F= '{print $2}')" - elif [ $(expr -- "$arg" : '--vname=') -eq 8 ]; then - vname="$(echo $arg | awk -F= '{print $2}')" - elif [ $(expr -- "$arg" : '--ofile=') -eq 8 ]; then - ofile="$(echo $arg | awk -F= '{print $2}')" - elif [ $(expr -- "$arg" : '--branch=') -eq 9 ]; then - branch="$(echo $arg | awk -F= '{print $2}')" - else - echo "ignoring unknown argument: $arg" 1>&2 - fi -done - -if [ -f releasenum ]; then - . releasenum -elif [ -f build/releasenum ]; then - . build/releasenum -else - echo "Couldn't locate 'releasenum' file in the build dir, make sure running in top-level dir" -fi - -if [ -z "$version" ]; then - echo "Version defaulting to 4.0-develop" - version=4.0 - release=develop -fi - -if [ -z "$release" ]; then - release=0 -fi - -blddir=bld.$$ -if [ -z "$vname" ]; then - vname=infinidb-${version} -fi -if [ -z "$ofile" ]; then - ofile=${basedir}/${vname}-${release}.src.tar.gz -fi -co_dir=$blddir/$vname - -trap "{ echo; cd ${basedir}; rm -rf $blddir; exit 1; }" TERM INT ERR - -cd ${basedir} -rm -rf $blddir -mkdir -p $co_dir - -cd $co_dir - -branch_arg= -if [ ! -z "$branch" ]; then - branch_arg="-b $branch" -else - branch=develop -fi - -echo -n "getting git source from $branch..." -git clone http://srvengcm1.calpont.com/repos/infinidb.git infinidb -cd infinidb -git checkout $branch -echo "done." - -test -d build || exit 1 - -echo -n "cleaning git source for " -echo -n "release..." -rm -rf .git .gitignore -rm -rf qa export -rm -f rules.mak -find . -name Makefile | xargs rm -f -find . -name \*.xmi | xargs rm -f -rm -f dbcon/joblist/bandeddl.h.set dbcon/joblist/bandeddl.h.vector primitives/blockcache/iomanager.cpp.single-read \ - primitives/udf/Makefile.sdk -rm -rf utils/scenarios -cd mysql -rm -rf scripts queries storage/maria -cd .. -rm -rf writeengine/test/* -rm -rf utils/net-snmp-built-libs -rm -rf utils/multicast -rm -rf utils/openpgm -rm -rf tools/reserveStacks -rm -rf dbcon/doc dbcon/ingres dbcon/plsql -cd tools -rm -rf brmtest dataPump evalidx fixdate nettest pgm pingproc rebuildEM snconvert updateGenii valprint -rm -rf cfread dbgen dbgen-ssb ddldriver dmldriver idb_comp sendPlan vbgen -cd .. -rm -rf utils/libstatgrab utils/libxml utils/testbc utils/configcpp/md5 -echo "done." - -cd .. - -echo -n "getting MySQL..." -git clone https://github.com/infinidb/mysql mysql -cd mysql -git checkout $branch -echo "done." - -cd .. - -echo -n "building tarball..." -cd infinidb -cp -r utils/autoconf/* . -cp INSTALL INSTALL.keep -cp install-sh install-sh.keep -rm -rf utils/autoconf -autoreconf --force --install -rm -f INSTALL install-sh -mv INSTALL.keep INSTALL -mv install-sh.keep install-sh -cd .. -ln -s infinidb/INSTALL -ln -s infinidb/README -cd .. -tar -zcf $ofile $vname -echo "done." - -echo "tarball left in $ofile" - -cd ${basedir} -rm -rf $blddir - diff --git a/build/releasenum.in b/build/releasenum.in index 1f5e4b4dc..7461afcf3 100644 --- a/build/releasenum.in +++ b/build/releasenum.in @@ -1,2 +1,2 @@ -version=@PACKAGE_VERSION@ -release=@PACKAGE_RELEASE@ \ No newline at end of file +version=${PACKAGE_VERSION} +release=${PACKAGE_RELEASE} diff --git a/columnstoreversion.h.in b/columnstoreversion.h.in new file mode 100644 index 000000000..ec642aa92 --- /dev/null +++ b/columnstoreversion.h.in @@ -0,0 +1,7 @@ +#ifndef VERSIONNUMBER_H_ +#define VERSIONNUMBER_H_ +#include +const std::string columnstore_version("${VERSION}"); +const std::string columnstore_release("${PACKAGE_RELEASE}"); +#endif + diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 461714ced..42c1f6974 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -1,12 +1,12 @@ IF(DEB) +CMAKE_MINIMUM_REQUIRED(VERSION 3.4) + SET(CMAKE_INSTALL_PREFIX ${INSTALL_ENGINE}) SET(CPACK_GENERATOR "DEB") SET(CPACK_DEBIAN_PACKAGE_DEBUG 1) SET(CPACK_PACKAGING_INSTALL_PREFIX ${INSTALL_ENGINE}) -CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7) - # Note that this variable is DEB not DEBIAN! http://public.kitware.com/pipermail/cmake/2014-July/058030.html SET(CPACK_DEB_COMPONENT_INSTALL ON) @@ -14,7 +14,7 @@ SET(CPACK_DEB_COMPONENT_INSTALL ON) SET(CPACK_COMPONENTS_ALL platform libs storage-engine) SET(CPACK_PACKAGE_NAME "mariadb-columnstore") -SET(ENGINE_ARCH "x86_64") +SET(ENGINE_ARCH "amd64") IF (NOT CPACK_DEBIAN_PACKAGE_VERSION) SET (CPACK_DEBIAN_PACKAGE_VERSION ${PACKAGE_VERSION}) @@ -24,7 +24,7 @@ IF (NOT CPACK_DEBIAN_PACKAGE_RELEASE) ENDIF() SET(CPACK_DEBIAN_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) -SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_DEBIAN_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_RELEASE}-${ENGINE_ARCH}-${DEB}") +SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_DEBIAN_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_RELEASE}-${ENGINE_ARCH}") SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MariaDB Columnstore: a very fast and robust SQL database server") SET(CPACK_PACKAGE_URL "http://mariadb.org") @@ -65,9 +65,11 @@ if (EXISTS "/etc/debian_version") set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") endif () if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs, libsnappy1") -else () - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs, libsnappy1v5") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1") +elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5") +else() + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, snappy, net-tools") endif () SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index a6013b280..5c340adfc 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -5,7 +5,6 @@ SET(CMAKE_INSTALL_PREFIX ${INSTALL_ENGINE}) SET(CPACK_GENERATOR "RPM") SET(CPACK_RPM_PACKAGE_DEBUG 1) SET(CPACK_PACKAGING_INSTALL_PREFIX ${INSTALL_ENGINE}) -CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7) SET(CPACK_RPM_COMPONENT_INSTALL ON) @@ -189,7 +188,6 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/etc/MessageFile.txt" "/usr/local/mariadb/columnstore/etc/ErrorMessage.txt" "/usr/local/mariadb/columnstore/local/module" -"/usr/local/mariadb/columnstore/releasenum" "/usr/local/mariadb/columnstore/bin/rollback" "/usr/local/mariadb/columnstore/bin/editem" "/usr/local/mariadb/columnstore/bin/getConfig" diff --git a/dbcon/joblist/resourcemanager.h b/dbcon/joblist/resourcemanager.h index 80619d7d1..5d7a7e873 100644 --- a/dbcon/joblist/resourcemanager.h +++ b/dbcon/joblist/resourcemanager.h @@ -94,7 +94,6 @@ const uint64_t defaultNumBuckets = 128; const uint64_t defaultMaxElementsPerBuckert = 16 * 1024 * 1024; const int defaultEMServerThreads = 50; -const int defaultEMServerQueueSize = 100; const int defaultEMSecondsBetweenMemChecks = 1; const int defaultEMMaxPct = 95; const int defaultEMPriority = 21; // @Bug 3385 @@ -152,27 +151,28 @@ public: typedef std::map MemMap; + // @MCOL-513 - Added threadpool to ExeMgr + int getEmServerThreads() const + { + return getIntVal(fExeMgrStr, "ThreadPoolSize", defaultEMServerThreads); + } + std::string getExeMgrThreadPoolDebug() const + { + return getStringVal(fExeMgrStr, "ThreadPoolDebug", "N"); + } - int getEmServerThreads() const - { - return getUintVal(fExeMgrStr, "ServerThreads", defaultEMServerThreads); - } - int getEmServerQueueSize() const - { - return getUintVal(fExeMgrStr, "ServerQueueSize", defaultEMServerQueueSize); - } - int getEmSecondsBetweenMemChecks() const + int getEmSecondsBetweenMemChecks() const { return getUintVal(fExeMgrStr, "SecondsBetweenMemChecks", defaultEMSecondsBetweenMemChecks); } - int getEmMaxPct() const + int getEmMaxPct() const { return getUintVal(fExeMgrStr, "MaxPct", defaultEMMaxPct); } - EXPORT int getEmPriority() const; - int getEmExecQueueSize() const + EXPORT int getEmPriority() const; + int getEmExecQueueSize() const { - return getUintVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); + return getIntVal(fExeMgrStr, "ExecQueueSize", defaultEMExecQueueSize); } int getHjMaxBuckets() const @@ -218,14 +218,18 @@ public: } // @MCOL-513 - Added threadpool to JobSteps - uint32_t getJLThreadPoolSize() const + int getJLThreadPoolSize() const { - return getUintVal(fJobListStr, "ThreadPoolSize", defaultJLThreadPoolSize); + return getIntVal(fJobListStr, "ThreadPoolSize", defaultJLThreadPoolSize); } std::string getJlThreadPoolDebug() const { return getStringVal(fJobListStr, "ThreadPoolDebug", "N"); } + std::string getDMLJlThreadPoolDebug() const + { + return getStringVal(fJobListStr, "DMLThreadPoolDebug", "N"); + } // @bug 1264 - Added LogicalBlocksPerScan configurable which determines the number of blocks contained in each BPS scan request. uint32_t getJlLogicalBlocksPerScan() const diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 9b7c6c5b6..1a1b0b5ec 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -108,7 +108,7 @@ */ #include "ha_calpont.h" -#include "versionnumber.h" +#include "columnstoreversion.h" #define NEED_CALPONT_EXTERNS #include "ha_calpont_impl.h" @@ -201,7 +201,7 @@ static int columnstore_init_func(void* p) tm.tm_year % 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", idb_version.c_str(), idb_release.c_str()); + fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(), columnstore_release.c_str()); calpont_hton = (handlerton*)p; #ifndef _MSC_VER @@ -234,7 +234,7 @@ static int infinidb_init_func(void* p) tm.tm_year % 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - fprintf(stderr, "InfiniDB: Started; Version: %s-%s\n", idb_version.c_str(), idb_release.c_str()); + fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(), columnstore_release.c_str()); calpont_hton = (handlerton*)p; diff --git a/dbcon/mysql/ha_calpont.h b/dbcon/mysql/ha_calpont.h index ebd7d120a..6abaf51ed 100644 --- a/dbcon/mysql/ha_calpont.h +++ b/dbcon/mysql/ha_calpont.h @@ -239,7 +239,10 @@ public: THR_LOCK_DATA** store_lock(THD* thd, THR_LOCK_DATA** to, enum thr_lock_type lock_type); ///< required const COND* cond_push(const COND* cond); - uint8 table_cache_type() { return HA_CACHE_TBL_NOCACHE; } + uint8 table_cache_type() + { + return HA_CACHE_TBL_NOCACHE; + } }; #endif //HA_CALPONT_H__ diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 29a1b7188..652314097 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -141,7 +141,7 @@ using namespace logging; using namespace funcexp; #include "installdir.h" -#include "versionnumber.h" +#include "columnstoreversion.h" namespace cal_impl_if { @@ -1980,6 +1980,134 @@ extern "C" { } +#ifdef _MSC_VER + __declspec(dllexport) +#endif +// Return 1 if system is ready for reads or 0 if not. + long long mcssystemready(UDF_INIT* initid, UDF_ARGS* args, + char* is_null, char* error) + { + long long rtn = 0; + Oam oam; + DBRM dbrm(true); + SystemStatus systemstatus; + + try + { + oam.getSystemStatus(systemstatus); + + if (systemstatus.SystemOpState == ACTIVE + && dbrm.getSystemReady() + && dbrm.getSystemQueryReady()) + { + return 1; + } + } + catch (...) + { + *error = 1; + } + + return rtn; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssystemready_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssystemready_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif +// Return 1 if system is read only; 0 if writeable + long long mcssystemreadonly(UDF_INIT* initid, UDF_ARGS* args, + char* is_null, char* error) + { + long long rtn = 0; + DBRM dbrm(true); + + try + { + if (dbrm.isReadWrite()) // Returns 0 for writable, 5 for read only + { + rtn = 1; + } + } + catch (...) + { + *error = 1; + rtn = 1; + } + + return rtn; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcssystemreadonly_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcssystemreadonly_deinit(UDF_INIT* initid) + { + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif +// Return 1 if system is read only; 0 if writeable + long long mcswritessuspended(UDF_INIT* initid, UDF_ARGS* args, + char* is_null, char* error) + { + long long rtn = 0; + DBRM dbrm(true); + + try + { + if (dbrm.getSystemSuspended()) + { + rtn = 1; + } + } + catch (...) + { + *error = 1; + rtn = 1; + } + + return rtn; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + my_bool mcswritessuspended_init(UDF_INIT* initid, UDF_ARGS* args, char* message) + { + return 0; + } + +#ifdef _MSC_VER + __declspec(dllexport) +#endif + void mcswritessuspended_deinit(UDF_INIT* initid) + { + } + #define MAXSTRINGLENGTH 50 const char* PmSmallSideMaxMemory = "pmmaxmemorysmallside"; @@ -2472,7 +2600,7 @@ extern "C" char* result, unsigned long* length, char* is_null, char* error) { - string version(idb_version); + string version(columnstore_version); *length = version.size(); memcpy(result, version.c_str(), *length); return result; @@ -4291,7 +4419,7 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) if ( ( ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || (thd->lex)->sql_command == SQLCOM_INSERT_SELECT) && !ci->singleInsert ) { - //@Bug 2438. Only load dta infile calls last batch process + //@Bug 2438. Only load data infile calls last batch process /* if ( ci->isLoaddataInfile && ((thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) || (ci->useCpimport == 0))) { //@Bug 2829 Handle ctrl-C if ( thd->killed > 0 ) diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index aaa17473a..17c6e1817 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -82,6 +82,9 @@ CREATE FUNCTION idbextentmin RETURNS STRING soname 'libcalmysql.so'; CREATE FUNCTION idbextentmax RETURNS STRING soname 'libcalmysql.so'; CREATE FUNCTION idbpartition RETURNS STRING soname 'libcalmysql.so'; CREATE FUNCTION idblocalpm RETURNS INTEGER soname 'libcalmysql.so'; +CREATE FUNCTION mcssystemready RETURNS INTEGER soname 'libcalmysql.so'; +CREATE FUNCTION mcssystemreadonly RETURNS INTEGER soname 'libcalmysql.so'; +CREATE FUNCTION mcswritessuspended RETURNS INTEGER soname 'libcalmysql.so'; CREATE DATABASE IF NOT EXISTS infinidb_vtable; CREATE DATABASE IF NOT EXISTS infinidb_querystats; diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index fbb7d407d..740075165 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -129,6 +129,17 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) continue; } + try + { + oam_instance.getDbrootPmConfig(iter->dbRoot, pmId); + } + catch (std::runtime_error) + { + // MCOL-1116: If we are here a DBRoot is offline/missing + iter++; + continue; + } + table->field[0]->store(oid); table->field[1]->store(iter->segmentNum); table->field[2]->store(iter->partitionNum); @@ -139,7 +150,7 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) std::string DbRootPath = config->getConfig("SystemConfig", DbRootName.str()); fileSize = compressedFileSize = 0; snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", DbRootPath.c_str(), oidDirName); - oam_instance.getDbrootPmConfig(iter->dbRoot, pmId); + std::ostringstream oss; oss << "pm" << pmId << "_WriteEngineServer"; std::string client = oss.str(); diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore index b49c646e2..02a0de162 100755 --- a/dbcon/mysql/mysql-Columnstore +++ b/dbcon/mysql/mysql-Columnstore @@ -61,7 +61,7 @@ datadir=$basedir/db # Value here is overriden by value in my.cnf. # 0 means don't wait at all # Negative numbers mean to wait indefinitely -service_startup_timeout=60 +service_startup_timeout=90 # Lock directory for RedHat / SuSE. lockdir='/var/lock/subsys' diff --git a/ddlproc/CMakeLists.txt b/ddlproc/CMakeLists.txt index cd089d170..222313a69 100644 --- a/ddlproc/CMakeLists.txt +++ b/ddlproc/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp) +set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp) add_executable(DDLProc ${DDLProc_SRCS}) diff --git a/ddlproc/ddlproc.cpp b/ddlproc/ddlproc.cpp index 15cafacbc..03cbe08cd 100644 --- a/ddlproc/ddlproc.cpp +++ b/ddlproc/ddlproc.cpp @@ -61,6 +61,7 @@ using namespace execplan; #include "IDBPolicy.h" #include "utils_utf8.h" +#include "crashtrace.h" namespace fs = boost::filesystem; @@ -100,6 +101,9 @@ int main(int argc, char* argv[]) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("DDLProc"); + setupCwd(); WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_DDLPROC ); @@ -119,6 +123,11 @@ int main(int argc, char* argv[]) sigaction(SIGHUP, &ign, 0); ign.sa_handler = SIG_IGN; sigaction(SIGPIPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif ddlprocessor::DDLProcessor ddlprocessor(1, 20); diff --git a/dmlproc/CMakeLists.txt b/dmlproc/CMakeLists.txt index 36f06657c..fece5b614 100644 --- a/dmlproc/CMakeLists.txt +++ b/dmlproc/CMakeLists.txt @@ -8,7 +8,8 @@ set(DMLProc_SRCS dmlproc.cpp dmlprocessor.cpp dmlresultbuffer.cpp - batchinsertprocessor.cpp) + batchinsertprocessor.cpp + ../utils/common/crashtrace.cpp) add_executable(DMLProc ${DMLProc_SRCS}) diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index 807d66a0c..64282d3ca 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -82,6 +82,8 @@ using namespace joblist; #include "utils_utf8.h" +#include "crashtrace.h" + namespace fs = boost::filesystem; namespace @@ -513,6 +515,9 @@ int main(int argc, char* argv[]) //BUG 5362 systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("DMLProc"); + Config* cf = Config::makeConfig(); setupCwd(); @@ -616,11 +621,11 @@ int main(int argc, char* argv[]) JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); JobStep::jobstepThreadPool.setName("DMLProcJobList"); -// if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") -// { -// JobStep::jobstepThreadPool.setDebug(true); -// JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); -// } + if (rm->getDMLJlThreadPoolDebug() == "Y" || rm->getDMLJlThreadPoolDebug() == "y") + { + JobStep::jobstepThreadPool.setDebug(true); + JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); + } //set ACTIVE state try @@ -641,6 +646,12 @@ int main(int argc, char* argv[]) sigaction(SIGHUP, &ign, 0); ign.sa_handler = SIG_IGN; sigaction(SIGPIPE, &ign, 0); + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif dmlserver.start(); diff --git a/exemgr/CMakeLists.txt b/exemgr/CMakeLists.txt index 929ec804c..cae1cf3ce 100644 --- a/exemgr/CMakeLists.txt +++ b/exemgr/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp) +set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) add_executable(ExeMgr ${ExeMgr_SRCS}) diff --git a/exemgr/femsghandler.cpp b/exemgr/femsghandler.cpp index dc43a27c2..10d1c772a 100644 --- a/exemgr/femsghandler.cpp +++ b/exemgr/femsghandler.cpp @@ -24,7 +24,7 @@ using namespace std; using namespace joblist; using namespace messageqcpp; -threadpool::ThreadPool FEMsgHandler::threadPool(50, 100); +threadpool::ThreadPool FEMsgHandler::threadPool; namespace { diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 60cb5c0ae..a4947ba41 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -98,6 +98,7 @@ using namespace querytele; #include "boost/filesystem.hpp" #include "threadpool.h" +#include "crashtrace.h" namespace { @@ -1538,20 +1539,19 @@ int main(int argc, char* argv[]) // creation, so it has no idea which way to set the flag. So we set the max here. JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize()); JobStep::jobstepThreadPool.setName("ExeMgrJobList"); -// if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") -// { -// JobStep::jobstepThreadPool.setDebug(true); -// JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); -// } + + if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y") + { + JobStep::jobstepThreadPool.setDebug(true); + JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); + } int serverThreads = rm->getEmServerThreads(); - int serverQueueSize = rm->getEmServerQueueSize(); int maxPct = rm->getEmMaxPct(); int pauseSeconds = rm->getEmSecondsBetweenMemChecks(); int priority = rm->getEmPriority(); FEMsgHandler::threadPool.setMaxThreads(serverThreads); - FEMsgHandler::threadPool.setQueueSize(serverQueueSize); FEMsgHandler::threadPool.setName("FEMsgHandler"); if (maxPct > 0) @@ -1574,8 +1574,8 @@ int main(int argc, char* argv[]) } } - cout << "Starting ExeMgr: st = " << serverThreads << ", sq = " << - serverQueueSize << ", qs = " << rm->getEmExecQueueSize() << ", mx = " << maxPct << ", cf = " << + cout << "Starting ExeMgr: st = " << serverThreads << + ", qs = " << rm->getEmExecQueueSize() << ", mx = " << maxPct << ", cf = " << rm->getConfig()->configFile() << endl; //set ACTIVE state @@ -1591,9 +1591,15 @@ int main(int argc, char* argv[]) } } - threadpool::ThreadPool exeMgrThreadPool(serverThreads, serverQueueSize); + threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); exeMgrThreadPool.setName("ExeMgrServer"); + if (rm->getExeMgrThreadPoolDebug() == "Y" || rm->getExeMgrThreadPoolDebug() == "y") + { + exeMgrThreadPool.setDebug(true); + exeMgrThreadPool.invoke(ThreadPoolMonitor(&exeMgrThreadPool)); + } + for (;;) { IOSocket ios; diff --git a/oam/install_scripts/binary_installer.sh b/oam/install_scripts/binary_installer.sh index 9baa26e4a..59a35dd8b 100644 --- a/oam/install_scripts/binary_installer.sh +++ b/oam/install_scripts/binary_installer.sh @@ -215,7 +215,7 @@ send_user "\n" send_user "Start ColumnStore service " send_user " \n" send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/columnstore restart'\n" -set timeout 60 +set timeout 120 # check return expect { "word: " { send "$PASSWORD\n" diff --git a/oam/install_scripts/disable-rep-columnstore.sh b/oam/install_scripts/disable-rep-columnstore.sh index 0a8c9b17f..1c730da4c 100644 --- a/oam/install_scripts/disable-rep-columnstore.sh +++ b/oam/install_scripts/disable-rep-columnstore.sh @@ -56,7 +56,7 @@ checkForError # # Run reset slave command # -echo "Run start slave command" >>/tmp/disable-rep-status.log +echo "Run reset slave command" >>/tmp/disable-rep-status.log cat >/tmp/idb_disable-rep.sql < /tmp/error.check + if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then + echo "ERROR - PASSWORD: check log file: /tmp/mariadb-command-line.log" + rm -f /tmp/error.check + exit 2 + fi + grep ERROR /tmp/mariadb-command-line.log > /tmp/error.check if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then echo "ERROR: check log file: /tmp/mariadb-command-line.log" diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index bd3d3fb55..fc60e32e7 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -190,12 +190,13 @@ if [ $user = "root" ]; then else $SUDO chmod 777 $RCFILE printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILEl > /dev/null 2>&1 + + if [ -n "$systemctl" ]; then + $SUDO systemctl start rc-local >/dev/null 2>&1 + $SUDO systemctl enable rc-local >/dev/null 2>&1 + fi fi -if [ -n "$systemctl" ]; then - $SUDO systemctl restart rc-local >/dev/null 2>&1 - $SUDO systemctl enable rc-local >/dev/null 2>&1 -fi #setup the columnstore service script rm -f /etc/init.d/columnstore >/dev/null 2>&1 diff --git a/oam/install_scripts/remote_command.sh b/oam/install_scripts/remote_command.sh index 9d0f7c2ed..75095632e 100755 --- a/oam/install_scripts/remote_command.sh +++ b/oam/install_scripts/remote_command.sh @@ -10,7 +10,7 @@ # Argument 5 - Remote user name (optional) # Argument 6 - Force a tty to be allocated (optional) set stty_init {cols 512 -opost}; -set timeout 10 +set timeout 30 set SERVER [lindex $argv 0] set PASSWORD [lindex $argv 1] set COMMAND [lindex $argv 2] diff --git a/oam/install_scripts/softwareReport.sh b/oam/install_scripts/softwareReport.sh index ee65f7332..09fcb4db6 100755 --- a/oam/install_scripts/softwareReport.sh +++ b/oam/install_scripts/softwareReport.sh @@ -36,15 +36,6 @@ echo "################# mcsadmin getcolumnstoresoftwareinfo #################" echo " " $INSTALLDIR/bin/mcsadmin getsoftwareinfo -if [ -e $INSTALLDIR/releasenum ];then - echo " " - echo "-- Columnstore Release Number file --" - echo " " - echo "################# cat $INSTALLDIR/releasenum #################" - echo " " - cat $INSTALLDIR/releasenum -fi - echo " " echo "-- Columnstore Storage Configuration --" echo " " diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index b6e9c72ab..91d82eeb1 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -46,6 +46,7 @@ #include #include +#include "columnstoreversion.h" #include "ddlpkg.h" #include "../../dbcon/dmlpackage/dmlpkg.h" #define LIBOAM_DLLEXPORT @@ -154,84 +155,13 @@ Oam::~Oam() /******************************************************************** * * get System Software information - * + * (for backward compatibility only) ********************************************************************/ void Oam::getSystemSoftware(SystemSoftware& systemsoftware) { - // parse releasenum file - - string rn = InstallDir + "/releasenum"; - ifstream File(rn.c_str()); - - if (!File) - // Open File error - return; - - char line[400]; - string buf; - - while (File.getline(line, 400)) - { - buf = line; - - for ( unsigned int i = 0;; i++) - { - if ( SoftwareData[i] == "") - //end of list - break; - - string data = ""; - string::size_type pos = buf.find(SoftwareData[i], 0); - - if (pos != string::npos) - { - string::size_type pos1 = buf.find("=", pos); - - if (pos1 != string::npos) - { - data = buf.substr(pos1 + 1, 80); - } - else - // parse error - exceptionControl("getSystemSoftware", API_FAILURE); - - //strip off any leading or trailing spaces - for (;;) - { - string::size_type pos = data.find(' ', 0); - - if (pos == string::npos) - // no more found - break; - - // strip leading - if (pos == 0) - { - data = data.substr(pos + 1, 10000); - } - else - { - // strip trailing - data = data.substr(0, pos); - } - } - - switch (i) - { - case (0): // line up with SoftwareData[] - systemsoftware.Version = data; - break; - - case (1): - systemsoftware.Release = data; - break; - } - } - } //end of for loop - } //end of while - - File.close(); + systemsoftware.Version = columnstore_version; + systemsoftware.Release = columnstore_release; } /******************************************************************** * @@ -2953,6 +2883,8 @@ oamModuleInfo_t Oam::getModuleInfo() // Get Server Type Install ID serverTypeInstall = atoi(sysConfig->getConfig("Installation", "ServerTypeInstall").c_str()); + + sysConfig; } catch (...) {} diff --git a/oamapps/calpontConsole/calpontConsole.cpp b/oamapps/calpontConsole/calpontConsole.cpp index 11ddbc4f8..ac4d06b5b 100644 --- a/oamapps/calpontConsole/calpontConsole.cpp +++ b/oamapps/calpontConsole/calpontConsole.cpp @@ -25,6 +25,7 @@ #include extern int h_errno; +#include "columnstoreversion.h" #include "calpontConsole.h" #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" @@ -785,7 +786,6 @@ int processCommand(string* arguments) case 4: // getSystemConfig { - SystemSoftware systemsoftware; SystemConfig systemconfig; string returnValue; @@ -796,14 +796,13 @@ int processCommand(string* arguments) try { - oam.getSystemSoftware(systemsoftware); oam.getSystemConfig(systemconfig); cout << endl << "System Configuration" << endl << endl; cout << "SystemName = " << systemconfig.SystemName << endl; - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; cout << "ParentOAMModuleName = " << systemconfig.ParentOAMModule << endl; cout << "StandbyOAMModuleName = " << systemconfig.StandbyOAMModule << endl; @@ -5383,21 +5382,15 @@ int processCommand(string* arguments) system("cat /tmp/calpont.txt"); else { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; } } } else { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; } cout << endl; diff --git a/oamapps/calpontSupport/softwareReport.sh b/oamapps/calpontSupport/softwareReport.sh deleted file mode 100755 index 81c57c715..000000000 --- a/oamapps/calpontSupport/softwareReport.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# -# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - INSTALLDIR=$2 -else - INSTALLDIR="/usr/local/mariadb/columnstore" -fi - -if [ $USER = "root" ]; then - SUDO=" " -else - SUDO="sudo" -fi - -$SUDO rm -f /tmp/${MODULE}_softwareReport.txt - -{ -echo " " -echo "******************** Software Report for ${MODULE} ********************" -echo " " - -echo " " -echo "-- Calpont Package Details --" -echo " " -echo "################# mcsadmin getcalpontsoftwareinfo #################" -echo " " -$INSTALLDIR/bin/mcsadmin getcalpontsoftwareinfo - -echo " " -echo "-- Calpont Release Number file --" -echo " " -echo "################# cat $INSTALLDIR/releasenum #################" -echo " " -cat $INSTALLDIR/releasenum - -echo " " -echo "-- Calpont Storage Configuration --" -echo " " -echo "################# mcsadmin getStorageConfig #################" -echo " " -$INSTALLDIR/bin/mcsadmin getStorageConfig - -} > /tmp/${MODULE}_softwareReport.txt - -exit 0 \ No newline at end of file diff --git a/oamapps/columnstoreSupport/softwareReport.sh b/oamapps/columnstoreSupport/softwareReport.sh index cb1468813..b7a6e90b0 100755 --- a/oamapps/columnstoreSupport/softwareReport.sh +++ b/oamapps/columnstoreSupport/softwareReport.sh @@ -33,14 +33,7 @@ echo "-- Columnstore Package Details --" echo " " echo "################# mcsadmin getcolumnstoresoftwareinfo #################" echo " " -$INSTALLDIR/bin/mcsadmin getcolumnstoresoftwareinfo - -echo " " -echo "-- Columnstore Release Number file --" -echo " " -echo "################# cat $INSTALLDIR/releasenum #################" -echo " " -cat $INSTALLDIR/releasenum +$INSTALLDIR/bin/mcsadmin getsoftwareinfo echo " " echo "-- Columnstore Storage Configuration --" diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 183b722f6..e49483c2b 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -26,6 +26,7 @@ #include extern int h_errno; +#include "columnstoreversion.h" #include "mcsadmin.h" #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" @@ -2099,7 +2100,7 @@ int processCommand(string* arguments) } string DataRedundancyConfig; - string DataRedundancyCopies; + int DataRedundancyCopies; string DataRedundancyStorageType; try @@ -3963,7 +3964,7 @@ int processCommand(string* arguments) } string DataRedundancyConfig; - string DataRedundancyCopies; + int DataRedundancyCopies; string DataRedundancyStorageType; try @@ -5625,21 +5626,15 @@ int processCommand(string* arguments) system("cat /tmp/columnstore.txt"); else { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; } } } else { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "SoftwareVersion = " << systemsoftware.Version << endl; - cout << "SoftwareRelease = " << systemsoftware.Release << endl; + cout << "SoftwareVersion = " << columnstore_version << endl; + cout << "SoftwareRelease = " << columnstore_release << endl; } cout << endl; @@ -6194,7 +6189,7 @@ int processCommand(string* arguments) } string command = startup::StartUp::installDir() + "/bin/remote_command.sh " + (*hostConfigIter).IPAddr + " " + password + " 'mkdir -p " + startup::StartUp::installDir() + "/gluster/brick" + oam.itoa(brickID) + "'"; - int status = system(command.c_str()); + system(command.c_str()); brickID++; } } @@ -6535,7 +6530,7 @@ int processCommand(string* arguments) } } - if ( DataRedundancyConfig == "y" && devicenetworklist.size() != DataRedundancyCopies) + if ( DataRedundancyConfig == "y" && devicenetworklist.size() != (size_t)DataRedundancyCopies) { cout << endl << "**** removeModule Failed : Data Redundancy requires you to remove modules in groups equal to number of copies" << endl; quit = true; @@ -7608,7 +7603,6 @@ int processCommand(string* arguments) { string DataRedundancyConfig = "n"; - int DataRedundancyCopies; try { diff --git a/oamapps/postConfigure/amazonInstaller.cpp b/oamapps/postConfigure/amazonInstaller.cpp index 4059877d4..ac46261c5 100644 --- a/oamapps/postConfigure/amazonInstaller.cpp +++ b/oamapps/postConfigure/amazonInstaller.cpp @@ -46,6 +46,7 @@ #include #include +#include "columnstoreversion.h" #include "liboamcpp.h" #include "configcpp.h" #include "alarmmanager.h" @@ -229,10 +230,7 @@ int main(int argc, char* argv[]) } else if ( string("-v") == argv[i] ) { - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << endl << "columnstore Version: " << systemsoftware.Version << "-" << systemsoftware.Release << endl; + cout << "columnstore Version: " << columnstore_version << "-" << columnstore_release << endl; exit (0); } else if ( string("-pc") == argv[i] ) @@ -1468,10 +1466,7 @@ int main(int argc, char* argv[]) cout << "===== Setting up system '" + systemName + "' based on these settings ===== " << endl << endl; - SystemSoftware systemsoftware; - oam.getSystemSoftware(systemsoftware); - - cout << "columnstore Version = " << systemsoftware.Version << "-" << systemsoftware.Release << endl; + cout << "columnstore Version = " << columnstore_version << "-" << columnstore_release << endl; cout << "System Type = " << systemType << endl; if ( subnetID != oam::UnassignedName ) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 32716b2f0..3bc02c6ae 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -269,7 +269,7 @@ int main(int argc, char* argv[]) catch (...) {} //get memory stats - long long total = myinfo.totalram / 1024 / 1000; +// long long total = myinfo.totalram / 1024 / 1000; // adjust max memory, 25% of total memory string percent = "25%"; @@ -901,7 +901,7 @@ int main(int argc, char* argv[]) cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; - cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias" << endl << endl; + cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias.sh" << endl << endl; } else { diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index eed7634bd..86a302f7d 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -69,6 +69,7 @@ #include "boost/filesystem/path.hpp" #include "boost/tokenizer.hpp" +#include "columnstoreversion.h" #include "liboamcpp.h" #include "configcpp.h" @@ -165,7 +166,7 @@ bool thread_remote_installer = true; string singleServerInstall = "1"; string reuseConfig = "n"; -string oldFileName; +string oldFileName = oam::UnassignedName; string glusterCopies; string glusterInstalled = "n"; string hadoopInstalled = "n"; @@ -399,7 +400,8 @@ int main(int argc, char* argv[]) exit(1); } - oldFileName = installDir + "/etc/Columnstore.xml.rpmsave"; + if ( oldFileName == oam::UnassignedName ) + oldFileName = installDir + "/etc/Columnstore.xml.rpmsave"; cout << endl; cout << "This is the MariaDB ColumnStore System Configuration and Installation tool." << endl; @@ -3789,7 +3791,7 @@ int main(int argc, char* argv[]) cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; - cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias" << endl << endl; + cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias.sh" << endl << endl; } else { diff --git a/oamapps/serverMonitor/CMakeLists.txt b/oamapps/serverMonitor/CMakeLists.txt index d318fd78d..91d13bbd0 100644 --- a/oamapps/serverMonitor/CMakeLists.txt +++ b/oamapps/serverMonitor/CMakeLists.txt @@ -13,7 +13,8 @@ set(ServerMonitor_SRCS procmonMonitor.cpp msgProcessor.cpp dbhealthMonitor.cpp - UMAutoSync.cpp) + UMAutoSync.cpp + ../../utils/common/crashtrace.cpp) add_executable(ServerMonitor ${ServerMonitor_SRCS}) diff --git a/oamapps/serverMonitor/diskMonitor.cpp b/oamapps/serverMonitor/diskMonitor.cpp index 4ce680c76..b86a4837b 100644 --- a/oamapps/serverMonitor/diskMonitor.cpp +++ b/oamapps/serverMonitor/diskMonitor.cpp @@ -23,6 +23,7 @@ ***************************************************************************/ #include "serverMonitor.h" +#include "installdir.h" using namespace std; using namespace oam; @@ -249,7 +250,7 @@ void diskMonitor() // check local if ( deviceName == "/") { - fileName = deviceName + "usr/local/mariadb/columnstore/releasenum"; + fileName = deviceName + startup::StartUp::installDir(); } else { diff --git a/oamapps/serverMonitor/main.cpp b/oamapps/serverMonitor/main.cpp index e78c2faa4..a4422aa04 100644 --- a/oamapps/serverMonitor/main.cpp +++ b/oamapps/serverMonitor/main.cpp @@ -18,6 +18,8 @@ #include "IDBPolicy.h" #include "serverMonitor.h" +#include "crashtrace.h" + using namespace std; using namespace servermonitor; using namespace oam; @@ -38,6 +40,14 @@ int main (int argc, char** argv) ServerMonitor serverMonitor; Oam oam; + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + //Launch Memory Monitor Thread and check if swap is in critical condition pthread_t memoryMonitorThread; pthread_create (&memoryMonitorThread, NULL, (void* (*)(void*)) &memoryMonitor, NULL); diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index e13071f12..477792b0d 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -18,7 +18,8 @@ set(PrimProc_SRCS primitiveserver.cpp pseudocc.cpp rtscommand.cpp - umsocketselector.cpp) + umsocketselector.cpp + ../../utils/common/crashtrace.cpp) #PrimProc_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS) diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index ca6dfb365..140926266 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -71,6 +71,8 @@ using namespace idbdatafile; #include "cgroupconfigurator.h" +#include "crashtrace.h" + namespace primitiveprocessor { @@ -127,6 +129,12 @@ void setupSignalHandlers() ign.sa_handler = SIG_IGN; sigaction(SIGUSR2, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + sigset_t sigset; sigemptyset(&sigset); sigaddset(&sigset, SIGPIPE); @@ -310,6 +318,9 @@ int main(int argc, char* argv[]) systemLang.find("UTF") != string::npos ) utf8 = true; + // This is unset due to the way we start it + program_invocation_short_name = const_cast("PrimProc"); + Config* cf = Config::makeConfig(); setupSignalHandlers(); diff --git a/procmgr/CMakeLists.txt b/procmgr/CMakeLists.txt index 1e7d1f33e..642890e13 100644 --- a/procmgr/CMakeLists.txt +++ b/procmgr/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ProcMgr_SRCS main.cpp processmanager.cpp) +set(ProcMgr_SRCS main.cpp processmanager.cpp ../utils/common/crashtrace.cpp) add_executable(ProcMgr ${ProcMgr_SRCS}) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index b8fcaaa81..338cd9826 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -27,11 +27,14 @@ #include +#include "columnstoreversion.h" #include "processmanager.h" #include "installdir.h" #include "utils_utf8.h" +#include "crashtrace.h" + using namespace std; using namespace logging; using namespace messageqcpp; @@ -98,6 +101,16 @@ int main(int argc, char** argv) setlocale(LC_ALL, systemLang.c_str()); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("ProcMgr"); + + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + Oam oam; //check if root-user @@ -542,25 +555,7 @@ static void startMgrProcessThread() log.writeLog(__LINE__, "startMgrProcessThread launched", LOG_TYPE_DEBUG); //get calpont software version and release - SystemSoftware systemsoftware; - string softwareVersion; - string softwareRelease; - - try - { - oam.getSystemSoftware(systemsoftware); - - softwareVersion = systemsoftware.Version; - softwareRelease = systemsoftware.Release; - } - catch (exception& e) - { - cout << endl << "ProcMon Construct Error reading getSystemSoftware = " << e.what() << endl; - exit(-1); - } - - string localSoftwareInfo = softwareVersion + softwareRelease; - + string localSoftwareInfo = columnstore_version + columnstore_release; //get systemStartupOffline string systemStartupOffline = "n"; @@ -1746,15 +1741,18 @@ void pingDeviceThread() processManager.setStandbyModule(moduleName); } - DBRootConfigList::iterator pt = dbrootConfigList.begin(); - - if (( DBRootStorageType == "DataRedundancy") && (*pt == 1)) + if ((moduleName.find("pm") == 0) && (dbrootConfigList.size() > 0)) { - log.writeLog(__LINE__, "stopModule, " + config.moduleName(), LOG_TYPE_DEBUG); - processManager.stopModule(config.moduleName(), oam::FORCEFUL, false); - processManager.switchParentOAMModule(moduleName); - processManager.stopProcess(config.moduleName(), "ProcessManager", oam::FORCEFUL, true); - break; + DBRootConfigList::iterator pt = dbrootConfigList.begin(); + + if (( DBRootStorageType == "DataRedundancy") && (*pt == 1)) + { + log.writeLog(__LINE__, "stopModule, " + config.moduleName(), LOG_TYPE_DEBUG); + processManager.stopModule(config.moduleName(), oam::FORCEFUL, false); + processManager.switchParentOAMModule(moduleName); + processManager.stopProcess(config.moduleName(), "ProcessManager", oam::FORCEFUL, true); + break; + } } } else @@ -1789,23 +1787,13 @@ void pingDeviceThread() oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); + //set recycle process + processManager.recycleProcess(moduleName); + //distribute config file processManager.distributeConfigFile("system"); sleep(1); - // if a PM module was started successfully, restart ACTIVE DBRM(s), ExeMgr(s) / mysqld - if ( moduleName.find("pm") == 0 ) - { - processManager.restartProcessType("DBRMControllerNode", moduleName); - processManager.restartProcessType("DBRMWorkerNode"); - processManager.stopProcessType("DDLProc"); - processManager.stopProcessType("DMLProc"); - processManager.stopProcessType("ExeMgr"); - processManager.restartProcessType("PrimProc"); - sleep(1); - processManager.restartProcessType("ExeMgr"); - } - string moduleType = moduleName.substr(0, MAX_MODULE_TYPE_SIZE); if ( MySQLRep == "y" ) @@ -1822,27 +1810,9 @@ void pingDeviceThread() DeviceNetworkConfig devicenetworkconfig; devicenetworkconfig.DeviceName = moduleName; devicenetworklist.push_back(devicenetworkconfig); - processManager.setMySQLReplication(devicenetworklist); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true); } } - else - { - if ( moduleName.find("pm") == 0 ) - { - processManager.restartProcessType("mysql", moduleName); - sleep(1); - } - } - - // if a PM module was started successfully, DMLProc/DDLProc - if ( moduleName.find("pm") == 0 ) - { - processManager.restartProcessType("WriteEngineServer"); - sleep(1); - processManager.restartProcessType("DDLProc"); - sleep(1); - processManager.restartProcessType("DMLProc"); - } //enable query stats dbrm.setSystemQueryReady(true); @@ -1851,89 +1821,6 @@ void pingDeviceThread() processManager.setQuerySystemState(true); processManager.setSystemState(oam::ACTIVE); - - //reset standby module - string newStandbyModule = processManager.getStandbyModule(); - - //send message to start new Standby Process-Manager, if needed - if ( !newStandbyModule.empty() && newStandbyModule != "NONE") - { - processManager.setStandbyModule(newStandbyModule); - } - else - { - Config* sysConfig = Config::makeConfig(); - - // clear Standby OAM Module - sysConfig->setConfig("SystemConfig", "StandbyOAMModuleName", oam::UnassignedName); - sysConfig->setConfig("ProcStatusControlStandby", "IPAddr", oam::UnassignedIpAddr); - - //update Calpont Config table - try - { - sysConfig->write(); - } - catch (...) - { - log.writeLog(__LINE__, "ERROR: sysConfig->write", LOG_TYPE_ERROR); - } - } - - if ( moduletypeconfig.RunType == SIMPLEX ) - { - //start SIMPLEX runtype processes on a SIMPLEX runtype module - string moduletype = moduleName.substr(0, MAX_MODULE_TYPE_SIZE); - DeviceNetworkList::iterator pt = moduletypeconfig.ModuleNetworkList.begin(); - - for ( ; pt != moduletypeconfig.ModuleNetworkList.end() ; pt++) - { - string launchModuleName = (*pt).DeviceName; - string launchModuletype = launchModuleName.substr(0, MAX_MODULE_TYPE_SIZE); - - if ( moduletype != launchModuletype ) - continue; - - //skip if active pm module (local module) - if ( launchModuleName == config.moduleName() ) - continue; - - //check if module is active before starting any SIMPLEX STANDBY apps - try - { - int launchopState = oam::ACTIVE; - bool degraded; - oam.getModuleStatus(launchModuleName, launchopState, degraded); - - if (launchopState != oam::ACTIVE && launchopState != oam::STANDBY ) - { - continue; - } - } - catch (exception& ex) - { -// string error = ex.what(); -// log.writeLog(__LINE__, "EXCEPTION ERROR on : " + error, LOG_TYPE_ERROR); - } - catch (...) - { -// log.writeLog(__LINE__, "EXCEPTION ERROR on getModuleStatus on module " + moduleName + ": Caught unknown exception!", LOG_TYPE_ERROR); - } - - int status; - log.writeLog(__LINE__, "Starting up STANDBY process on module " + launchModuleName, LOG_TYPE_DEBUG); - - for ( int j = 0 ; j < 20 ; j ++ ) - { - status = processManager.startModule(launchModuleName, oam::FORCEFUL, oam::AUTO_OFFLINE); - - if ( status == API_SUCCESS) - break; - } - - log.writeLog(__LINE__, "pingDeviceThread: ACK received from '" + launchModuleName + "' Process-Monitor, return status = " + oam.itoa(status), LOG_TYPE_DEBUG); - } - } - //clear count moduleInfoList[moduleName] = 0; } @@ -2009,9 +1896,19 @@ void pingDeviceThread() break; // if disabled and not amazon, skip - if (opState == oam::AUTO_DISABLED && !amazon) + if ( (opState == oam::AUTO_DISABLED) && !amazon) break; + // if disabled, amazon,and NOT terminated skip + if ( (opState == oam::AUTO_DISABLED) && amazon) + { + // return values = 'ip address' for running or rebooting, stopped or terminated + string currentIPAddr = oam.getEC2InstanceIpAddress(hostName); + + if ( currentIPAddr != "terminated") + break; + } + log.writeLog(__LINE__, "module failed to respond to pings: " + moduleName, LOG_TYPE_WARNING); //bump module ping failure counter @@ -2026,73 +1923,88 @@ void pingDeviceThread() if (LANOUTAGEACTIVE) break; - //Log failure, issue alarm, set moduleOpState - Configuration config; - log.writeLog(__LINE__, "module is down: " + moduleName, LOG_TYPE_CRITICAL); + //check if down module is PrimaryUMModuleName + bool downPrimaryUM = false; + string PrimaryUMModuleName; - //set query system state not ready - BRM::DBRM dbrm; - dbrm.setSystemQueryReady(false); - - processManager.setQuerySystemState(false); - - processManager.setSystemState(oam::BUSY_INIT); - - processManager.reinitProcessType("cpimport"); - - // halt the dbrm - oam.dbrmctl("halt"); - log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG); - - processManager.setSystemState(oam::BUSY_INIT); - - //string cmd = "/etc/init.d/glusterd restart > /dev/null 2>&1"; - //system(cmd.c_str()); - - //send notification - oam.sendDeviceNotification(moduleName, MODULE_DOWN); - - //Issue an alarm - aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, SET); - - //mark all processes running on module auto-offline - processManager.setProcessStates(moduleName, oam::AUTO_OFFLINE); - - //set module to disable state - processManager.disableModule(moduleName, false); - - //call dbrm control - oam.dbrmctl("reload"); - log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); - - // if pm, move dbroots to other pms - if ( ( moduleName.find("pm") == 0 && !amazon && ( DBRootStorageType != "internal") ) || - ( moduleName.find("pm") == 0 && amazon && downActiveOAMModule ) || - ( moduleName.find("pm") == 0 && amazon && AmazonPMFailover == "y") ) + try { - try + oam.getSystemConfig("PrimaryUMModuleName", PrimaryUMModuleName); + } + catch (...) {} + + if ( PrimaryUMModuleName == moduleName ) + downPrimaryUM = true; + + // if not disabled and amazon, skip + if (opState != oam::AUTO_DISABLED ) + { + //Log failure, issue alarm, set moduleOpState + Configuration config; + log.writeLog(__LINE__, "module is down: " + moduleName, LOG_TYPE_CRITICAL); + + //set query system state not ready + BRM::DBRM dbrm; + dbrm.setSystemQueryReady(false); + + processManager.setQuerySystemState(false); + + processManager.setSystemState(oam::BUSY_INIT); + + processManager.reinitProcessType("cpimport"); + + // halt the dbrm + oam.dbrmctl("halt"); + log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG); + + processManager.setSystemState(oam::BUSY_INIT); + + //string cmd = "/etc/init.d/glusterd restart > /dev/null 2>&1"; + //system(cmd.c_str()); + + //send notification + oam.sendDeviceNotification(moduleName, MODULE_DOWN); + + //Issue an alarm + aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, SET); + + //mark all processes running on module auto-offline + processManager.setProcessStates(moduleName, oam::AUTO_OFFLINE); + + //set module to disable state + processManager.disableModule(moduleName, false); + + //call dbrm control + oam.dbrmctl("reload"); + log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); + + // if pm, move dbroots to other pms + if ( ( moduleName.find("pm") == 0 && !amazon && ( DBRootStorageType != "internal") ) || + ( moduleName.find("pm") == 0 && amazon && downActiveOAMModule ) || + ( moduleName.find("pm") == 0 && amazon && AmazonPMFailover == "y") ) { - log.writeLog(__LINE__, "Call autoMovePmDbroot", LOG_TYPE_DEBUG); - oam.autoMovePmDbroot(moduleName); - log.writeLog(__LINE__, "autoMovePmDbroot success", LOG_TYPE_DEBUG); - //distribute config file - processManager.distributeConfigFile("system"); - } - catch (exception& ex) - { - string error = ex.what(); - log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: " + error, LOG_TYPE_DEBUG); - } - catch (...) - { - log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: Caught unknown exception!", LOG_TYPE_ERROR); + try + { + log.writeLog(__LINE__, "Call autoMovePmDbroot", LOG_TYPE_DEBUG); + oam.autoMovePmDbroot(moduleName); + log.writeLog(__LINE__, "autoMovePmDbroot success", LOG_TYPE_DEBUG); + //distribute config file + processManager.distributeConfigFile("system"); + } + catch (exception& ex) + { + string error = ex.what(); + log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: " + error, LOG_TYPE_DEBUG); + } + catch (...) + { + log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: Caught unknown exception!", LOG_TYPE_ERROR); + } } } // if Cloud Instance - // state = running, then instance is rebooting, monitor for recovery - // state = stopped, then try starting, if fail, remove/addmodule to launch new instance - // state = terminate or nothing, remove/addmodule to launch new instance + // state = terminate, remove/addmodule to launch new instance if ( amazon ) { if ( moduleName.find("um") == 0 ) @@ -2315,7 +2227,9 @@ void pingDeviceThread() } } - if ( moduleName.find("pm") == 0 ) + if ( ( moduleName.find("pm") == 0 ) && + ( opState != oam::AUTO_DISABLED ) ) + { // resume the dbrm oam.dbrmctl("resume"); @@ -2343,45 +2257,57 @@ void pingDeviceThread() //set query system state ready processManager.setQuerySystemState(true); + } - sleep(2); + //check if down module was Standby OAM, if so find another one + if ( moduleName == config.OAMStandbyName() ) + { - //check if down module was Standby OAM, if so find another one - if ( moduleName == config.OAMStandbyName() ) + //set down module ProcessManager to AOS + processManager.setProcessState(moduleName, "ProcessManager", oam::AUTO_OFFLINE, 0); + + //get another standby OAM module + string newStandbyModule = processManager.getStandbyModule(); + + //send message to start new Standby Process-Manager, if needed + if ( !newStandbyModule.empty() && newStandbyModule != "NONE") { + processManager.setStandbyModule(newStandbyModule); + } + else + { + Config* sysConfig = Config::makeConfig(); - //set down module ProcessManager to AOS - processManager.setProcessState(moduleName, "ProcessManager", oam::AUTO_OFFLINE, 0); + // clear Standby OAM Module + sysConfig->setConfig("SystemConfig", "StandbyOAMModuleName", oam::UnassignedName); + sysConfig->setConfig("ProcStatusControlStandby", "IPAddr", oam::UnassignedIpAddr); - //get another standby OAM module - string newStandbyModule = processManager.getStandbyModule(); - - //send message to start new Standby Process-Manager, if needed - if ( !newStandbyModule.empty() && newStandbyModule != "NONE") + //update Calpont Config table + try { - processManager.setStandbyModule(newStandbyModule); + sysConfig->write(); } - else + catch (...) { - Config* sysConfig = Config::makeConfig(); - - // clear Standby OAM Module - sysConfig->setConfig("SystemConfig", "StandbyOAMModuleName", oam::UnassignedName); - sysConfig->setConfig("ProcStatusControlStandby", "IPAddr", oam::UnassignedIpAddr); - - //update Calpont Config table - try - { - sysConfig->write(); - } - catch (...) - { - log.writeLog(__LINE__, "ERROR: sysConfig->write", LOG_TYPE_ERROR); - } + log.writeLog(__LINE__, "ERROR: sysConfig->write", LOG_TYPE_ERROR); } } } + // reset up mysql rep slaves is master changed + if ( downPrimaryUM && + ( MySQLRep == "y" ) ) + { + //setup MySQL Replication for started modules + log.writeLog(__LINE__, "Setup MySQL Replication for module outage on " + moduleName, LOG_TYPE_DEBUG); + DeviceNetworkList devicenetworklist; + processManager.setMySQLReplication(devicenetworklist); + } + + // if disabled and amazon, break out + if ( (opState == oam::AUTO_DISABLED ) && amazon ) + break; + //start SIMPLEX runtype processes on a SIMPLEX runtype module string moduletype = moduleName.substr(0, MAX_MODULE_TYPE_SIZE); diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index d6fae0223..9fb9e399f 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -24,6 +24,7 @@ //#define NDEBUG #include +#include "columnstoreversion.h" #include "processmanager.h" #include "installdir.h" #include "dbrm.h" @@ -3772,7 +3773,6 @@ void ProcessManager::recycleProcess(string module) string moduleType = module.substr(0, MAX_MODULE_TYPE_SIZE); - // if a UM module, send a restart on DMLProc/DDLProc to get started on another UM, if needed string PrimaryUMModuleName; try @@ -3781,42 +3781,41 @@ void ProcessManager::recycleProcess(string module) } catch (...) {} - //restart ExeMgrs/mysql if module is a pm - if ( moduleType == "pm" ) + //recycle DBRM processes in all cases + restartProcessType("DBRMControllerNode", module); + restartProcessType("DBRMWorkerNode"); + + // only recycle dmlproc, if down/up module is non-parent UM + if ( ( moduleType == "um" ) && + ( PrimaryUMModuleName != module) ) + { + restartProcessType("DMLProc", module); + return; + } + + if ( PrimaryUMModuleName == module) { - restartProcessType("DBRMControllerNode", module); - restartProcessType("DBRMWorkerNode"); stopProcessType("DDLProc"); stopProcessType("DMLProc"); - stopProcessType("ExeMgr"); - restartProcessType("PrimProc"); - sleep(1); - restartProcessType("ExeMgr"); - sleep(1); - restartProcessType("mysql"); - } - else - { - restartProcessType("DBRMControllerNode", module); - restartProcessType("DBRMWorkerNode"); - restartProcessType("ExeMgr"); } - if ( PrimaryUMModuleName == module ) - { - restartProcessType("DDLProc", module); - sleep(1); - restartProcessType("DMLProc", module); - } + stopProcessType("ExeMgr"); - if ( moduleType == "pm" && PrimaryUMModuleName != module) - { - restartProcessType("WriteEngineServer"); - sleep(1); - restartProcessType("DDLProc"); - sleep(1); - restartProcessType("DMLProc", module); - } + restartProcessType("PrimProc"); + sleep(1); + + restartProcessType("ExeMgr"); + sleep(1); + + restartProcessType("mysql"); + + restartProcessType("WriteEngineServer"); + sleep(1); + + restartProcessType("DDLProc", module); + sleep(1); + + restartProcessType("DMLProc", module); return; } @@ -4654,7 +4653,6 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski SystemProcessStatus systemprocessstatus; ProcessStatus processstatus; int retStatus = API_SUCCESS; - bool setPMProcIPs = true; log.writeLog(__LINE__, "restartProcessType: Restart all " + processName, LOG_TYPE_DEBUG); @@ -4710,7 +4708,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) ) continue; - if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) && setPMProcIPs ) + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { string procModuleType = systemprocessstatus.processstatus[i].Module.substr(0, MAX_MODULE_TYPE_SIZE); @@ -4740,11 +4738,14 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski // if DDL or DMLProc, change IP Address if ( retStatus == oam::API_SUCCESS ) { - if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) && setPMProcIPs ) +// sleep(5); + ProcessStatus procstat; + oam.getProcessStatus(processName, systemprocessstatus.processstatus[i].Module, procstat); + + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { processManager.setPMProcIPs(systemprocessstatus.processstatus[i].Module, processName); - setPMProcIPs = false; - continue; + break; } } } @@ -4979,11 +4980,11 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } if ( packageType == "rpm") - calpontPackage = homedir + "/mariadb-columnstore*" + systemsoftware.Version + "-" + systemsoftware.Release + "*.rpm.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm.tar.gz"; else if ( packageType == "deb") - calpontPackage = homedir + "/mariadb-columnstore*" + systemsoftware.Version + "-" + systemsoftware.Release + "*.deb.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb.tar.gz"; else - calpontPackage = homedir + "/mariadb-columnstore*" + systemsoftware.Version + "-" + systemsoftware.Release + "*.bin.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.bin.tar.gz"; if ( DistributedInstall == "y" ) { @@ -10273,22 +10274,6 @@ int ProcessManager::OAMParentModuleChange() sleep(1); } - //set status to BUSY_INIT while failover is in progress - processManager.setSystemState(oam::BUSY_INIT); - - // graceful start snmptrap-daemon - string EnableSNMP = "y"; - - try - { - oam.getSystemConfig("EnableSNMP", EnableSNMP); - } - catch (...) - {} - - if ( EnableSNMP == "y" ) - startProcess(config.moduleName(), "SNMPTrapDaemon", oam::GRACEFUL); - // set alarm ALARMManager aManager; aManager.sendAlarmReport(config.moduleName().c_str(), MODULE_SWITCH_ACTIVE, SET); @@ -10315,7 +10300,6 @@ int ProcessManager::OAMParentModuleChange() processManager.stopModule(config.moduleName(), oam::FORCEFUL, true); string localModule = config.moduleName(); -// processManager.setModuleState(localModule, oam::AUTO_INIT); pthread_t startmodulethread; int status = pthread_create (&startmodulethread, NULL, (void* (*)(void*)) &startModuleThread, &localModule); @@ -10328,19 +10312,49 @@ int ProcessManager::OAMParentModuleChange() status = startsystemthreadStatus; } + // waiting until dml are ACTIVE + while (true) + { + ProcessStatus DMLprocessstatus; + + try + { + oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); + } + catch (exception& ex) + {} + catch (...) + {} + + if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) + log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback", LOG_TYPE_DEBUG); + + if (DMLprocessstatus.ProcessOpState == oam::ACTIVE) + break; + + if (DMLprocessstatus.ProcessOpState == oam::FAILED) + break; + + // wait some more + sleep(2); + } + + //set recycle process + processManager.recycleProcess(downOAMParentName); + //restart/reinit processes to force their release of the controller node port if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) && - ( moduleNameList.size() <= 1 && config.moduleType() == "pm") ) + ( moduleNameList.size() <= 0 && config.moduleType() == "pm") ) { - status = 0; + int status = 0; } else { - processManager.restartProcessType("mysql", localModule); - processManager.restartProcessType("ExeMgr", localModule); - processManager.restartProcessType("WriteEngineServer", localModule); +// processManager.restartProcessType("mysql", localModule); +// processManager.restartProcessType("ExeMgr", localModule); +// processManager.restartProcessType("WriteEngineServer", localModule); - processManager.reinitProcessType("DBRMWorkerNode"); +// processManager.reinitProcessType("DBRMWorkerNode"); //send message to start new Standby Process-Manager, if needed newStandbyModule = getStandbyModule(); @@ -10424,13 +10438,12 @@ int ProcessManager::OAMParentModuleChange() //restart DDLProc/DMLProc to perform any rollbacks, if needed //dont rollback in amazon, wait until down pm recovers - if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) - && !amazon ) - { - processManager.restartProcessType("DDLProc", config.moduleName()); - sleep(1); - processManager.restartProcessType("DMLProc", config.moduleName()); - } +// if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) +// && !amazon ) { +// processManager.restartProcessType("DDLProc", config.moduleName()); +// sleep(1); +// processManager.restartProcessType("DMLProc", config.moduleName()); +// } if ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) { @@ -10605,13 +10618,17 @@ std::string ProcessManager::getStandbyModule() //already have a hot-standby return ""; - if ( backupStandbyModule != "NONE" ) + if ( ( backupStandbyModule != "NONE" ) || + ( newStandbyModule != "NONE" ) ) continue; if ( systemprocessstatus.processstatus[i].ProcessName == "ProcessManager" && systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY ) + { // Found a ProcessManager in a COLD_STANDBY state newStandbyModule = systemprocessstatus.processstatus[i].Module; + continue; + } if ( systemprocessstatus.processstatus[i].ProcessName == "ProcessManager" && systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_OFFLINE && @@ -11281,6 +11298,20 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) { @@ -11367,6 +11398,20 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) { @@ -11416,12 +11461,27 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist for ( ; listPT != devicenetworklist.end() ; listPT++) { string remoteModuleName = (*listPT).DeviceName; - log.writeLog(__LINE__, "Setup SlavMySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); //skip master if ( remoteModuleName == masterModule ) continue; + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + + log.writeLog(__LINE__, "Setup Slave MySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); + ByteStream msg1; ByteStream::byte requestID = oam::SLAVEREP; diff --git a/procmon/CMakeLists.txt b/procmon/CMakeLists.txt index 151b9511d..8bfff3c5a 100644 --- a/procmon/CMakeLists.txt +++ b/procmon/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ProcMon_SRCS main.cpp processmonitor.cpp) +set(ProcMon_SRCS main.cpp processmonitor.cpp ../utils/common/crashtrace.cpp) add_executable(ProcMon ${ProcMon_SRCS}) diff --git a/procmon/main.cpp b/procmon/main.cpp index 79455aaad..8cd131455 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -26,6 +26,8 @@ namespace bi = boost::interprocess; #include "IDBPolicy.h" +#include "crashtrace.h" + using namespace std; using namespace messageqcpp; using namespace processmonitor; @@ -55,10 +57,12 @@ void updateShareMemory(processStatusList* aPtr); bool runStandby = false; bool processInitComplete = false; bool rootUser = true; +bool mainResumeFlag; string USER = "root"; string PMwithUM = "n"; bool startProcMon = false; + //extern std::string gOAMParentModuleName; extern bool gOAMParentModuleFlag; @@ -76,6 +80,14 @@ int main(int argc, char** argv) setuid(0); // set effective ID to root; ignore return status #endif + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + if (argc > 1 && string(argv[1]) == "--daemon") { if (fork() != 0) return 0; @@ -175,7 +187,7 @@ int main(int argc, char** argv) //re-read local system info with updated Columnstore.xml sleep(1); - Config* sysConfig = Config::makeConfig(); +// Config* sysConfig = Config::makeConfig(); MonitorConfig config; //PMwithUM config @@ -561,6 +573,10 @@ int main(int argc, char** argv) unlink ("/var/log/mariadb/columnstore/activeAlarms"); } + //Clear mainResumeFlag + + mainResumeFlag = false; + //launch Status table control thread on 'pm' modules pthread_t statusThread; int ret = pthread_create (&statusThread, NULL, (void* (*)(void*)) &statusControlThread, NULL); @@ -568,7 +584,14 @@ int main(int argc, char** argv) if ( ret != 0 ) log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); - sleep(6); // give the Status thread time to fully initialize + //wait for flag to be set + + while (!mainResumeFlag) + { + log.writeLog(__LINE__, "WATING FOR mainResumeFlag to be set", LOG_TYPE_DEBUG); + + sleep(1); + } } SystemStatus systemstatus; @@ -877,6 +900,8 @@ int main(int argc, char** argv) } } + log.writeLog(__LINE__, "SYSTEM STATUS = " + oam.itoa(systemstatus.SystemOpState), LOG_TYPE_DEBUG); + if ( systemstatus.SystemOpState != MAN_OFFLINE && !DISABLED) { @@ -2320,6 +2345,10 @@ static void statusControlThread() log.writeLog(__LINE__, "Dbroot Status shared Memory allociated and Initialized", LOG_TYPE_DEBUG); } + //Set mainResumeFlag, to start up main thread + + mainResumeFlag = true; + string portName = "ProcStatusControl"; if (runStandby) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 279e8c814..cda141e84 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -23,6 +23,7 @@ #include +#include "columnstoreversion.h" #include "IDBDataFile.h" #include "IDBPolicy.h" #include "processmonitor.h" @@ -113,19 +114,8 @@ MonitorConfig::MonitorConfig() // } //get calpont software version and release - SystemSoftware systemsoftware; - - try - { - oam.getSystemSoftware(systemsoftware); - - fsoftwareVersion = systemsoftware.Version; - fsoftwareRelease = systemsoftware.Release; - } - catch (exception& e) - { - cout << endl << "ProcMon Construct Error reading getSystemSoftware = " << e.what() << endl; - } + fsoftwareVersion = columnstore_version; + fsoftwareRelease = columnstore_release; } @@ -5358,6 +5348,15 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master { string moduleName = (*pt).DeviceName; + //skip if module is not ACTIVE + + int opState = oam::ACTIVE; + bool degraded; + oam.getModuleStatus(moduleName, opState, degraded); + + if (opState != oam::ACTIVE) + continue; + bool passwordError = false; string moduleType = systemModuleTypeConfig.moduletypeconfig[i].ModuleType; diff --git a/tools/configMgt/buildCalpontPackages.sh b/tools/configMgt/buildCalpontPackages.sh deleted file mode 100755 index 4ed8ebe51..000000000 --- a/tools/configMgt/buildCalpontPackages.sh +++ /dev/null @@ -1,48 +0,0 @@ -#! /bin/sh -# -# $1 - release number or 'Latest' (default 'Latest') -# -if [ "$1" = "" ] ; then - DIR=Latest - REL="3.6-0" -elif [ "$1" = "Latest" ] ; then - DIR=Latest - REL="3.6-0" -elif [ "$2" = "" ] ; then - DIR=$1 - REL=$1 -else - DIR=$1 - REL=$2 -fi -# -if [[ $REL != *-* ]] ; then - REL=$REL"-0" -fi -# -{ -cd /root/autoOAM/ -rpm -e infinidb-libs infinidb-platform infinidb-enterprise --nodeps -rpm -e infinidb-storage-engine infinidb-mysql --nodeps -rpm -e infinidb-mysql --nodeps -rm -rf /usr/local/Calpont -rm -rf calpont* -rm -f *gz -# -smbclient //srvhill01/shared -WMARIADB -Uroot%Calpont1 -c "cd packages/$DIR;prompt OFF;mget *.rpm" -rpm -ivh calpont*.x86_64.rpm --nodeps -rpm -iq calpont >> /usr/local/Calpont/releasenum -cd /usr/local/ -tar -zcvf calpont-infinidb-ent-$REL.x86_64.bin.tar.gz Calpont -mv calpont-infinidb-ent-$REL.x86_64.bin.tar.gz /root/autoOAM/ -cd /root/autoOAM/ -alien -ck calpont*.x86_64.rpm -tar -zcvf calpont-infinidb-ent-$REL.x86_64.rpm.tar.gz *$REL*.rpm -tar -zcvf calpont-infinidb-ent-$REL.amd64.deb.tar.gz *$REL*.deb - -smbclient //srvhill01/shared -WMARIADB -Uroot%Calpont1 -c "cd packages/$DIR;mkdir packages;cd packages;prompt OFF;del calpont-infinidb-ent*gz;mput *gz" -} > /root/autoOAM/buildCalpontPackages-$DIR.log 2>&1 -# -echo "Calpont Packages Build Successfully Completed" -exit 0 - diff --git a/tools/configMgt/calpont-build b/tools/configMgt/calpont-build deleted file mode 100755 index 6b0408e15..000000000 --- a/tools/configMgt/calpont-build +++ /dev/null @@ -1,102 +0,0 @@ -#! /bin/sh -# -# $Id: calpont-build 421 2007-14-15 15:46:55Z dhill $ -# -# build calpont rpm and install on call6500 -# $1 - release number or 'Latest' -# $2 - svn branch -# -if [ "$2" = "" ] ; then - echo "enter release and svn-branch arguments" - exit 0 -fi -# !!! set for OS type 32 or 64 bit !!! - -OS=x86_64 - -# -REL=$1 -DIR=$2 - -#remove mysqld before new one is built -#prevents old version from being packaged in new build -rm -f /usr/local/Calpont/mysql/libexec/* - -if [ $DIR = "genii" ] ; then - ./mysql-build-genii > mysql-build-genii.log & -else - ./mysql-build-branch $DIR > mysql-build-$DIR.log & -fi -# -HOME=/home/nightly/$DIR -rm -rf $HOME -rm -fr /home/nightly/rpm/ -# - -if [ $DIR = "genii" ] ; then - su - nightly -c "export CXXFLAGS='-O3 -fno-strict-aliasing ';mkdir $DIR;updateGenii.pl > ~/$DIR/updateGenii.log 2>&1;cd ~/$DIR;make clean;make > ~/$DIR/make.log 2>&1;make > ~/$DIR/make1.log 2>&1;./build/build_rpms > ~/$DIR/build_rpms.log 2>&1" -else - su - nightly -c "export CXXFLAGS='-O3 -fno-strict-aliasing ';mkdir $DIR;updateGenii.pl -b $DIR > ~/$DIR/updateGenii.log 2>&1;cd ~/$DIR;make clean;make > ~/$DIR/make.log 2>&1;make > ~/$DIR/make1.log 2>&1;./build/build_rpms > ~/$DIR/build_rpms.log 2>&1" -fi - -# -# generate rpm info text files - -if [ ! -f $HOME/build/releasenum ]; then - #default - version=3.0 -else - . $HOME/build/releasenum -fi - -cd /home/nightly/rpm/ > /dev/null 2>&1 -cd RPMS/$OS/ > /dev/null 2>&1 -test -f infinidb-platform-$version*.rpm || echo "no rpm" -test -f infinidb-platform-$version*.rpm || exit -1 -rpm -qi -p infinidb-platform-$version*.rpm > infinidb-platform-$OS-rpm-info.txt -echo " " >> infinidb-platform-$OS-rpm-info.txt -echo "MD5SUM" >> infinidb-platform-$OS-rpm-info.txt -md5sum infinidb-platform-$version*.rpm >> infinidb-platform-$OS-rpm-info.txt -echo " " >> infinidb-platform-$OS-rpm-info.txt -# -test -f infinidb-storage-engine-*.rpm || echo "no rpm" -test -f infinidb-storage-engine-*.rpm || exit -1 -rpm -qi -p infinidb-storage-engine-*.rpm > infinidb-storage-engine-$OS-rpm-info.txt -echo " " >> infinidb-storage-engine-$OS-rpm-info.txt -echo "MD5SUM" >> infinidb-storage-engine-$OS-rpm-info.txt -md5sum infinidb-storage-engine-*.rpm >> infinidb-storage-engine-$OS-rpm-info.txt -echo " " >> infinidb-storage-engine-$OS-rpm-info.txt -# -test -f infinidb-mysql-*.rpm || echo "no rpm" -test -f infinidb-mysql-*.rpm || exit -1 -rpm -qi -p infinidb-mysql-*.rpm > infinidb-mysql-$OS-rpm-info.txt -echo " " >> infinidb-mysql-$OS-rpm-info.txt -echo "MD5SUM" >> infinidb-mysql-$OS-rpm-info.txt -md5sum infinidb-mysql-*.rpm >> infinidb-mysql-$OS-rpm-info.txt -echo " " >> infinidb-mysql-$OS-rpm-info.txt -# -{ -if [ $REL != "Latest" ] ; then - # copy rpms to calweb - DATE=`date +%F` - cd /home/nightly/rpm/RPMS/$OS/ - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations;mkdir $REL;cd $REL;rm *.$OS.rpm;prompt OFF;mput *" - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/nightly;mkdir $REL-$DATE;cd $REL-$DATE;rm *.$OS.rpm;prompt OFF;mput *" - cd /home/nightly/$DIR/export/etc/ - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/$REL;rm Columnstore.xml;prompt OFF;mput Columnstore.xml" - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/nightly/$REL-$DATE;rm Columnstore.xml;prompt OFF;mput Columnstore.xml" - -else - # copy nightly on calweb - DATE=`date +%F` - cd /home/nightly/rpm/RPMS/$OS/ - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/Latest;rm *.$OS.rpm;prompt OFF;mput *" - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/nightly;mkdir genii-$DATE;cd genii-$DATE;prompt OFF;mput *" - cd /home/nightly/$DIR/export/etc/ - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/Latest;rm Columnstore.xml;prompt OFF;mput Columnstore.xml" - smbclient //calweb/shared -Wcalpont -Uoamuser%Calpont1 -c "cd Iterations/nightly/genii-$DATE;prompt OFF;mput Columnstore.xml" -fi -} 2> /root/infinidb-build-$REL-$DIR.log -# -echo "Build Successfully Completed" -exit 0 diff --git a/tools/configMgt/dm_parent_installer.sh b/tools/configMgt/dm_parent_installer.sh deleted file mode 100755 index 57498924c..000000000 --- a/tools/configMgt/dm_parent_installer.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/expect -# -# $Id$ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -#set PACKAGE [lindex $argv 2] -set PACKAGE *.rpm -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set USERNAME [lindex $argv 5] -set CEFLAG [lindex $argv 6] -set PACKAGENAME [lindex $argv 7] -set INSTALLLOCATION [lindex $argv 8] -set DEBUG [lindex $argv 9] - -set INSTALLDIR "/usr/local" - -set SHARED "//srvhill01/shared" - -log_user $DEBUG -spawn -noecho /bin/bash - -set CONFIG "Calpont" -if { $PACKAGENAME != "columnstore"} { - set CONFIG "Calpont" -} - - -set timeout 10 -send "ssh $USERNAME@$SERVER 'rm -f /root/Calpont-*.rpm /root/$PACKAGENAME*.rpm'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } -} -expect { - -re {[$#] } { } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -# -# erase InfiniDB packages -# -set timeout 60 -send_user "Erase Old InfiniDB Packages " -send "ssh $USERNAME@$SERVER 'pkill -9 mysqld'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re {[$#] } { } -} -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^mariadb-columnstore') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } - -re {[$#] } { } - -re "not installed" { } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/$INSTALLLOCATION/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall && $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re {[$#] } { send_user "DONE" } - -re "uninstall completed" { send_user "DONE" } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# get the InfiniDB package -# -send_user "Get InfiniDB Packages " -send "smbclient $SHARED -WMARIADB -Uroot%Calpont1 -c 'cd packages/$RELEASE/;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } -} -send_user "\n" -# -# send the InfiniDB package -# -set timeout 120 -send_user "Copy InfiniDB Packages " -send "scp -q $PACKAGE $USERNAME@$SERVER:/root/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re {[$#] } { send_user "DONE" } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -#send "rm -f $PACKAGE\n" -# -# install InfiniDB package -# -send_user "Install New InfiniDB Packages " -send "ssh $USERNAME@$SERVER ' rpm -iv --nodeps --force $PACKAGENAME-*'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { expect { - -re "completed" { expect { - -re "completed" { send_user "DONE" } - } - } - } - } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -set timeout 120 -expect -re {[$#] } -# -if { $CONFIGFILE != "NULL"} { - # - # copy over Columnstore.xml file - # - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/$INSTALLLOCATION/etc/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send_user "Copy InfiniDB Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/$INSTALLLOCATION/etc/$CONFIG.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE"} - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - #do a dummy scp command - send "scp $CONFIGFILE $USERNAME@$SERVER:/tmp/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user " " } - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved InfiniDB Configuration File " - send "ssh $USERNAME@$SERVER 'cd /usr/local/$INSTALLLOCATION/etc/;mv -f Columnstore.xml Columnstore.xml.install;cp -v Columnstore.xml.rpmsave Columnstore.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "Columnstore.xml" { send_user "DONE" } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -# -exit - diff --git a/tools/configMgt/dm_parent_installer_v3.sh b/tools/configMgt/dm_parent_installer_v3.sh deleted file mode 100755 index 8bb899705..000000000 --- a/tools/configMgt/dm_parent_installer_v3.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -set PACKAGE [lindex $argv 2] -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set USERNAME [lindex $argv 5] -set CEFLAG [lindex $argv 6] -set DEBUG [lindex $argv 7] - -set CALPONTPACKAGE calpont-$PACKAGE -set MYSQLPACKAGE calpont-mysql-$PACKAGE -set MYSQLDPACKAGE calpont-mysqld-$PACKAGE -set INSTALLDIR "/usr/local" - -#set SHARED "//cal6500/shared" -set SHARED "//calweb/shared" - -log_user $DEBUG -spawn -noecho /bin/bash -send "rm -f $CALPONTPACKAGE\n" -# -# delete and erase all old packages from Director Module -# -set timeout 10 -send "ssh $USERNAME@$SERVER 'rm -f /root/$CALPONTPACKAGE'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } abort - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } abort -} -expect { - -re "#" { } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -# -# erase calpont packages -# -set timeout 60 -send_user "Erase Old Calpont Packages " -send "ssh $USERNAME@$SERVER 'pkill -9 mysqld'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { } abort -} -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^calpont') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } abort - -re "# " { } abort - -re "not installed" { } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/Calpont/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/Calpont/bin/pre-uninstall && $INSTALLDIR/Calpont/bin/pre-uninstall'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { send_user "DONE" } abort - -re "uninstall completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# get the calpont package -# -send_user "Get Calpont Packages " -send "smbclient $SHARED -Wcalpont -Uoamuser%Calpont1 -c 'cd Iterations/$RELEASE/;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } abort -} -send_user "\n" -# -# send the calpont package -# -send_user "Copy Calpont Packages " -send "scp -q $PACKAGE $USERNAME@$SERVER:/root/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re "#" { send_user "DONE" } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -#send "rm -f $PACKAGE\n" -# -# install calpont package -# -set timeout 120 -send_user "Install New Calpont Packages " -send "ssh $USERNAME@$SERVER ' rpm -iv --nodeps /root/$CALPONTPACKAGE'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { expect { - -re "completed" { expect { - -re "completed" { send_user "DONE" } abort - } - } - } - } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -set timeout 120 -expect -re "# " -# -if { $CONFIGFILE != "NULL"} { - # - # copy over Columnstore.xml file - # - send_user "Copy Calpont Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/mariadb/columnstore/etc/Columnstore.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/mariadb/columnstore/etc/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:/tmp/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user " " } abort - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved Calpont Configuration File " - send "ssh $USERNAME@$SERVER 'cd /usr/local/mariadb/columnstore/etc/;mv -f Columnstore.xml Columnstore.xml.install;cp -v Columnstore.xml.rpmsave Columnstore.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "Columnstore.xml" { send_user "DONE" } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -# -exit - diff --git a/tools/configMgt/parent_binary_installer.sh b/tools/configMgt/parent_binary_installer.sh deleted file mode 100755 index dc87420e6..000000000 --- a/tools/configMgt/parent_binary_installer.sh +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -set PACKAGE [lindex $argv 2] -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set USERNAME [lindex $argv 5] -set INSTALLDIR [lindex $argv 6] -set PACKAGENAME [lindex $argv 7] -set INSTALLLOCATION [lindex $argv 8] -set DEBUG [lindex $argv 9] - -set CALPONTPACKAGE infinidb-ent-*$PACKAGE - -set SHARED "//srvhill01/shared" - -set INSTALLDIRARG " " -set HOME "/root" -if { $USERNAME != "root" } { - set INSTALLDIRARG "--installdir=$INSTALLDIR/$INSTALLLOCATION" - set HOME $INSTALLDIR -} - -log_user $DEBUG -spawn -noecho /bin/bash -send "rm -f $PACKAGE\n" -# -# delete binary package on Parent OAM Module -# -set timeout 30 -send_user "Remove Calpont Packages from System " -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/$INSTALLLOCATION*.gz;rm -f /root/$INSTALLLOCATION*.rpm;rm -f /root/$INSTALLLOCATION*.gz'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } abort - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } abort -} -expect { - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "DONE\n" -# -# get the calpont package -# -send_user "Get Calpont Packages " -send "smbclient $SHARED -WMARIADB -Uroot%Calpont1 -c 'cd packages/$RELEASE/packages;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } abort -} -send_user "\n" -# -# send the calpont package -# -send_user "Copy Calpont Packages " -send "scp $PACKAGE $USERNAME@$SERVER:$HOME/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -# -# uninstall calpont package -# -send_user "Erase Old Calpont Package " -send "ssh $USERNAME@$SERVER 'pkill -9 mysqld'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { } abort -} -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^calpont') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } abort - -re "# " { } abort - -re "not installed" { } abort - -re "Failed dependencies" { } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/$INSTALLLOCATION/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall && $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall $INSTALLDIRARG'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { send_user "DONE" } abort - -re "uninstall completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# install calpont package -# -set timeout 30 -send_user "Install New Calpont Package " -send "ssh $USERNAME@$SERVER 'tar -C $INSTALLDIR --exclude db -zxf $HOME/$CALPONTPACKAGE;cat $INSTALLDIR/$INSTALLLOCATION/releasenum'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { send_user "DONE" } abort - -re "release=" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "exiting now" { send_user "FAILED: Error in tar command\n" ; exit -1 } - -} -send_user "\n" -set timeout 30 -send "rm -f $PACKAGE\n" -# -send_user "Run post-install script " -send " \n" -send date\n -send "ssh $USERNAME@$SERVER '$INSTALLDIR/$INSTALLLOCATION/bin/post-install $INSTALLDIRARG'\n" -set timeout 10 -expect { - -re "word: " { send "$PASSWORD\n" } abort - -re "passphrase" { send "$PASSWORD\n" } abort -} -set timeout 60 -# check return -expect { - -re "InfiniDB syslog logging not working" { send_user "ERROR: InfiniDB System logging not setup\n" ; exit 1 } - -re "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } - -re "Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1} - -re "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } - -re "closed" { send_user "ERROR: Connection closed\n" ; exit 1 } - -re "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } - -re "postConfigure" { send_user "DONE" } abort - -re "# " { send_user "DONE" } abort -} -send_user "\n" -sleep 10 -# -if { $CONFIGFILE != "NULL"} { - # - # copy over $CONFIG.xml file - # - send_user "Copy Calpont Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:$INSTALLDIR/$INSTALLLOCATION/etc/$CONFIG.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:$INSTALLDIR/$INSTALLLOCATION/etc/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved Calpont Configuration File " - send "ssh $USERNAME@$SERVER 'cd $INSTALLDIR/$INSTALLLOCATION/etc/;mv -f $CONFIG.xml $CONFIG.xml.install;cp -v $CONFIG.xml.rpmsave $CONFIG.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "$CONFIG.xml" { send_user "DONE" } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -# -exit - diff --git a/tools/configMgt/parent_binary_installer_v3.sh b/tools/configMgt/parent_binary_installer_v3.sh deleted file mode 100755 index 155004a90..000000000 --- a/tools/configMgt/parent_binary_installer_v3.sh +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -set PACKAGE [lindex $argv 2] -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set USERNAME [lindex $argv 5] -set INSTALLDIR [lindex $argv 6] -set DEBUG [lindex $argv 7] - -set CALPONTPACKAGE calpont-infinidb-ent-*$PACKAGE - -set SHARED "//calweb/shared" - -set INSTALLDIRARG " " -set HOME "/root" -if { $USERNAME != "root" } { - set INSTALLDIRARG "--installdir=$INSTALLDIR/Calpont" - set HOME $INSTALLDIR -} - -log_user $DEBUG -spawn -noecho /bin/bash -send "rm -f $PACKAGE\n" -# -# delete binary package on Parent OAM Module -# -set timeout 30 -send_user "Remove Calpont Packages from System " -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/calpont*.gz;rm -f /root/calpont*.rpm;rm -f /root/calpont*.gz'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } abort - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } abort -} -expect { - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "DONE\n" -# -# get the calpont package -# -send_user "Get Calpont Packages " -send "smbclient $SHARED -Wcalpont -Uoamuser%Calpont1 -c 'cd Iterations/$RELEASE/packages;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } abort -} -send_user "\n" -# -# send the calpont package -# -send_user "Copy Calpont Packages " -send "scp $PACKAGE $USERNAME@$SERVER:$HOME/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -# -# uninstall calpont package -# -send_user "Erase Old Calpont Package " -send "ssh $USERNAME@$SERVER 'pkill -9 mysqld'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { } abort -} -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^calpont') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n"expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } abort - -re "# " { } abort - -re "not installed" { } abort - -re "Failed dependencies" { } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/Calpont/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/Calpont/bin/pre-uninstall && $INSTALLDIR/Calpont/bin/pre-uninstall $INSTALLDIRARG'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { send_user "DONE" } abort - -re "uninstall completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# install calpont package -# -set timeout 30 -send_user "Install New Calpont Package " -send "ssh $USERNAME@$SERVER 'tar -C $INSTALLDIR --exclude db -zxf $HOME/$CALPONTPACKAGE;cat $INSTALLDIR/Calpont/releasenum'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { send_user "DONE" } abort - -re "release=" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "exiting now" { send_user "FAILED: Error in tar command\n" ; exit -1 } - -} -send_user "\n" -set timeout 30 -send "rm -f $PACKAGE\n" -# -send_user "Run post-install script " -send " \n" -send date\n -send "ssh $USERNAME@$SERVER '$INSTALLDIR/Calpont/bin/post-install $INSTALLDIRARG'\n" -set timeout 10 -expect { - -re "word: " { send "$PASSWORD\n" } abort - -re "passphrase" { send "$PASSWORD\n" } abort -} -set timeout 60 -# check return -expect { - -re "InfiniDB syslog logging not working" { send_user "ERROR: InfiniDB System logging not setup\n" ; exit 1 } - -re "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } - -re "Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1} - -re "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } - -re "closed" { send_user "ERROR: Connection closed\n" ; exit 1 } - -re "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } - -re "postConfigure" { send_user "DONE" } abort - -re "# " { send_user "DONE" } abort -} -send_user "\n" -sleep 10 -# -if { $CONFIGFILE != "NULL"} { - # - # copy over Columnstore.xml file - # - send_user "Copy Calpont Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:$INSTALLDIR/Calpont/etc/Columnstore.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:$INSTALLDIR/Calpont/etc/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved Calpont Configuration File " - send "ssh $USERNAME@$SERVER 'cd $INSTALLDIR/Calpont/etc/;mv -f Columnstore.xml Columnstore.xml.install;cp -v Columnstore.xml.rpmsave Columnstore.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "Columnstore.xml" { send_user "DONE" } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -# -exit - diff --git a/tools/configMgt/pm_parent_installer.sh b/tools/configMgt/pm_parent_installer.sh deleted file mode 100755 index c8e135b9b..000000000 --- a/tools/configMgt/pm_parent_installer.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set USERNAME root -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -#set PACKAGE [lindex $argv 2] -set PACKAGE *.rpm -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set PREFIX [lindex $argv 5] -set USERNAME [lindex $argv 6] -set PACKAGENAME [lindex $argv 7] -set INSTALLLOCATION [lindex $argv 8] -set DEBUG [lindex $argv 9] - -set SHARED "//srvhill01/shared" -set INSTALLDIR "/usr/local" - -log_user $DEBUG -spawn -noecho /bin/bash - -set CONFIG "Calpont" -if { $PACKAGENAME != "columnstore"} { - set CONFIG "Calpont" -} - -send "rm -f $PACKAGE\n" -# -# delete and erase all old packages from Parent OAM Module -# -set timeout 30 -send "ssh $USERNAME@$SERVER 'rm -f /root/Calpont-*.rpm /root/$PACKAGENAME*.rpm /root/mariadb*.rpm '\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } -} -expect { - -re {[$#] } { } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -# -# get the InfiniDB package -# -expect -re {[$#] } -send_user "Get InfiniDB Packages " -send "smbclient $SHARED -WMARIADB -Uroot%Calpont1 -c 'cd packages/$RELEASE/;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } -} -send_user "\n" -# -# send the InfiniDB package -# -set timeout 120 -send_user "Copy InfiniDB Packages " -send "scp -q $PACKAGE $USERNAME@$SERVER:/root/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re {[$#] } { send_user "DONE" } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -#send "rm -f $PACKAGE\n" -# -# erase InfiniDB package -# -set timeout 60 -expect -re {[$#] } -send_user "Erase Old InfiniDB Package " -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^mariadb-columnstore') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } - -re {[$#] } { } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/$INSTALLLOCATION/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall && $INSTALLDIR/$INSTALLLOCATION/bin/pre-uninstall'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re {[$#] } { send_user "DONE" } - -re "uninstall completed" { send_user "DONE" } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# install InfiniDB package -# -set timeout 160 -send_user "Install New InfiniDB Packages " -send "ssh $USERNAME@$SERVER ' rpm -ivh --nodeps --force $PACKAGENAME-*'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { send_user "DONE" } - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "File not found" { send_user "FAILED: File not found\n" ; exit -1 } -} -send_user "\n" -set timeout 120 -expect -re {[$#] } -send "rm -f $PACKAGE\n" -# -if { $CONFIGFILE != "NULL"} { - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/$INSTALLLOCATION/etc/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - # - # copy over $CONFIG.xml file - # - send_user "Copy InfiniDB Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/$INSTALLLOCATION/etc/$CONFIG.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - #do a dummy scp command - send "scp $CONFIGFILE $USERNAME@$SERVER:/tmp/$CONFIG.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user " " } - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved InfiniDB Configuration File " - send "ssh $USERNAME@$SERVER 'cd /usr/local/$INSTALLLOCATION/etc/;mv -f $CONFIG.xml $CONFIG.xml.install;cp -v $CONFIG.xml.rpmsave $CONFIG.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "$CONFIG.xml" { send_user "DONE" } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -expect -re {[$#] } -# -exit - diff --git a/tools/configMgt/pm_parent_installer_v3.sh b/tools/configMgt/pm_parent_installer_v3.sh deleted file mode 100755 index 88931ba38..000000000 --- a/tools/configMgt/pm_parent_installer_v3.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/expect -# -# $Id: parent_installer.sh 421 2007-04-05 15:46:55Z dhill $ -# -# Parent OAM Installer, copy RPM's and custom OS files from postConfigure script -# Argument 0 - Parent OAM IP address -# Argument 1 - Root Password of Parent OAM Module -# Argument 2 - Calpont Config File -# Argument 3 - Debug flag 1 for on, 0 for off - -set USERNAME root -set SERVER [lindex $argv 0] -set PASSWORD [lindex $argv 1] -set PACKAGE [lindex $argv 2] -set RELEASE [lindex $argv 3] -set CONFIGFILE [lindex $argv 4] -set PREFIX [lindex $argv 5] -set USERNAME [lindex $argv 6] -set DEBUG [lindex $argv 7] - -set CALPONTPACKAGE calpont-$PREFIX$PACKAGE -set MYSQLPACKAGE calpont-mysql-$PACKAGE -set MYSQLDPACKAGE calpont-mysqld-$PACKAGE -#set SHARED "//cal6500/shared" -set SHARED "//calweb/shared" -set INSTALLDIR "/usr/local" - -log_user $DEBUG -spawn -noecho /bin/bash -send "rm -f $PACKAGE\n" -# -# delete and erase all old packages from Parent OAM Module -# -set timeout 30 -send "ssh $USERNAME@$SERVER 'rm -f /root/calpont*.rpm /root/infinidb*.rpm'\n" -expect { - -re "authenticity" { send "yes\n" - expect { - -re "word: " { send "$PASSWORD\n" } abort - } - } - -re "service not known" { send_user "FAILED: Invalid Host\n" ; exit -1 } - -re "word: " { send "$PASSWORD\n" } abort -} -expect { - -re "#" { } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -# -# get the calpont package -# -expect -re "# " -send_user "Get Calpont Packages " -send "smbclient $SHARED -Wcalpont -Uoamuser%Calpont1 -c 'cd Iterations/$RELEASE/;prompt OFF;mget $PACKAGE'\n" -expect { - -re "NT_STATUS_NO_SUCH_FILE" { send_user "FAILED: $PACKAGE not found\n" ; exit -1 } - -re "getting" { send_user "DONE" } abort -} -send_user "\n" -# -# send the calpont package -# -send_user "Copy Calpont Packages " -send "scp -q $PACKAGE $USERNAME@$SERVER:/root/.\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -expect { - -re "#" { send_user "DONE" } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } -} -send_user "\n" -#send "rm -f $PACKAGE\n" -# -# erase calpont package -# -expect -re "# " -send_user "Erase Old Calpont Package " -send "ssh $USERNAME@$SERVER 'rpm -e --nodeps \$(rpm -qa | grep '^calpont') >/dev/null 2>&1; rpm -e --nodeps \$(rpm -qa | grep '^infinidb-')'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "uninstall completed" { } abort - -re "# " { } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -sleep 10 -send "ssh $USERNAME@$SERVER 'rm -f $INSTALLDIR/Calpont/releasenum >/dev/null 2>&1; test -x $INSTALLDIR/Calpont/bin/pre-uninstall && $INSTALLDIR/Calpont/bin/pre-uninstall'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "# " { send_user "DONE" } abort - -re "uninstall completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -sleep 5 -# -# install calpont package -# -set timeout 160 -send_user "Install New Calpont Package " -send "ssh $USERNAME@$SERVER ' rpm -ivh --nodeps /root/$CALPONTPACKAGE'\n" -expect -re "word: " -# password for ssh -send "$PASSWORD\n" -# check return -expect { - -re "completed" { send_user "DONE" } abort - -re "Failed dependencies" { send_user "FAILED: Failed dependencies\n" ; - send_user "\n*** Installation Failed\n" ; - exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } -} -send_user "\n" -set timeout 120 -expect -re "# " -send "rm -f $PACKAGE\n" -# -if { $CONFIGFILE != "NULL"} { - # - # copy over Columnstore.xml file - # - send_user "Copy Calpont Configuration File " - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/mariadb/columnstore/etc/Columnstore.xml.rpmsave\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:/usr/local/mariadb/columnstore/etc/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user "DONE" } abort - -re "scp" { send_user "FAILED: SCP failure\n" ; exit -1 } - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - -re "No such file or directory" { send_user "FAILED: Invalid package\n" ; exit -1 } - } - send "scp $CONFIGFILE $USERNAME@$SERVER:/tmp/Columnstore.xml\n" - expect -re "word: " - # send the password - send "$PASSWORD\n" - expect { - -re "100%" { send_user " " } abort - } -} else { - # - # rename previous installed config file - # - send_user "Copy RPM-saved Calpont Configuration File " - send "ssh $USERNAME@$SERVER 'cd /usr/local/mariadb/columnstore/etc/;mv -f Columnstore.xml Columnstore.xml.install;cp -v Columnstore.xml.rpmsave Columnstore.xml'\n" - expect -re "word: " - # password for ssh - send "$PASSWORD\n" - # check return - expect { - -re "Columnstore.xml" { send_user "DONE" } abort - -re "Permission denied, please try again" { send_user "FAILED: Invalid password\n" ; exit -1 } - } -} -send_user "\n" -expect -re "# " -# -exit - diff --git a/tools/configMgt/stackReleaseChecker.cpp b/tools/configMgt/stackReleaseChecker.cpp deleted file mode 100644 index 025516760..000000000 --- a/tools/configMgt/stackReleaseChecker.cpp +++ /dev/null @@ -1,194 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporaton - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/****************************************************************************************** -* $Id: stackReleaseChecker.cpp 64 2006-10-12 22:21:51Z dhill $ -* -* -* List of files being updated by configure: -* Calpont/etc/Columnstore.xml -* -* -******************************************************************************************/ -/** - * @file - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "stdio.h" -#include "ctype.h" -#include - -#include -#include - -#include "liboamcpp.h" -#include "configcpp.h" - -using namespace std; -using namespace oam; -using namespace config; - -int main(int argc, char* argv[]) -{ - Oam oam; - string systemUser = "root"; - string installParentModuleHostName; - string password; - string debug_flag = "1"; - string systemName = "all"; - bool VERBOSE = false; - - Config* sysConfig = Config::makeConfig("./systems/CalpontSystems.xml"); - - for ( int i = 1; i < argc; i++ ) - { - if ( string("-h") == argv[i] ) - { - cout << endl; - cout << "'stackReleaseChecker' retrieves and prints the release installed on a stack." << endl; - cout << endl; - cout << "The list of systems are located in './systems/CalpontSystems.xml" << endl; - cout << endl; - cout << "Usage: stackReleaseChecker [-h][-s system][-v]" << endl; - cout << " -s system-name or leave blank for all systems" << endl; - cout << " -v verbose will give the version numbers" << endl; - exit (0); - } - else if ( string("-s") == argv[i] ) - { - i++; - - if ( argc == i ) - { - cout << "ERROR: missing system argument" << endl; - exit(-1); - } - - systemName = argv[i]; - } - else if ( string("-v") == argv[i] ) - VERBOSE = true; - } - - int systemCount; - - try - { - systemCount = strtol(sysConfig->getConfig("NetworkConfig", "SystemCount").c_str(), 0, 0); - - if ( systemCount == 0 ) - { - cout << "ERROR: SystemCount in ./systems/CalpontSystems.xml equal to 0" << endl; - exit(-1); - } - } - catch (...) - { - cout << "ERROR: Problem getting SystemCount from ./systems/CalpontSystems.xml" << endl; - exit(-1); - } - - bool FOUND = false; - - for ( int i = 1 ; i < systemCount + 1 ; i++) - { - string SystemName = "SystemName" + oam.itoa(i); - string oamParentModule = "OAMParentModule" + oam.itoa(i); - string SystemPassword = "SystemPassword" + oam.itoa(i); - string SystemUser = "SystemUser" + oam.itoa(i); - - string tempSystem; - - try - { - tempSystem = sysConfig->getConfig("NetworkConfig", SystemName ); - } - catch (...) - { - cout << "ERROR: Problem getting SystemName from ./systems/CalpontSystems.xml" << endl; - exit(-1); - } - - if ( tempSystem == systemName || systemName == "all") - { - try - { - installParentModuleHostName = sysConfig->getConfig("NetworkConfig", oamParentModule ); - password = sysConfig->getConfig("NetworkConfig", SystemPassword ); - systemUser = sysConfig->getConfig("NetworkConfig", SystemUser ); - FOUND = true; - } - catch (...) - { - cout << "ERROR: Problem getting SystemName from ./systems/CalpontSystems.xml" << endl; - exit(-1); - } - - if ( tempSystem == "unassigned") - continue; - - //get if root or no-root user install - string installDir = "/usr/local"; - - if ( systemUser != "root" ) - installDir = "/home/" + systemUser; - - installDir = installDir + "/Calpont"; - - string cmd; - - if (!VERBOSE) - cmd = "./remote_command.sh " + installParentModuleHostName + " " + systemUser + " " + password + " 'cat " + installDir + "/releasenum' 'release=' cat: 10 " + debug_flag; - else - cmd = "./remote_command.sh " + installParentModuleHostName + " " + systemUser + " " + password + " '" + installDir + "/bin/mcsadmin getcalpontsoftware' 'tools' Error 10 " + debug_flag; - - cout << "***** Calpont InfiniDB Package Release Information for stack '" << tempSystem << "' *****" << endl << endl; - - system(cmd.c_str()); - - cout << endl; - - cout << "***** Status of stack '" << tempSystem << "' *****" << endl << endl; - - cmd = "./remote_command.sh " + installParentModuleHostName + " " + systemUser + " " + password + " '" + installDir + "/bin/mcsadmin getsystems' dummy Error 10 " + debug_flag; - - system(cmd.c_str()); - - cout << endl << endl; - } - } - - if ( !FOUND ) - { - cout << "ERROR: System Name '" + systemName + "' not in ./systems/CalpontSystems.xml" << endl; - exit(-1); - } -} diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 78fa3f889..b512eeb20 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -43,7 +43,7 @@ checkContinue() { helpPrint () { ################################################################################ echo "" - echo "This is the MariaDB ColumnStore Cluster System Test tool." + echo "This is the MariaDB ColumnStore Cluster System Test Tool." echo "" echo "It will run a set of test to validate the setup of the MariaDB Columnstore system." echo "This can be run prior to the install of MariaDB ColumnStore to make sure the" @@ -711,6 +711,49 @@ checkTime() fi } +checkMysqlPassword() +{ + # Locale check + # + echo "" + echo "** Run MariaDB Console Password check" + echo "" + + #get MariaDB password + pass=true + `$COLUMNSTORE_INSTALL_DIR/mysql/mysql-Columnstore start > /dev/null 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/mariadb-command-line.sh > /dev/null 2>&1` + if [ "$?" -eq 2 ]; then + echo "${bold}Failed${normal}, Local Node MariaDB login failed with missing password file, /root/.my.cnf" + fi + + if [ "$IPADDRESSES" != "" ]; then + `/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/mariadb-command-line.sh /tmp/.` + + for ipadd in "${NODE_IPADDRESS[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD $COLUMNSTORE_INSTALL_DIR/mysql/mysql-Columnstore start > /dev/null 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD /tmp/mariadb-command-line.sh 1 > /tmp/remote_scp_put_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "Error running remote_scp_put.sh to $ipadd Node, check /tmp/remote_scp_put_check" + exit 1 + else + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD /tmp/mariadb-command-line.sh 1 > /tmp/remote_command_check` + `cat /tmp/remote_command_check | grep "ERROR - PASSWORD" > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node MariaDB login failed with missing password file, /root/.my.cnf" + pass=false + fi + fi + done + fi + + if ! $pass; then + checkContinue + else + echo "Passed, no problems detected with a MariaDB password being set without an associated /root/.my.cnf" + fi +} + checkPackages() { # @@ -722,6 +765,7 @@ checkPackages() echo "" declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "perl-DBD-MySQL") + declare -a CENTOS_PKG_NOT=("mariadb-libs") if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then if [ ! `which yum 2>/dev/null` ] ; then @@ -757,6 +801,24 @@ checkPackages() checkContinue fi + #check for package that shouldnt be installed + pass=true + for PKG in "${CENTOS_PKG_NOT[@]}"; do + `yum list installed "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep Installed > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if [ $pass == true ] ; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi + echo "" pass=true if [ "$IPADDRESSES" != "" ]; then @@ -792,11 +854,37 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${CENTOS_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + rc="$?" + if [ $rc -eq 2 ] ; then + echo "${bold}Failed${normal}, $ipadd Node, 'yum' not installed" + pass=false + REPORTPASS=false + break + elif [ $rc -ne 1 ] ; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "perl-DBD-mysql") + declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18") if [ "$OS" == "suse12" ]; then if [ ! `which rpm 2>/dev/null` ] ; then @@ -821,6 +909,24 @@ checkPackages() else checkContinue fi + + #check for package that shouldnt be installed + pass=true + for PKG in "${SUSE_PKG_NOT[@]}"; do + `rpm -qi "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep "not installed" > /dev/null 2>&1` + if [ "$?" -ne 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi fi echo "" @@ -844,11 +950,32 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${SUSE_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + rc="$?" + if [ $rc -eq 0 ] ; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libdbd-mysql-perl") + declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") if [ "$OS" == "ubuntu16" ] ; then if [ ! `which dpkg 2>/dev/null` ] ; then @@ -873,6 +1000,24 @@ checkPackages() else checkContinue fi + + #check for package that shouldnt be installed + pass=true + for PKG in "${UBUNTU_PKG_NOT[@]}"; do + `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi fi echo "" @@ -909,11 +1054,45 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${UBUNTU_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + else + `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" + pass=false + break + else + `cat pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + fi + + `rm -f pkg_check` + fi + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libdbd-mysql-perl") + declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian8" ]; then if [ ! `which dpkg 2>/dev/null` ] ; then @@ -938,6 +1117,24 @@ checkPackages() else checkContinue fi + + #check for package that shouldnt be installed + pass=true + for PKG in "${DEBIAN_PKG_NOT[@]}"; do + `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi fi echo "" @@ -974,11 +1171,45 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${DEBIAN_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + else + `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" + pass=false + break + else + `cat pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + fi + + `rm -f pkg_check` + fi + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") + declare -a DEBIAN9_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian9" ]; then if [ ! `which dpkg 2>/dev/null` ] ; then @@ -1003,6 +1234,25 @@ checkPackages() else checkContinue fi + + #check for package that shouldnt be installed + pass=true + for PKG in "${DEBIAN9_PKG_NOT[@]}"; do + `dpkg -s "$PKG" > /tmp/pkg_check 2>&1` + `cat /tmp/pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, Local Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + REPORTPASS=false + fi + done + + if $pass; then + echo "Local Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + fi + fi echo "" @@ -1039,6 +1289,39 @@ checkPackages() pass=true fi echo "" + + #check for package that shouldnt be installed + for PKG in "${DEBIAN9_PKG_NOT[@]}"; do + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + if [ "$?" -ne 0 ]; then + echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + else + `cat /tmp/remote_command_check | grep 'command not found' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node ${bold}dpkg${normal} package not installed" + pass=false + break + else + `cat pkg_check | grep 'install ok installed' > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is installed, please un-install" + pass=false + fi + + `rm -f pkg_check` + fi + fi + done + + if $pass; then + echo "$ipadd Node - Passed, all packages that should not be installed aren't installed" + else + checkContinue + pass=true + fi + echo "" + done fi fi @@ -1047,7 +1330,7 @@ checkPackages() } echo "" -echo "*** This is the MariaDB Columnstore Cluster System test tool ***" +echo "*** This is the MariaDB Columnstore Cluster System Test Tool ***" echo "" checkLocalOS @@ -1063,11 +1346,13 @@ if [ "$IPADDRESSES" != "" ]; then checkPorts checkTime fi + +checkMysqlPassword checkPackages if [ $REPORTPASS == true ] ; then echo "" - echo "*** Finished Validation of the Cluster, all Test Passed ***" + echo "*** Finished Validation of the Cluster, all Tests Passed ***" echo "" exit 0 else diff --git a/utils/common/crashtrace.cpp b/utils/common/crashtrace.cpp new file mode 100644 index 000000000..2b39ed07c --- /dev/null +++ b/utils/common/crashtrace.cpp @@ -0,0 +1,51 @@ +/* Copyright (C) 2018 MariaDB Corporaton + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#include +#include +#include +#include +#include + +void fatalHandler(int sig) +{ + char filename[128]; + void* addrs[128]; + snprintf(filename, 128, "/var/log/mariadb/columnstore/trace/%s.%d.log", program_invocation_short_name, getpid()); + FILE* logfile = fopen(filename, "w"); + char s[30]; + struct tm tim; + time_t now; + now = time(NULL); + tim = *(localtime(&now)); + strftime(s, 30, "%F %T", &tim); + fprintf(logfile, "Date/time: %s\n", s); + fprintf(logfile, "Signal: %d\n\n", sig); + fflush(logfile); + int fd = fileno(logfile); + int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0])); + backtrace_symbols_fd(addrs, count, fd); + fclose(logfile); + struct sigaction sigact; + memset(&sigact, 0, sizeof(sigact)); + sigact.sa_handler = SIG_DFL; + sigaction(sig, &sigact, NULL); + raise(sig); +} diff --git a/utils/common/crashtrace.h b/utils/common/crashtrace.h new file mode 100644 index 000000000..3b9cb4036 --- /dev/null +++ b/utils/common/crashtrace.h @@ -0,0 +1,18 @@ +/* Copyright (C) 2018 MariaDB Corporaton + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +void fatalHandler(int sig); diff --git a/utils/configcpp/libconfigcpp.vcxproj b/utils/configcpp/libconfigcpp.vcxproj index 4856a76cd..4f7510e5f 100644 --- a/utils/configcpp/libconfigcpp.vcxproj +++ b/utils/configcpp/libconfigcpp.vcxproj @@ -272,7 +272,7 @@ - + @@ -300,4 +300,4 @@ - \ No newline at end of file + diff --git a/utils/configcpp/libconfigcpp.vcxproj.filters b/utils/configcpp/libconfigcpp.vcxproj.filters index 33ce5c88d..706be8841 100644 --- a/utils/configcpp/libconfigcpp.vcxproj.filters +++ b/utils/configcpp/libconfigcpp.vcxproj.filters @@ -41,7 +41,7 @@ Header Files - + Header Files @@ -56,4 +56,4 @@ Resource Files - \ No newline at end of file + diff --git a/utils/libmysql_client/libmysql_client.cpp b/utils/libmysql_client/libmysql_client.cpp index 03408841d..c12abafdf 100644 --- a/utils/libmysql_client/libmysql_client.cpp +++ b/utils/libmysql_client/libmysql_client.cpp @@ -68,7 +68,7 @@ int LibMySQL::init(const char* h, unsigned int p, const char* u, const char* w, if (!(TLSCA.empty() || TLSClientCert.empty() || TLSClientKey.empty())) { mysql_ssl_set(fCon, TLSClientKey.c_str(), TLSClientCert.c_str(), - TLSCA.c_str(), NULL, NULL); + TLSCA.c_str(), NULL, NULL); } if (fCon != NULL) @@ -117,7 +117,7 @@ int LibMySQL::run(const char* query) return ret; } -void LibMySQL::handleMySqlError(const char* errStr, unsigned int errCode) +void LibMySQL::handleMySqlError(const char* errStr, unsigned int errCode) { ostringstream oss; oss << errStr << "(" << errCode << ")"; diff --git a/utils/querystats/querystats.cpp b/utils/querystats/querystats.cpp index 4d45be4b7..306ed7c11 100644 --- a/utils/querystats/querystats.cpp +++ b/utils/querystats/querystats.cpp @@ -198,7 +198,7 @@ void QueryStats::insert() ret = mysql.init(host.c_str(), port, user.c_str(), pwd.c_str(), SCHEMA.c_str()); - if (ret != 0) + if (ret != 0) mysql.handleMySqlError(mysql.getError().c_str(), ret); // escape quote characters @@ -267,12 +267,12 @@ uint32_t QueryStats::userPriority(string _host, const string _user) ret = mysql.init(host.c_str(), port, user.c_str(), pwd.c_str(), SCHEMA.c_str()); - if (ret != 0) + if (ret != 0) mysql.handleMySqlError(mysql.getError().c_str(), ret); // get the part of host string befor ':' if there is. size_t pos = _host.find(':', 0); - + if (pos != string::npos) _host = _host.substr(0, pos); @@ -288,18 +288,19 @@ uint32_t QueryStats::userPriority(string _host, const string _user) << "') and upper(a.priority) = upper(b.priority)"; ret = mysql.run(query.str().c_str()); - if (ret != 0) + + if (ret != 0) mysql.handleMySqlError(mysql.getError().c_str(), ret); char** rowIn; rowIn = mysql.nextRow(); - - if(rowIn) + + if (rowIn) { fPriority = rowIn[0]; fPriorityLevel = atoi(rowIn[1]); } - + return fPriorityLevel; } diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index dac387cbd..8b3b40ecd 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -206,6 +206,25 @@ uint64_t ThreadPool::invoke(const Functor_T& threadfunc) bAdded = true; } + if (fDebug) + { + ostringstream oss; + oss << "invoke thread " << " on " << fName + << " max " << fMaxThreads + << " queue " << fQueueSize + << " threads " << fThreadCount + << " running " << fIssued + << " waiting " << (waitingFunctorsSize - fIssued) + << " total " << waitingFunctorsSize; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format( args ); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage( message ); + } + // fQueueSize = 0 disables the queue and is an indicator to allow any number of threads to actually run. if (fThreadCount < fMaxThreads || fQueueSize == 0) { @@ -214,20 +233,6 @@ uint64_t ThreadPool::invoke(const Functor_T& threadfunc) lock1.unlock(); fThreads.create_thread(beginThreadFunc(*this)); - if (fDebug) - { - ostringstream oss; - oss << "invoke: Starting thread " << fThreadCount << " max " << fMaxThreads - << " queue " << fQueueSize; - logging::Message::Args args; - logging::Message message(0); - args.add(oss.str()); - message.format( args ); - logging::LoggingID lid(22); - logging::MessageLog ml(lid); - ml.logWarningMessage( message ); - } - if (bAdded) break; @@ -314,6 +319,26 @@ void ThreadPool::beginThread() throw() { Container_T::iterator todo = fNextFunctor++; ++fIssued; + + if (fDebug) + { + ostringstream oss; + oss << "starting thread " << " on " << fName + << " max " << fMaxThreads + << " queue " << fQueueSize + << " threads " << fThreadCount + << " running " << fIssued + << " waiting " << (waitingFunctorsSize - fIssued) + << " total " << waitingFunctorsSize; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format( args ); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage( message ); + } + lock1.unlock(); try @@ -466,7 +491,7 @@ void ThreadPoolMonitor::operator()() << setw(4) << tv.tv_usec / 100 << " Name " << fPool->fName << " Active " << fPool->waitingFunctorsSize - << " Most " << fPool->fThreadCount + << " ThdCnt " << fPool->fThreadCount << " Max " << fPool->fMaxThreads << " Q " << fPool->fQueueSize << endl; diff --git a/utils/winport/win_setup_mysql_part2.sql b/utils/winport/win_setup_mysql_part2.sql index 894e1583d..0a2a972fb 100644 --- a/utils/winport/win_setup_mysql_part2.sql +++ b/utils/winport/win_setup_mysql_part2.sql @@ -9,6 +9,9 @@ CREATE FUNCTION calonlinealter RETURNS INTEGER SONAME 'libcalmysql.dll'; CREATE FUNCTION calviewtablelock RETURNS STRING SONAME 'libcalmysql.dll'; CREATE FUNCTION calcleartablelock RETURNS STRING SONAME 'libcalmysql.dll'; CREATE FUNCTION calgetsqlcount RETURNS STRING SONAME 'libcalmysql.dll'; +CREATE FUNCTION mcssystemready RETURNS INTEGER SONAME 'libcalmysql.dll'; +CREATE FUNCTION mcssystemreadonly RETURNS INTEGER SONAME 'libcalmysql.dll'; +CREATE FUNCTION mcswritessuspended RETURNS INTEGER SONAME 'libcalmysql.dll'; create database if not exists calpontsys; use calpontsys; diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 2ff7e9bfc..8182ffc33 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -39,7 +39,7 @@ install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) ########### next target ############### -set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp) +set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp ../../utils/common/crashtrace.cpp) add_executable(controllernode ${controllernode_SRCS}) @@ -50,7 +50,7 @@ install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### -set(workernode_SRCS slavenode.cpp) +set(workernode_SRCS slavenode.cpp ../../utils/common/crashtrace.cpp) add_executable(workernode ${workernode_SRCS}) diff --git a/versioning/BRM/masternode.cpp b/versioning/BRM/masternode.cpp index 046608ee5..cb3eb5024 100644 --- a/versioning/BRM/masternode.cpp +++ b/versioning/BRM/masternode.cpp @@ -35,6 +35,8 @@ #include "brmtypes.h" #include "utils_utf8.h" +#include "crashtrace.h" + #define MAX_RETRIES 10 BRM::MasterDBRMNode* m; @@ -135,6 +137,13 @@ int main(int argc, char** argv) signal(SIGUSR1, restart); signal(SIGPIPE, SIG_IGN); #endif + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); idbdatafile::IDBPolicy::configIDBPolicy(); diff --git a/versioning/BRM/slavenode.cpp b/versioning/BRM/slavenode.cpp index ad669ccef..1fd89ff7d 100644 --- a/versioning/BRM/slavenode.cpp +++ b/versioning/BRM/slavenode.cpp @@ -37,6 +37,8 @@ #include "utils_utf8.h" #include "IDBPolicy.h" +#include "crashtrace.h" + using namespace BRM; using namespace std; @@ -125,6 +127,13 @@ int main(int argc, char** argv) signal(SIGPIPE, SIG_IGN); #endif + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + if (!(argc >= 3 && (arg = argv[2]) == "fg")) err = fork(); diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index 20a3a4f16..cb11ff375 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -14,7 +14,8 @@ set(WriteEngineServer_SRCS we_dmlcommandproc.cpp we_cleartablelockcmd.cpp we_cpifeederthread.cpp - we_getfilesizes.cpp) + we_getfilesizes.cpp + ../../utils/common/crashtrace.cpp) add_executable(WriteEngineServer ${WriteEngineServer_SRCS}) diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 6f8053b00..bf5adf906 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2218,6 +2218,26 @@ uint8_t WE_DMLCommandProc::commitBatchAutoOn(messageqcpp::ByteStream& bs, std::s cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); TableMetaData::removeTableMetaData(tableOid); + + // MCOL-1160 For API bulk insert flush the PrimProc cached dictionary + // blocks tounched + std::tr1::unordered_map::iterator mapIter; + mapIter = fWEWrapper.getDictMap().find(txnID); + + if (mapIter != fWEWrapper.getDictMap().end()) + { + std::set::iterator lbidIter; + std::vector dictFlushBlks; + + for (lbidIter = (*mapIter).second.begin(); lbidIter != (*mapIter).second.end(); lbidIter++) + { + dictFlushBlks.push_back((*lbidIter)); + } + + cacheutils::flushPrimProcAllverBlocks(dictFlushBlks); + fWEWrapper.getDictMap().erase(txnID); + } + return rc; } diff --git a/writeengine/server/we_server.cpp b/writeengine/server/we_server.cpp index af286f453..31e0d8792 100644 --- a/writeengine/server/we_server.cpp +++ b/writeengine/server/we_server.cpp @@ -51,6 +51,8 @@ using namespace oam; #include "utils_utf8.h" #include "dbrm.h" +#include "crashtrace.h" + namespace { void added_a_pm(int) @@ -103,6 +105,9 @@ int main(int argc, char** argv) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("WriteEngineServ"); + printf ("Locale is : %s\n", systemLang.c_str() ); //set BUSY_INIT state @@ -127,6 +132,12 @@ int main(int argc, char** argv) sigaction(SIGHUP, &sa, 0); sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, 0); + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = fatalHandler; + sigaction(SIGSEGV, &sa, 0); + sigaction(SIGABRT, &sa, 0); + sigaction(SIGFPE, &sa, 0); #endif // Init WriteEngine Wrapper (including Config Columnstore.xml cache) diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index e04d3c7f6..cb6eb3745 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1650,6 +1650,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, Column curCol; ColStruct curColStruct; ColStructList newColStructList; + std::vector colNewValueList; DctnryStructList newDctnryStructList; HWM hwm = 0; HWM oldHwm = 0; @@ -1660,6 +1661,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, bool newExtent = false; RIDList ridList; ColumnOp* colOp = NULL; + std::vector dictLbids; // Set tmp file suffix to modify HDFS db file bool useTmpSuffix = false; @@ -2115,6 +2117,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, timer.stop("tokenize"); #endif memcpy(colValPtr, &dctTuple.token, 8); + dictLbids.push_back(dctTuple.token.fbo); } dctStr_iter++; @@ -2171,6 +2174,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, timer.stop("tokenize"); #endif memcpy(colValPtr, &dctTuple.token, 8); + dictLbids.push_back(dctTuple.token.fbo); } dctStr_iter++; @@ -2297,6 +2301,19 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); } + //-------------------------------------------------------------------------- + //Prepare the valuelist for the new extent + //-------------------------------------------------------------------------- + + for (unsigned i = 1; i <= totalColumns; i++) + { + // Copy values to second value list + for (uint64_t j = rowsLeft; j > 0; j--) + { + colNewValueList.push_back(colValueList[(totalRow * i) - j]); + } + } + // end of allocate row id #ifdef PROFILE @@ -2337,6 +2354,26 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, } } } + + // If we create a new extent for this batch + for (unsigned i = 0; i < newColStructList.size(); i++) + { + colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + width = newColStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) + { + if (curFbo != lastFbo) + { + RETURN_ON_ERROR(AddLBIDtoList(txnid, + lbids, + colDataTypes, + newColStructList[i], + curFbo)); + } + } + } } if (lbids.size() > 0) @@ -2346,7 +2383,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, // Write row(s) to database file(s) //---------------------------------------------------------------------- bool versioning = !(isAutoCommitOn && insertSelect); - rc = writeColumnRecBinary(txnid, colStructList, colValueList, rowIdArray, newColStructList, tableOid, useTmpSuffix, versioning); // @bug 5572 HDFS tmp file + AddDictToList(txnid, dictLbids); + rc = writeColumnRecBinary(txnid, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, useTmpSuffix, versioning); // @bug 5572 HDFS tmp file } return rc; @@ -5085,6 +5123,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, std::vector& colValueList, RID* rowIdArray, const ColStructList& newColStructList, + std::vector& newColValueList, const int32_t tableOid, bool useTmpSuffix, bool versioning) @@ -5095,7 +5134,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, Column curCol; ColStructList::size_type totalColumn; ColStructList::size_type i; - size_t totalRow; + size_t totalRow1, totalRow2; setTransId(txnid); @@ -5103,11 +5142,22 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, #ifdef PROFILE StopWatch timer; #endif - totalRow = colValueList.size() / totalColumn; - valArray = malloc(sizeof(uint64_t) * totalRow); + totalRow1 = colValueList.size() / totalColumn; - if (totalRow == 0) + if (newColValueList.size() > 0) + { + totalRow2 = newColValueList.size() / newColStructList.size(); + totalRow1 -= totalRow2; + } + else + { + totalRow2 = 0; + } + + valArray = malloc(sizeof(uint64_t) * totalRow1); + + if (totalRow1 == 0) return rc; TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); @@ -5160,7 +5210,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, if (versioning) { rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], - colStructList[i].colWidth, totalRow, firstPart, rangeList); + colStructList[i].colWidth, totalRow1, firstPart, rangeList); if (rc != NO_ERROR) { @@ -5181,9 +5231,9 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, uint16_t tmp16; uint32_t tmp32; - for (size_t j = 0; j < totalRow; j++) + for (size_t j = 0; j < totalRow1; j++) { - uint64_t curValue = colValueList[(totalRow * i) + j]; + uint64_t curValue = colValueList[((totalRow1 + totalRow2) * i) + j]; switch (colStructList[i].colType) { @@ -5226,7 +5276,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, #ifdef PROFILE timer.start("writeRow "); #endif - rc = colOp->writeRow(curCol, totalRow, firstPart, valArray); + rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); #ifdef PROFILE timer.stop("writeRow "); #endif @@ -5242,7 +5292,149 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, } // end of for (i = 0 if (valArray != NULL) + { free(valArray); + valArray = NULL; + } + + // MCOL-1176 - Write second extent + if (totalRow2) + { + valArray = malloc(sizeof(uint64_t) * totalRow2); + + for (i = 0; i < newColStructList.size(); i++) + { + //@Bug 2205 Check if all rows go to the new extent + //Write the first batch + RID* secondPart = rowIdArray + totalRow1; + ColumnOp* colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, newColStructList[i].colWidth, + newColStructList[i].colDataType, newColStructList[i].colType, newColStructList[i].dataOid, + newColStructList[i].fCompressionType, newColStructList[i].fColDbRoot, + newColStructList[i].fColPartition, newColStructList[i].fColSegment); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(newColStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = newColStructList[i].fColDbRoot; + aExt.partNum = newColStructList[i].fColPartition; + aExt.segNum = newColStructList[i].fColSegment; + aExt.compType = newColStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); + } + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + // handling versioning + vector rangeList; + + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, newColStructList[i], + newColStructList[i].colWidth, totalRow2, secondPart, rangeList); + + if (rc != NO_ERROR) + { + if (newColStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + //totalRow1 -= totalRow2; + // have to init the size here + // nullArray = (bool*) malloc(sizeof(bool) * totalRow); + uint8_t tmp8; + uint16_t tmp16; + uint32_t tmp32; + + for (size_t j = 0; j < totalRow2; j++) + { + uint64_t curValue = newColValueList[(totalRow2 * i) + j]; + + switch (newColStructList[i].colType) + { + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + ((uint64_t*)valArray)[j] = curValue; + break; + + case WriteEngine::WR_INT: + case WriteEngine::WR_UINT: + case WriteEngine::WR_FLOAT: + tmp32 = curValue; + ((uint32_t*)valArray)[j] = tmp32; + break; + + case WriteEngine::WR_ULONGLONG: + case WriteEngine::WR_LONGLONG: + case WriteEngine::WR_DOUBLE: + case WriteEngine::WR_TOKEN: + ((uint64_t*)valArray)[j] = curValue; + break; + + case WriteEngine::WR_BYTE: + case WriteEngine::WR_UBYTE: + tmp8 = curValue; + ((uint8_t*)valArray)[j] = tmp8; + break; + + case WriteEngine::WR_SHORT: + case WriteEngine::WR_USHORT: + tmp16 = curValue; + ((uint16_t*)valArray)[j] = tmp16; + break; + } + } + + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow2, secondPart, valArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + colOp->closeColumnFile(curCol); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + // check error + if (rc != NO_ERROR) + break; + + } // end of for (i = 0 + } + + if (valArray != NULL) + free(valArray); + #ifdef PROFILE timer.finish(); @@ -5766,6 +5958,7 @@ int WriteEngineWrapper::rollbackCommon(const TxnID& txnid, int sessionId) // BUG 4312 RemoveTxnFromLBIDMap(txnid); + RemoveTxnFromDictMap(txnid); config::Config* config = config::Config::makeConfig(); prefix = config->getConfig("SystemConfig", "DBRMRoot"); @@ -6013,6 +6206,7 @@ int WriteEngineWrapper::rollbackVersion(const TxnID& txnid, int sessionId) { // BUG 4312 RemoveTxnFromLBIDMap(txnid); + RemoveTxnFromDictMap(txnid); return BRMWrapper::getInstance()->rollBackVersion(txnid, sessionId); } @@ -6114,6 +6308,7 @@ int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, int WriteEngineWrapper::flushDataFiles(int rc, const TxnID txnId, std::map& columnOids) { RemoveTxnFromLBIDMap(txnId); + RemoveTxnFromDictMap(txnId); for (int i = 0; i < TOTAL_COMPRESS_OP; i++) { @@ -6129,6 +6324,28 @@ int WriteEngineWrapper::flushDataFiles(int rc, const TxnID txnId, std::map& lbids) +{ + std::tr1::unordered_map::iterator mapIter; + + mapIter = m_dictLBIDMap.find(txnid); + + if (mapIter == m_dictLBIDMap.end()) + { + dictLBIDRec_t tempRecord; + tempRecord.insert(lbids.begin(), lbids.end()); + m_dictLBIDMap[txnid] = tempRecord; + return; + } + else + { + dictLBIDRec_t& txnRecord = mapIter->second; + txnRecord.insert(lbids.begin(), lbids.end()); + } + +} + /*********************************************************** * DESCRIPTION: * Add an lbid to a list of lbids for sending to markExtentsInvalid. @@ -6225,6 +6442,17 @@ int WriteEngineWrapper::AddLBIDtoList(const TxnID txnid, return rtn; } +void WriteEngineWrapper::RemoveTxnFromDictMap(const TxnID txnid) +{ + std::tr1::unordered_map::iterator mapIter; + + mapIter = m_dictLBIDMap.find(txnid); + + if (mapIter != m_dictLBIDMap.end()) + { + m_dictLBIDMap.erase(txnid); + } +} /*********************************************************** * DESCRIPTION: diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index ad98a5b94..f0fe5c995 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -85,6 +85,7 @@ struct TxnLBIDRec }; typedef boost::shared_ptr SP_TxnLBIDRec_t; +typedef std::set dictLBIDRec_t; /** Class WriteEngineWrapper */ class WriteEngineWrapper : public WEObj @@ -433,6 +434,10 @@ public: return m_txnLBIDMap; }; + std::tr1::unordered_map& getDictMap() + { + return m_dictLBIDMap; + }; /** * @brief Flush the ChunkManagers. */ @@ -689,6 +694,7 @@ private: int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, std::vector& colValueList, RID* rowIdArray, const ColStructList& newColStructList, + std::vector& newColValueList, const int32_t tableOid, bool useTmpSuffix, bool versioning = true); @@ -712,6 +718,9 @@ private: const RID filesPerColumnPartition, const RID extentsPerSegmentFile, const RID extentRows, uint16_t startDBRoot, unsigned dbrootCnt); + void AddDictToList(const TxnID txnid, std::vector& lbids); + void RemoveTxnFromDictMap(const TxnID txnid); + // Bug 4312: We use a hash set to hold the set of starting LBIDS for a given // txn so that we don't waste time marking the same extent as invalid. This // list should be trimmed if it gets too big. @@ -732,6 +741,10 @@ private: // This is a Map of sets of LBIDS for each transaction. A Transaction's list will be removed upon commit or rollback. std::tr1::unordered_map m_txnLBIDMap; + // MCOL-1160: We need to track dictionary LBIDs so we can tell PrimProc + // to flush the blocks after an API bulk-write. + std::tr1::unordered_map m_dictLBIDMap; + ColumnOp* m_colOp[TOTAL_COMPRESS_OP]; // column operations Dctnry* m_dctnry[TOTAL_COMPRESS_OP]; // dictionary operations OpType m_opType; // operation type