From 44c06f2cc3002d0d2ecf39bc00add6c4dff72a00 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 12 Dec 2017 16:32:49 -0600 Subject: [PATCH 1/6] MCOL-1106 - increased mysqld startup timeouts --- dbcon/mysql/mysql-Columnstore | 7 +++- oam/install_scripts/binary_installer.sh | 54 +++---------------------- oam/install_scripts/user_installer.sh | 2 +- 3 files changed, 11 insertions(+), 52 deletions(-) diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore index de090329f..02a0de162 100755 --- a/dbcon/mysql/mysql-Columnstore +++ b/dbcon/mysql/mysql-Columnstore @@ -61,7 +61,7 @@ datadir=$basedir/db # Value here is overriden by value in my.cnf. # 0 means don't wait at all # Negative numbers mean to wait indefinitely -service_startup_timeout=30 +service_startup_timeout=90 # Lock directory for RedHat / SuSE. lockdir='/var/lock/subsys' @@ -262,6 +262,8 @@ wait_for_gone () { done log_failure_msg + kill_by_pid + return 1 } @@ -310,7 +312,7 @@ fi kill_by_pid() { # let's see if we can kill the 2 mysql procs by hand # get the our mysql from ps - eval $(ps -ef | grep "$COLUMNSTORE_INSTALL_DIR/mysql//bin/mysqld " | grep -v grep | head -1 | awk '{printf "pid=%d\n", $2}') + eval $(ps -ef | grep "$COLUMNSTORE_INSTALL_DIR/mysql/bin/mysqld" | grep -v grep | head -1 | awk '{printf "pid=%d\n", $2}') if [ -n "$pid" ]; then ppid=$(ps -o ppid= -p $pid) @@ -369,6 +371,7 @@ case "$mode" in wait_for_gone $mysqld_pid "$mysqld_pid_file_path"; return_value=$? else log_failure_msg "MySQL server process #$mysqld_pid is not running!" + kill_by_pid rm "$mysqld_pid_file_path" fi diff --git a/oam/install_scripts/binary_installer.sh b/oam/install_scripts/binary_installer.sh index ead0dde37..280b10c22 100644 --- a/oam/install_scripts/binary_installer.sh +++ b/oam/install_scripts/binary_installer.sh @@ -238,11 +238,15 @@ send_user "\n" # # Start module installer to setup Customer OS files # +if { $SERVERTYPE == "2" || $SERVERTYPE == "pmwithum" } { + set MODULETYPE "um" +} + send_user "Run Module Installer " send " \n" send date\n send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=$MODULETYPE --port=$MYSQLPORT --installdir=$INSTALLDIR'\n" -set timeout 60 +set timeout 120 expect { "word: " { send "$PASSWORD\n" exp_continue @@ -256,54 +260,6 @@ expect { } send_user "\n" -if { $MODULETYPE == "um" || $SERVERTYPE == "2" || $SERVERTYPE == "pmwithum" } { - # - # run mysql setup scripts - # - send_user "Run MySQL Setup Scripts on Module " - send " \n" - send date\n - send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/post-mysqld-install --installdir=$INSTALLDIR'\n" - set timeout 60 - expect { - "word: " { send "$PASSWORD\n" - exp_continue - } - "passphrase" { send "$PASSWORD\n" - exp_continue - } - "Exit status 0" { send_user "DONE" } - "Exit status 1" { send_user "ERROR: scp failed" ; exit 1 } - "ERROR" { send_user "ERROR: Daemon failed to run"; - exit 1 } - "FAILED" { send_user "ERROR: Daemon failed to run"; - exit 1 } - timeout { send_user "ERROR: Timeout\n" ; exit 2 } - } - - send " \n" - send date\n - send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/post-mysql-install --installdir=$INSTALLDIR'\n" - set timeout 120 - expect { - "word: " { send "$PASSWORD\n" - exp_continue - } - "passphrase" { send "$PASSWORD\n" - exp_continue - } - "Exit status 0" { send_user "DONE" } - "Exit status 1" { send_user "ERROR: scp failed" ; exit 1 } - "ERROR" { send_user "ERROR: Daemon failed to run"; - exit 1 } - "FAILED" { send_user "ERROR: Daemon failed to run"; - exit 1 } - timeout { send_user "ERROR: Timeout\n" ; exit 2 } - } - send_user "\n" -} - - send_user "\nInstallation Successfully Completed on '$MODULE'\n" exit 0 diff --git a/oam/install_scripts/user_installer.sh b/oam/install_scripts/user_installer.sh index a31367aed..4d30492d8 100644 --- a/oam/install_scripts/user_installer.sh +++ b/oam/install_scripts/user_installer.sh @@ -255,7 +255,7 @@ send_user "\n" # send_user "Run Module Installer " send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=um --port=$MYSQLPORT'\n" -set timeout 60 +set timeout 120 expect { "word: " { send "$PASSWORD\n" exp_continue From de24edcf8f1b7499525be9d896133f04b6a80bf3 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 14 Dec 2017 14:47:47 -0600 Subject: [PATCH 2/6] Update README --- README | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README b/README index 291da0ab7..8186d1ed7 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.11 -MariaDB ColumnStore 1.0.11 is the development version of MariaDB ColumnStore. +This is MariaDB ColumnStore 1.0.12 +MariaDB ColumnStore 1.0.12 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.11 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.12 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 9f5bd31e37dd330ae01b5da04123985c6450e3f3 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 14 Dec 2017 14:49:58 -0600 Subject: [PATCH 3/6] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 397a1b1d8..40c1e99ec 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.11 -MariaDB ColumnStore 1.0.11 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.12 +MariaDB ColumnStore 1.0.12 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.11 is an GA release. +#MariaDB ColumnStore 1.0.12 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. From e9aff27e59f8a9dc7fec58c35e3968aa2018ec46 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 15 Dec 2017 20:33:07 +0000 Subject: [PATCH 4/6] MCOL-1116 Fix I_S.CS_FILES for missing dbroot If a dbroot is missing/offline mysqld would crash on information_schema.columnstore_files due to not catching an exception. This patch now catches the exception. --- dbcon/mysql/is_columnstore_files.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index e9479e254..ce00b8aae 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -124,6 +124,17 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) iter++; continue; } + + try + { + oam_instance.getDbrootPmConfig(iter->dbRoot, pmId); + } + catch (std::runtime_error) + { + // MCOL-1116: If we are here a DBRoot is offline/missing + iter++; + continue; + } table->field[0]->store(oid); table->field[1]->store(iter->segmentNum); table->field[2]->store(iter->partitionNum); @@ -134,7 +145,7 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) std::string DbRootPath = config->getConfig("SystemConfig", DbRootName.str()); fileSize = compressedFileSize = 0; snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", DbRootPath.c_str(), oidDirName); - oam_instance.getDbrootPmConfig(iter->dbRoot, pmId); + std::ostringstream oss; oss << "pm" << pmId << "_WriteEngineServer"; std::string client = oss.str(); From c31c836352ee921141e8bbdc5264700b59ae97a9 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 3 Jan 2018 09:20:09 +0000 Subject: [PATCH 5/6] MCOL-1085 Add crash dump to daemons This patch adds an automated crash dump which logs in /var/log/mariadb/columnstore/trace/ when one of the ColumnStore daemons crashes. --- CMakeLists.txt | 8 ++--- ddlproc/CMakeLists.txt | 2 +- ddlproc/ddlproc.cpp | 9 +++++ dmlproc/CMakeLists.txt | 3 +- dmlproc/dmlproc.cpp | 11 ++++++ exemgr/CMakeLists.txt | 2 +- exemgr/main.cpp | 14 +++++++- oamapps/serverMonitor/CMakeLists.txt | 3 +- oamapps/serverMonitor/main.cpp | 10 ++++++ primitives/primproc/CMakeLists.txt | 3 +- primitives/primproc/primproc.cpp | 11 ++++++ procmgr/CMakeLists.txt | 2 +- procmgr/main.cpp | 12 +++++++ procmon/CMakeLists.txt | 2 +- procmon/main.cpp | 10 ++++++ utils/common/crashtrace.cpp | 51 ++++++++++++++++++++++++++++ utils/common/crashtrace.h | 18 ++++++++++ versioning/BRM/CMakeLists.txt | 4 +-- versioning/BRM/masternode.cpp | 9 +++++ versioning/BRM/slavenode.cpp | 9 +++++ writeengine/server/CMakeLists.txt | 3 +- writeengine/server/we_server.cpp | 11 ++++++ 22 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 utils/common/crashtrace.cpp create mode 100644 utils/common/crashtrace.h diff --git a/CMakeLists.txt b/CMakeLists.txt index df20c3e1b..b40c663a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,12 +98,12 @@ endif() FOREACH(BUILD_TYPE RELEASE RELWITHDEBINFO MINSIZEREL) - SET(CMAKE_CXX_FLAGS_${BUILD_TYPE} "-g -O3 -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H") - SET(CMAKE_C_FLAGS_${BUILD_TYPE} "-g -O3 -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H") + SET(CMAKE_CXX_FLAGS_${BUILD_TYPE} "-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H") + SET(CMAKE_C_FLAGS_${BUILD_TYPE} "-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H") ENDFOREACH() -SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H") -SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H") +SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H") +SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H") SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") diff --git a/ddlproc/CMakeLists.txt b/ddlproc/CMakeLists.txt index a56fb4683..7c7fdf70c 100644 --- a/ddlproc/CMakeLists.txt +++ b/ddlproc/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp) +set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp) add_executable(DDLProc ${DDLProc_SRCS}) diff --git a/ddlproc/ddlproc.cpp b/ddlproc/ddlproc.cpp index 45cf09327..2589f9c29 100644 --- a/ddlproc/ddlproc.cpp +++ b/ddlproc/ddlproc.cpp @@ -61,6 +61,7 @@ using namespace execplan; #include "IDBPolicy.h" #include "utils_utf8.h" +#include "crashtrace.h" namespace fs = boost::filesystem; @@ -97,6 +98,9 @@ int main(int argc, char* argv[]) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("DDLProc"); + setupCwd(); WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_DDLPROC ); @@ -116,6 +120,11 @@ int main(int argc, char* argv[]) sigaction(SIGHUP, &ign, 0); ign.sa_handler = SIG_IGN; sigaction(SIGPIPE, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif ddlprocessor::DDLProcessor ddlprocessor(1, 20); diff --git a/dmlproc/CMakeLists.txt b/dmlproc/CMakeLists.txt index 8ed489aa8..957cfdff6 100644 --- a/dmlproc/CMakeLists.txt +++ b/dmlproc/CMakeLists.txt @@ -8,7 +8,8 @@ set(DMLProc_SRCS dmlproc.cpp dmlprocessor.cpp dmlresultbuffer.cpp - batchinsertprocessor.cpp) + batchinsertprocessor.cpp + ../utils/common/crashtrace.cpp) add_executable(DMLProc ${DMLProc_SRCS}) diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index fe542e3a4..350cc22e3 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -82,6 +82,8 @@ using namespace joblist; #include "utils_utf8.h" +#include "crashtrace.h" + namespace fs = boost::filesystem; namespace @@ -473,6 +475,9 @@ int main(int argc, char* argv[]) //BUG 5362 systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("DMLProc"); + Config* cf = Config::makeConfig(); setupCwd(); @@ -578,6 +583,12 @@ int main(int argc, char* argv[]) sigaction(SIGHUP, &ign, 0); ign.sa_handler = SIG_IGN; sigaction(SIGPIPE, &ign, 0); + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif dmlserver.start(); diff --git a/exemgr/CMakeLists.txt b/exemgr/CMakeLists.txt index 5a9f197d7..c7276ad05 100644 --- a/exemgr/CMakeLists.txt +++ b/exemgr/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp) +set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp) add_executable(ExeMgr ${ExeMgr_SRCS}) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 002ba6aab..ca837a33e 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -97,6 +97,8 @@ using namespace querytele; #include "utils_utf8.h" #include "boost/filesystem.hpp" +#include "crashtrace.h" + namespace { //If any flags other than the table mode flags are set, produce output to screeen @@ -1165,10 +1167,12 @@ public: } }; +#ifdef _MSC_VER void exit_(int) { exit(0); } +#endif void added_a_pm(int) { @@ -1213,7 +1217,6 @@ void printTotalUmMemory(int sig) void setupSignalHandlers() { #ifdef _MSC_VER - signal(SIGSEGV, exit_); signal(SIGINT, exit_); signal(SIGTERM, exit_); #else @@ -1229,6 +1232,12 @@ void setupSignalHandlers() sigaction(SIGHUP, &ign, 0); ign.sa_handler = printTotalUmMemory; sigaction(SIGUSR1, &ign, 0); + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif } @@ -1302,6 +1311,9 @@ int main(int argc, char* argv[]) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("ExeMgr"); + gDebug = 0; bool eFlg = false; int c; diff --git a/oamapps/serverMonitor/CMakeLists.txt b/oamapps/serverMonitor/CMakeLists.txt index 04296114a..81d14c9ea 100644 --- a/oamapps/serverMonitor/CMakeLists.txt +++ b/oamapps/serverMonitor/CMakeLists.txt @@ -13,7 +13,8 @@ set(ServerMonitor_SRCS procmonMonitor.cpp msgProcessor.cpp dbhealthMonitor.cpp - UMAutoSync.cpp) + UMAutoSync.cpp + ../../utils/common/crashtrace.cpp) add_executable(ServerMonitor ${ServerMonitor_SRCS}) diff --git a/oamapps/serverMonitor/main.cpp b/oamapps/serverMonitor/main.cpp index 10903d6a4..56b8e3825 100644 --- a/oamapps/serverMonitor/main.cpp +++ b/oamapps/serverMonitor/main.cpp @@ -18,6 +18,8 @@ #include "IDBPolicy.h" #include "serverMonitor.h" +#include "crashtrace.h" + using namespace std; using namespace servermonitor; using namespace oam; @@ -38,6 +40,14 @@ int main (int argc, char** argv) ServerMonitor serverMonitor; Oam oam; + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + //Launch Memory Monitor Thread and check if swap is in critical condition pthread_t memoryMonitorThread; pthread_create (&memoryMonitorThread, NULL, (void*(*)(void*)) &memoryMonitor, NULL); diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index ddfa56abc..cb5a7c163 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -18,7 +18,8 @@ set(PrimProc_SRCS primitiveserver.cpp pseudocc.cpp rtscommand.cpp - umsocketselector.cpp) + umsocketselector.cpp + ../../utils/common/crashtrace.cpp) #PrimProc_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS) diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 7fc0c36f2..200786e46 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -71,6 +71,8 @@ using namespace idbdatafile; #include "cgroupconfigurator.h" +#include "crashtrace.h" + namespace primitiveprocessor { @@ -126,6 +128,12 @@ void setupSignalHandlers() ign.sa_handler = SIG_IGN; sigaction(SIGUSR2, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + sigset_t sigset; sigemptyset(&sigset); sigaddset(&sigset, SIGPIPE); @@ -288,6 +296,9 @@ int main(int argc, char* argv[]) systemLang.find("UTF") != string::npos ) utf8 = true; + // This is unset due to the way we start it + program_invocation_short_name = const_cast("PrimProc"); + Config* cf = Config::makeConfig(); setupSignalHandlers(); diff --git a/procmgr/CMakeLists.txt b/procmgr/CMakeLists.txt index 7ced90a35..aa2fa15fc 100644 --- a/procmgr/CMakeLists.txt +++ b/procmgr/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ProcMgr_SRCS main.cpp processmanager.cpp) +set(ProcMgr_SRCS main.cpp processmanager.cpp ../utils/common/crashtrace.cpp) add_executable(ProcMgr ${ProcMgr_SRCS}) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 3c1a870fc..16c3ad5b0 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -32,6 +32,8 @@ #include "utils_utf8.h" +#include "crashtrace.h" + using namespace std; using namespace logging; using namespace messageqcpp; @@ -100,6 +102,16 @@ int main(int argc, char **argv) setlocale(LC_ALL, systemLang.c_str()); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("ProcMgr"); + + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + Oam oam; //check if root-user diff --git a/procmon/CMakeLists.txt b/procmon/CMakeLists.txt index cb6ecbb49..cb61e8044 100644 --- a/procmon/CMakeLists.txt +++ b/procmon/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### -set(ProcMon_SRCS main.cpp processmonitor.cpp) +set(ProcMon_SRCS main.cpp processmonitor.cpp ../utils/common/crashtrace.cpp) add_executable(ProcMon ${ProcMon_SRCS}) diff --git a/procmon/main.cpp b/procmon/main.cpp index 88deeeb77..25dc07d6a 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -26,6 +26,8 @@ namespace bi=boost::interprocess; #include "IDBPolicy.h" +#include "crashtrace.h" + using namespace std; using namespace messageqcpp; using namespace processmonitor; @@ -75,6 +77,14 @@ int main(int argc, char **argv) setuid(0); // set effective ID to root; ignore return status #endif + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + if (argc > 1 && string(argv[1]) == "--daemon") { if (fork() != 0) return 0; diff --git a/utils/common/crashtrace.cpp b/utils/common/crashtrace.cpp new file mode 100644 index 000000000..41626361e --- /dev/null +++ b/utils/common/crashtrace.cpp @@ -0,0 +1,51 @@ +/* Copyright (C) 2018 MariaDB Corporaton + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#include +#include +#include +#include +#include + +void fatalHandler(int sig) +{ + char filename[128]; + void* addrs[128]; + snprintf(filename, 128, "/var/log/mariadb/columnstore/trace/%s.%d.log", program_invocation_short_name, getpid()); + FILE* logfile = fopen(filename, "w"); + char s[30]; + struct tm tim; + time_t now; + now = time(NULL); + tim = *(localtime(&now)); + strftime(s,30,"%F %T",&tim); + fprintf(logfile, "Date/time: %s\n", s); + fprintf(logfile, "Signal: %d\n\n", sig); + fflush(logfile); + int fd = fileno(logfile); + int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0])); + backtrace_symbols_fd(addrs, count, fd); + fclose(logfile); + struct sigaction sigact; + memset(&sigact, 0, sizeof(sigact)); + sigact.sa_handler = SIG_DFL; + sigaction(sig, &sigact, NULL); + raise(sig); +} diff --git a/utils/common/crashtrace.h b/utils/common/crashtrace.h new file mode 100644 index 000000000..3b9cb4036 --- /dev/null +++ b/utils/common/crashtrace.h @@ -0,0 +1,18 @@ +/* Copyright (C) 2018 MariaDB Corporaton + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +void fatalHandler(int sig); diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 2ce60254d..2c904e872 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -41,7 +41,7 @@ install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) ########### next target ############### -set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp) +set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp ../../utils/common/crashtrace.cpp) add_executable(controllernode ${controllernode_SRCS}) @@ -52,7 +52,7 @@ install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### next target ############### -set(workernode_SRCS slavenode.cpp) +set(workernode_SRCS slavenode.cpp ../../utils/common/crashtrace.cpp) add_executable(workernode ${workernode_SRCS}) diff --git a/versioning/BRM/masternode.cpp b/versioning/BRM/masternode.cpp index 3932f9a37..4bdf42553 100644 --- a/versioning/BRM/masternode.cpp +++ b/versioning/BRM/masternode.cpp @@ -35,6 +35,8 @@ #include "brmtypes.h" #include "utils_utf8.h" +#include "crashtrace.h" + #define MAX_RETRIES 10 BRM::MasterDBRMNode *m; @@ -128,6 +130,13 @@ int main(int argc, char **argv) signal(SIGUSR1, restart); signal(SIGPIPE, SIG_IGN); #endif + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); idbdatafile::IDBPolicy::configIDBPolicy(); diff --git a/versioning/BRM/slavenode.cpp b/versioning/BRM/slavenode.cpp index 2d36bc3a4..5ec20f69f 100644 --- a/versioning/BRM/slavenode.cpp +++ b/versioning/BRM/slavenode.cpp @@ -37,6 +37,8 @@ #include "utils_utf8.h" #include "IDBPolicy.h" +#include "crashtrace.h" + using namespace BRM; using namespace std; @@ -117,6 +119,13 @@ int main(int argc, char **argv) signal(SIGPIPE, SIG_IGN); #endif + struct sigaction ign; + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + if (!(argc >= 3 && (arg = argv[2]) == "fg")) err = fork(); diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index 365e98036..5029cdb15 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -14,7 +14,8 @@ set(WriteEngineServer_SRCS we_dmlcommandproc.cpp we_cleartablelockcmd.cpp we_cpifeederthread.cpp - we_getfilesizes.cpp) + we_getfilesizes.cpp + ../../utils/common/crashtrace.cpp) add_executable(WriteEngineServer ${WriteEngineServer_SRCS}) diff --git a/writeengine/server/we_server.cpp b/writeengine/server/we_server.cpp index b5365fe0f..1e4415eac 100644 --- a/writeengine/server/we_server.cpp +++ b/writeengine/server/we_server.cpp @@ -51,6 +51,8 @@ using namespace oam; #include "utils_utf8.h" #include "dbrm.h" +#include "crashtrace.h" + namespace { void added_a_pm(int) @@ -96,6 +98,9 @@ int main(int argc, char** argv) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("WriteEngineServ"); + printf ("Locale is : %s\n", systemLang.c_str() ); //set BUSY_INIT state @@ -119,6 +124,12 @@ int main(int argc, char** argv) sigaction(SIGHUP, &sa, 0); sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, 0); + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = fatalHandler; + sigaction(SIGSEGV, &sa, 0); + sigaction(SIGABRT, &sa, 0); + sigaction(SIGFPE, &sa, 0); #endif // Init WriteEngine Wrapper (including Config Columnstore.xml cache) From 44989fbd74900ef045c07bd40b25d9a08a7a322f Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 10 Jan 2018 11:41:44 -0600 Subject: [PATCH 6/6] MCOL-1149 - add a main resume flag --- procmon/main.cpp | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/procmon/main.cpp b/procmon/main.cpp index 88deeeb77..f6a0d10dc 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -26,6 +26,8 @@ namespace bi=boost::interprocess; #include "IDBPolicy.h" +#include "crashtrace.h" + using namespace std; using namespace messageqcpp; using namespace processmonitor; @@ -55,9 +57,11 @@ void updateShareMemory(processStatusList* aPtr); bool runStandby = false; bool processInitComplete = false; bool rootUser = true; +bool mainResumeFlag; string USER = "root"; string PMwithUM = "n"; + //extern std::string gOAMParentModuleName; extern bool gOAMParentModuleFlag; @@ -75,6 +79,14 @@ int main(int argc, char **argv) setuid(0); // set effective ID to root; ignore return status #endif + struct sigaction ign; + + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); + if (argc > 1 && string(argv[1]) == "--daemon") { if (fork() != 0) return 0; @@ -435,13 +447,24 @@ int main(int argc, char **argv) unlink ("/var/log/mariadb/columnstore/activeAlarms"); } + //Clear mainResumeFlag + + mainResumeFlag = false; + //launch Status table control thread on 'pm' modules pthread_t statusThread; int ret = pthread_create (&statusThread, NULL, (void*(*)(void*)) &statusControlThread, NULL); if ( ret != 0 ) log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); - sleep(6); // give the Status thread time to fully initialize + //wait for flag to be set + + while(!mainResumeFlag) + { + log.writeLog(__LINE__, "WATING FOR mainResumeFlag to be set", LOG_TYPE_DEBUG); + + sleep(1); + } } SystemStatus systemstatus; @@ -723,6 +746,8 @@ int main(int argc, char **argv) } } + log.writeLog(__LINE__, "SYSTEM STATUS = " + oam.itoa(systemstatus.SystemOpState), LOG_TYPE_DEBUG); + if ( systemstatus.SystemOpState != MAN_OFFLINE && !DISABLED) { // Loop through the process list to check the process current state @@ -2026,6 +2051,10 @@ static void statusControlThread() log.writeLog(__LINE__, "Dbroot Status shared Memory allociated and Initialized", LOG_TYPE_DEBUG); } + //Set mainResumeFlag, to start up main thread + + mainResumeFlag = true; + string portName = "ProcStatusControl"; if (runStandby) { portName = "ProcStatusControlStandby";