MCOL-1085 Add crash dump to daemons

This patch adds an automated crash dump which logs in
/var/log/mariadb/columnstore/trace/ when one of the ColumnStore daemons
crashes.

CMakeLists.txt

@@ -98,12 +98,12 @@ endif()
 
 
 FOREACH(BUILD_TYPE RELEASE RELWITHDEBINFO MINSIZEREL)
-    SET(CMAKE_CXX_FLAGS_${BUILD_TYPE} "-g -O3 -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H")
+    SET(CMAKE_CXX_FLAGS_${BUILD_TYPE} "-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H")
-    SET(CMAKE_C_FLAGS_${BUILD_TYPE} "-g -O3 -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H")
+    SET(CMAKE_C_FLAGS_${BUILD_TYPE} "-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -DDBUG_OFF -DHAVE_CONFIG_H")
 ENDFOREACH()
 
-SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H")
+SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H")
-SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H")
+SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H")
 
 SET (ENGINE_LDFLAGS    "-Wl,--no-as-needed -Wl,--add-needed")  
 

ddlproc/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp)
+set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp)
 
 add_executable(DDLProc ${DDLProc_SRCS})
 

ddlproc/ddlproc.cpp

@@ -61,6 +61,7 @@ using namespace execplan;
 #include "IDBPolicy.h"
 #include "utils_utf8.h"
 
+#include "crashtrace.h"
 
 namespace fs = boost::filesystem;
 
@@ -97,6 +98,9 @@ int main(int argc, char* argv[])
 	string systemLang = "C";
 	systemLang = funcexp::utf8::idb_setlocale();
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("DDLProc");
+
     setupCwd();
 
     WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_DDLPROC );
@@ -116,6 +120,11 @@ int main(int argc, char* argv[])
 	sigaction(SIGHUP, &ign, 0);
 	ign.sa_handler = SIG_IGN;
 	sigaction(SIGPIPE, &ign, 0);
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
 #endif
 
     ddlprocessor::DDLProcessor ddlprocessor(1, 20);

dmlproc/CMakeLists.txt

@@ -8,7 +8,8 @@ set(DMLProc_SRCS
     dmlproc.cpp
     dmlprocessor.cpp
     dmlresultbuffer.cpp
-    batchinsertprocessor.cpp)
+    batchinsertprocessor.cpp
+    ../utils/common/crashtrace.cpp)
 
 add_executable(DMLProc ${DMLProc_SRCS})
 

dmlproc/dmlproc.cpp

@@ -82,6 +82,8 @@ using namespace joblist;
 
 #include "utils_utf8.h"
 
+#include "crashtrace.h"
+
 namespace fs = boost::filesystem;
 
 namespace
@@ -473,6 +475,9 @@ int main(int argc, char* argv[])
 	//BUG 5362
 	systemLang = funcexp::utf8::idb_setlocale();
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("DMLProc");
+
     Config* cf = Config::makeConfig();
 
     setupCwd();
@@ -578,6 +583,12 @@ int main(int argc, char* argv[])
 	sigaction(SIGHUP, &ign, 0);
 	ign.sa_handler = SIG_IGN;
 	sigaction(SIGPIPE, &ign, 0);
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
 #endif
 
     dmlserver.start();

exemgr/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp)
+set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp)
 
 add_executable(ExeMgr ${ExeMgr_SRCS})
 

exemgr/main.cpp

@@ -97,6 +97,8 @@ using namespace querytele;
 #include "utils_utf8.h"
 #include "boost/filesystem.hpp"
 
+#include "crashtrace.h"
+
 namespace {
 
 //If any flags other than the table mode flags are set, produce output to screeen
@@ -1165,10 +1167,12 @@ public:
     }
 };
 
+#ifdef _MSC_VER
 void exit_(int)
 {
     exit(0);
 }
+#endif
 
 void added_a_pm(int)
 {
@@ -1213,7 +1217,6 @@ void printTotalUmMemory(int sig)
 void setupSignalHandlers()
 {
 #ifdef _MSC_VER
-    signal(SIGSEGV, exit_);
     signal(SIGINT, exit_);
     signal(SIGTERM, exit_);
 #else
@@ -1229,6 +1232,12 @@ void setupSignalHandlers()
     sigaction(SIGHUP, &ign, 0);
     ign.sa_handler = printTotalUmMemory;
     sigaction(SIGUSR1, &ign, 0);
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
 #endif
 }
 
@@ -1302,6 +1311,9 @@ int main(int argc, char* argv[])
     string systemLang = "C";
 	systemLang = funcexp::utf8::idb_setlocale();
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("ExeMgr");
+
 	gDebug = 0;
 	bool eFlg = false;
 	int c;

oamapps/serverMonitor/CMakeLists.txt

@@ -13,7 +13,8 @@ set(ServerMonitor_SRCS
     procmonMonitor.cpp
     msgProcessor.cpp
     dbhealthMonitor.cpp
-    UMAutoSync.cpp)
+    UMAutoSync.cpp
+    ../../utils/common/crashtrace.cpp)
 
 add_executable(ServerMonitor ${ServerMonitor_SRCS})
 

oamapps/serverMonitor/main.cpp

@@ -18,6 +18,8 @@
 #include "IDBPolicy.h"
 #include "serverMonitor.h"
 
+#include "crashtrace.h"
+
 using namespace std;
 using namespace servermonitor;
 using namespace oam;
@@ -38,6 +40,14 @@ int main (int argc, char** argv)
 	ServerMonitor serverMonitor;
 	Oam oam;
 
+    struct sigaction ign;
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	//Launch Memory Monitor Thread and check if swap is in critical condition
 	pthread_t memoryMonitorThread;
 	pthread_create (&memoryMonitorThread, NULL, (void*(*)(void*)) &memoryMonitor, NULL);

primitives/primproc/CMakeLists.txt

@@ -18,7 +18,8 @@ set(PrimProc_SRCS
     primitiveserver.cpp
     pseudocc.cpp
     rtscommand.cpp
-    umsocketselector.cpp)
+    umsocketselector.cpp
+    ../../utils/common/crashtrace.cpp)
 
 #PrimProc_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS)
 

primitives/primproc/primproc.cpp

@@ -71,6 +71,8 @@ using namespace idbdatafile;
 
 #include "cgroupconfigurator.h"
 
+#include "crashtrace.h"
+
 namespace primitiveprocessor
 {
 
@@ -126,6 +128,12 @@ void setupSignalHandlers()
 	ign.sa_handler = SIG_IGN;
 	sigaction(SIGUSR2, &ign, 0);
 
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	sigset_t sigset;
 	sigemptyset(&sigset);
 	sigaddset(&sigset, SIGPIPE);
@@ -288,6 +296,9 @@ int main(int argc, char* argv[])
 			systemLang.find("UTF") != string::npos )
 		utf8 = true;
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("PrimProc");
+
 	Config* cf = Config::makeConfig();
 
 	setupSignalHandlers();

procmgr/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(ProcMgr_SRCS main.cpp processmanager.cpp)
+set(ProcMgr_SRCS main.cpp processmanager.cpp ../utils/common/crashtrace.cpp)
 
 add_executable(ProcMgr ${ProcMgr_SRCS})
 

procmgr/main.cpp

@@ -32,6 +32,8 @@
 
 #include "utils_utf8.h"
 
+#include "crashtrace.h"
+
 using namespace std;
 using namespace logging;
 using namespace messageqcpp;
@@ -100,6 +102,16 @@ int main(int argc, char **argv)
 
     	setlocale(LC_ALL, systemLang.c_str());
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("ProcMgr");
+
+    struct sigaction ign;
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	Oam oam;
 
 	//check if root-user

procmon/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(ProcMon_SRCS main.cpp processmonitor.cpp)
+set(ProcMon_SRCS main.cpp processmonitor.cpp ../utils/common/crashtrace.cpp)
 
 add_executable(ProcMon ${ProcMon_SRCS})
 

procmon/main.cpp

@@ -26,6 +26,8 @@ namespace bi=boost::interprocess;
 
 #include "IDBPolicy.h"
 
+#include "crashtrace.h"
+
 using namespace std;
 using namespace messageqcpp;
 using namespace processmonitor;
@@ -75,6 +77,14 @@ int main(int argc, char **argv)
     setuid(0); // set effective ID to root; ignore return status
 #endif
 
+    struct sigaction ign;
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	if (argc > 1 && string(argv[1]) == "--daemon")
 	{
 		if (fork() != 0) return 0;

utils/common/crashtrace.cpp

@@ -0,0 +1,51 @@
+/* Copyright (C) 2018 MariaDB Corporaton
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+#include <execinfo.h>
+#include <errno.h>
+#include <cstdio>
+#include <ctime>
+#include <cstring>
+#include <signal.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+void fatalHandler(int sig)
+{
+    char filename[128];
+    void* addrs[128];
+    snprintf(filename, 128, "/var/log/mariadb/columnstore/trace/%s.%d.log", program_invocation_short_name, getpid());
+    FILE* logfile = fopen(filename, "w");
+    char s[30];
+    struct tm tim;
+    time_t now;
+    now = time(NULL);
+    tim = *(localtime(&now));
+    strftime(s,30,"%F %T",&tim);
+    fprintf(logfile, "Date/time: %s\n", s);
+    fprintf(logfile, "Signal: %d\n\n", sig);
+    fflush(logfile);
+    int fd = fileno(logfile);
+    int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0]));
+    backtrace_symbols_fd(addrs, count, fd);
+    fclose(logfile);
+    struct sigaction sigact;
+    memset(&sigact, 0, sizeof(sigact));
+    sigact.sa_handler = SIG_DFL;
+    sigaction(sig, &sigact, NULL);
+    raise(sig);
+}

utils/common/crashtrace.h

@@ -0,0 +1,18 @@
+/* Copyright (C) 2018 MariaDB Corporaton
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+void fatalHandler(int sig);

versioning/BRM/CMakeLists.txt

@@ -41,7 +41,7 @@ install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs)
 
 ########### next target ###############
 
-set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp)
+set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp ../../utils/common/crashtrace.cpp)
 
 add_executable(controllernode ${controllernode_SRCS})
 
@@ -52,7 +52,7 @@ install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
 
 ########### next target ###############
 
-set(workernode_SRCS slavenode.cpp)
+set(workernode_SRCS slavenode.cpp ../../utils/common/crashtrace.cpp)
 
 add_executable(workernode ${workernode_SRCS})
 

versioning/BRM/masternode.cpp

@@ -35,6 +35,8 @@
 #include "brmtypes.h"
 #include "utils_utf8.h"
 
+#include "crashtrace.h"
+
 #define MAX_RETRIES 10
 
 BRM::MasterDBRMNode *m;
@@ -128,6 +130,13 @@ int main(int argc, char **argv)
 	signal(SIGUSR1, restart);
 	signal(SIGPIPE, SIG_IGN);
 #endif
+    struct sigaction ign;
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
 
 	idbdatafile::IDBPolicy::configIDBPolicy();
 

versioning/BRM/slavenode.cpp

@@ -37,6 +37,8 @@
 #include "utils_utf8.h"
 #include "IDBPolicy.h"
 
+#include "crashtrace.h"
+
 using namespace BRM;
 using namespace std;
 
@@ -117,6 +119,13 @@ int main(int argc, char **argv)
 	signal(SIGPIPE, SIG_IGN);
 #endif
 
+    struct sigaction ign;
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	if (!(argc >= 3 && (arg = argv[2]) == "fg"))
 		err = fork();
 

writeengine/server/CMakeLists.txt

@@ -14,7 +14,8 @@ set(WriteEngineServer_SRCS
     we_dmlcommandproc.cpp
     we_cleartablelockcmd.cpp
     we_cpifeederthread.cpp
-    we_getfilesizes.cpp)
+    we_getfilesizes.cpp
+    ../../utils/common/crashtrace.cpp)
 
 add_executable(WriteEngineServer ${WriteEngineServer_SRCS})
 

writeengine/server/we_server.cpp

@@ -51,6 +51,8 @@ using namespace oam;
 #include "utils_utf8.h"
 #include "dbrm.h"
 
+#include "crashtrace.h"
+
 namespace
 {
 	void added_a_pm(int)
@@ -96,6 +98,9 @@ int main(int argc, char** argv)
     string systemLang = "C";
 	systemLang = funcexp::utf8::idb_setlocale();
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("WriteEngineServ");
+
     printf ("Locale is : %s\n", systemLang.c_str() );
 
 	//set BUSY_INIT state
@@ -119,6 +124,12 @@ int main(int argc, char** argv)
 	sigaction(SIGHUP, &sa, 0);
 	sa.sa_handler = SIG_IGN;
 	sigaction(SIGPIPE, &sa, 0);
+
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &sa, 0);
+    sigaction(SIGABRT, &sa, 0);
+    sigaction(SIGFPE, &sa, 0);
 #endif
 
 	// Init WriteEngine Wrapper (including Config Columnstore.xml cache)