Merge branch 'develop-1.0' into 1.1-merge-up-20180118

CMakeLists.txt

@@ -76,7 +76,8 @@ ENDIF("${isSystemDir}" STREQUAL "-1")
 
 INCLUDE (configureEngine.cmake)
 
-CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h)
+CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum @ONLY IMMEDIATE)
+INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL_ENGINE} COMPONENT platform)
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h)
 
 INCLUDE(bison.cmake)

README

@@ -9,5 +9,4 @@ series are included in this release.
 
 Additional features will be pushed in future releases. 
 A few things to notice:
-- Do not use Beta releases on production systems.
 - The building of the ColumnStore engine needs a special build environment. We're working on making it available for everyone to build.

dbcon/mysql/is_columnstore_files.cpp

@@ -124,6 +124,17 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
                 iter++;
                 continue;
             }
+
+            try
+            {
+                oam_instance.getDbrootPmConfig(iter->dbRoot, pmId);
+            }
+            catch (std::runtime_error)
+            {
+                // MCOL-1116: If we are here a DBRoot is offline/missing
+                iter++;
+                continue;
+            }
             table->field[0]->store(oid);
             table->field[1]->store(iter->segmentNum);
             table->field[2]->store(iter->partitionNum);
@@ -134,7 +145,7 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
             std::string DbRootPath = config->getConfig("SystemConfig", DbRootName.str());
             fileSize = compressedFileSize = 0;
             snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", DbRootPath.c_str(), oidDirName);
-            oam_instance.getDbrootPmConfig(iter->dbRoot, pmId);
+
             std::ostringstream oss;
             oss << "pm" << pmId << "_WriteEngineServer";
             std::string client = oss.str();

dbcon/mysql/mysql-Columnstore

@@ -61,7 +61,7 @@ datadir=$basedir/db
 # Value here is overriden by value in my.cnf. 
 # 0 means don't wait at all
 # Negative numbers mean to wait indefinitely
-service_startup_timeout=60
+service_startup_timeout=90
 
 # Lock directory for RedHat / SuSE.
 lockdir='/var/lock/subsys'

ddlproc/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp)
+set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp)
 
 add_executable(DDLProc ${DDLProc_SRCS})
 

ddlproc/ddlproc.cpp

@@ -61,6 +61,7 @@ using namespace execplan;
 #include "IDBPolicy.h"
 #include "utils_utf8.h"
 
+#include "crashtrace.h"
 
 namespace fs = boost::filesystem;
 
@@ -97,6 +98,9 @@ int main(int argc, char* argv[])
 	string systemLang = "C";
 	systemLang = funcexp::utf8::idb_setlocale();
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("DDLProc");
+
     setupCwd();
 
     WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_DDLPROC );
@@ -116,6 +120,11 @@ int main(int argc, char* argv[])
 	sigaction(SIGHUP, &ign, 0);
 	ign.sa_handler = SIG_IGN;
 	sigaction(SIGPIPE, &ign, 0);
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
 #endif
 
     ddlprocessor::DDLProcessor ddlprocessor(1, 20);

dmlproc/CMakeLists.txt

@@ -8,7 +8,8 @@ set(DMLProc_SRCS
     dmlproc.cpp
     dmlprocessor.cpp
     dmlresultbuffer.cpp
-    batchinsertprocessor.cpp)
+    batchinsertprocessor.cpp
+    ../utils/common/crashtrace.cpp)
 
 add_executable(DMLProc ${DMLProc_SRCS})
 

dmlproc/dmlproc.cpp

@@ -82,6 +82,8 @@ using namespace joblist;
 
 #include "utils_utf8.h"
 
+#include "crashtrace.h"
+
 namespace fs = boost::filesystem;
 
 namespace
@@ -473,6 +475,9 @@ int main(int argc, char* argv[])
 	//BUG 5362
 	systemLang = funcexp::utf8::idb_setlocale();
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("DMLProc");
+
     Config* cf = Config::makeConfig();
 
     setupCwd();
@@ -592,6 +597,12 @@ int main(int argc, char* argv[])
 	sigaction(SIGHUP, &ign, 0);
 	ign.sa_handler = SIG_IGN;
 	sigaction(SIGPIPE, &ign, 0);
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
 #endif
 
     dmlserver.start();

exemgr/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp)
+set(ExeMgr_SRCS main.cpp activestatementcounter.cpp femsghandler.cpp ../utils/common/crashtrace.cpp)
 
 add_executable(ExeMgr ${ExeMgr_SRCS})
 

exemgr/main.cpp

@@ -98,6 +98,7 @@ using namespace querytele;
 #include "boost/filesystem.hpp"
 
 #include "threadpool.h"
+#include "crashtrace.h"
 
 namespace {
 

oam/install_scripts/binary_installer.sh

@@ -215,7 +215,7 @@ send_user "\n"
 send_user "Start ColumnStore service                       "
 send_user " \n"
 send "ssh -v $USERNAME@$SERVER '$INSTALLDIR/bin/columnstore restart'\n"
-set timeout 60
+set timeout 120
 # check return
 expect {
 	"word: " { send "$PASSWORD\n"

oamapps/serverMonitor/CMakeLists.txt

@@ -13,7 +13,8 @@ set(ServerMonitor_SRCS
     procmonMonitor.cpp
     msgProcessor.cpp
     dbhealthMonitor.cpp
-    UMAutoSync.cpp)
+    UMAutoSync.cpp
+    ../../utils/common/crashtrace.cpp)
 
 add_executable(ServerMonitor ${ServerMonitor_SRCS})
 

oamapps/serverMonitor/main.cpp

@@ -18,6 +18,8 @@
 #include "IDBPolicy.h"
 #include "serverMonitor.h"
 
+#include "crashtrace.h"
+
 using namespace std;
 using namespace servermonitor;
 using namespace oam;
@@ -38,6 +40,14 @@ int main (int argc, char** argv)
 	ServerMonitor serverMonitor;
 	Oam oam;
 
+    struct sigaction ign;
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	//Launch Memory Monitor Thread and check if swap is in critical condition
 	pthread_t memoryMonitorThread;
 	pthread_create (&memoryMonitorThread, NULL, (void*(*)(void*)) &memoryMonitor, NULL);

primitives/primproc/CMakeLists.txt

@@ -18,7 +18,8 @@ set(PrimProc_SRCS
     primitiveserver.cpp
     pseudocc.cpp
     rtscommand.cpp
-    umsocketselector.cpp)
+    umsocketselector.cpp
+    ../../utils/common/crashtrace.cpp)
 
 #PrimProc_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS)
 

primitives/primproc/primproc.cpp

@@ -71,6 +71,8 @@ using namespace idbdatafile;
 
 #include "cgroupconfigurator.h"
 
+#include "crashtrace.h"
+
 namespace primitiveprocessor
 {
 
@@ -126,6 +128,12 @@ void setupSignalHandlers()
 	ign.sa_handler = SIG_IGN;
 	sigaction(SIGUSR2, &ign, 0);
 
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	sigset_t sigset;
 	sigemptyset(&sigset);
 	sigaddset(&sigset, SIGPIPE);
@@ -288,6 +296,9 @@ int main(int argc, char* argv[])
 			systemLang.find("UTF") != string::npos )
 		utf8 = true;
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("PrimProc");
+
 	Config* cf = Config::makeConfig();
 
 	setupSignalHandlers();

procmgr/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(ProcMgr_SRCS main.cpp processmanager.cpp)
+set(ProcMgr_SRCS main.cpp processmanager.cpp ../utils/common/crashtrace.cpp)
 
 add_executable(ProcMgr ${ProcMgr_SRCS})
 

procmgr/main.cpp

@@ -33,6 +33,8 @@
 
 #include "utils_utf8.h"
 
+#include "crashtrace.h"
+
 using namespace std;
 using namespace logging;
 using namespace messageqcpp;
@@ -99,6 +101,16 @@ int main(int argc, char **argv)
 
     	setlocale(LC_ALL, systemLang.c_str());
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("ProcMgr");
+
+    struct sigaction ign;
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	Oam oam;
 
 	//check if root-user

procmon/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(ProcMon_SRCS main.cpp processmonitor.cpp)
+set(ProcMon_SRCS main.cpp processmonitor.cpp ../utils/common/crashtrace.cpp)
 
 add_executable(ProcMon ${ProcMon_SRCS})
 

procmon/main.cpp

@@ -26,6 +26,8 @@ namespace bi=boost::interprocess;
 
 #include "IDBPolicy.h"
 
+#include "crashtrace.h"
+
 using namespace std;
 using namespace messageqcpp;
 using namespace processmonitor;
@@ -55,10 +57,12 @@ void updateShareMemory(processStatusList* aPtr);
 bool runStandby = false;
 bool processInitComplete = false;
 bool rootUser = true;
+bool mainResumeFlag;
 string USER = "root";
 string PMwithUM = "n";
 bool startProcMon = false;
 
+
 //extern std::string gOAMParentModuleName;
 extern bool gOAMParentModuleFlag;
 
@@ -76,6 +80,14 @@ int main(int argc, char **argv)
     setuid(0); // set effective ID to root; ignore return status
 #endif
 
+    struct sigaction ign;
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	if (argc > 1 && string(argv[1]) == "--daemon")
 	{
 		if (fork() != 0) return 0;
@@ -499,13 +511,24 @@ int main(int argc, char **argv)
 			unlink ("/var/log/mariadb/columnstore/activeAlarms");
 		}
 
+     	//Clear mainResumeFlag
+     	     
+    	mainResumeFlag = false;
+     
 		//launch Status table control thread on 'pm' modules
 		pthread_t statusThread;
 		int ret = pthread_create (&statusThread, NULL, (void*(*)(void*)) &statusControlThread, NULL);
 		if ( ret != 0 )
 			log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR);
 
-		sleep(6);	// give the Status thread time to fully initialize
+		//wait for flag to be set
+
+		while(!mainResumeFlag)
+		{
+			log.writeLog(__LINE__, "WATING FOR mainResumeFlag to be set", LOG_TYPE_DEBUG);
+
+			sleep(1);
+		}
 	}
 
 	SystemStatus systemstatus;
@@ -786,6 +809,8 @@ int main(int argc, char **argv)
 		}
 	}
 	
+	log.writeLog(__LINE__, "SYSTEM STATUS = " + oam.itoa(systemstatus.SystemOpState), LOG_TYPE_DEBUG);
+
 	if ( systemstatus.SystemOpState != MAN_OFFLINE && !DISABLED) {
 
 		// Loop through the process list to check the process current state
@@ -2099,6 +2124,10 @@ static void statusControlThread()
 		log.writeLog(__LINE__, "Dbroot Status shared Memory allociated and Initialized", LOG_TYPE_DEBUG);
 	}
 
+	//Set mainResumeFlag, to start up main thread
+
+	mainResumeFlag = true;
+
 	string portName = "ProcStatusControl";
 	if (runStandby) {
 		portName = "ProcStatusControlStandby";

utils/common/crashtrace.cpp

@@ -0,0 +1,51 @@
+/* Copyright (C) 2018 MariaDB Corporaton
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+#include <execinfo.h>
+#include <errno.h>
+#include <cstdio>
+#include <ctime>
+#include <cstring>
+#include <signal.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+void fatalHandler(int sig)
+{
+    char filename[128];
+    void* addrs[128];
+    snprintf(filename, 128, "/var/log/mariadb/columnstore/trace/%s.%d.log", program_invocation_short_name, getpid());
+    FILE* logfile = fopen(filename, "w");
+    char s[30];
+    struct tm tim;
+    time_t now;
+    now = time(NULL);
+    tim = *(localtime(&now));
+    strftime(s,30,"%F %T",&tim);
+    fprintf(logfile, "Date/time: %s\n", s);
+    fprintf(logfile, "Signal: %d\n\n", sig);
+    fflush(logfile);
+    int fd = fileno(logfile);
+    int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0]));
+    backtrace_symbols_fd(addrs, count, fd);
+    fclose(logfile);
+    struct sigaction sigact;
+    memset(&sigact, 0, sizeof(sigact));
+    sigact.sa_handler = SIG_DFL;
+    sigaction(sig, &sigact, NULL);
+    raise(sig);
+}

utils/common/crashtrace.h

@@ -0,0 +1,18 @@
+/* Copyright (C) 2018 MariaDB Corporaton
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+void fatalHandler(int sig);

versioning/BRM/CMakeLists.txt

@@ -39,7 +39,7 @@ install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs)
 
 ########### next target ###############
 
-set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp)
+set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp ../../utils/common/crashtrace.cpp)
 
 add_executable(controllernode ${controllernode_SRCS})
 
@@ -50,7 +50,7 @@ install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
 
 ########### next target ###############
 
-set(workernode_SRCS slavenode.cpp)
+set(workernode_SRCS slavenode.cpp ../../utils/common/crashtrace.cpp)
 
 add_executable(workernode ${workernode_SRCS})
 

versioning/BRM/masternode.cpp

@@ -35,6 +35,8 @@
 #include "brmtypes.h"
 #include "utils_utf8.h"
 
+#include "crashtrace.h"
+
 #define MAX_RETRIES 10
 
 BRM::MasterDBRMNode *m;
@@ -128,6 +130,13 @@ int main(int argc, char **argv)
 	signal(SIGUSR1, restart);
 	signal(SIGPIPE, SIG_IGN);
 #endif
+    struct sigaction ign;
+
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
 
 	idbdatafile::IDBPolicy::configIDBPolicy();
 

versioning/BRM/slavenode.cpp

@@ -37,6 +37,8 @@
 #include "utils_utf8.h"
 #include "IDBPolicy.h"
 
+#include "crashtrace.h"
+
 using namespace BRM;
 using namespace std;
 
@@ -117,6 +119,13 @@ int main(int argc, char **argv)
 	signal(SIGPIPE, SIG_IGN);
 #endif
 
+    struct sigaction ign;
+    memset(&ign, 0, sizeof(ign));
+    ign.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &ign, 0);
+    sigaction(SIGABRT, &ign, 0);
+    sigaction(SIGFPE, &ign, 0);
+
 	if (!(argc >= 3 && (arg = argv[2]) == "fg"))
 		err = fork();
 

writeengine/server/CMakeLists.txt

@@ -14,7 +14,8 @@ set(WriteEngineServer_SRCS
     we_dmlcommandproc.cpp
     we_cleartablelockcmd.cpp
     we_cpifeederthread.cpp
-    we_getfilesizes.cpp)
+    we_getfilesizes.cpp
+    ../../utils/common/crashtrace.cpp)
 
 add_executable(WriteEngineServer ${WriteEngineServer_SRCS})
 

writeengine/server/we_server.cpp

@@ -51,6 +51,8 @@ using namespace oam;
 #include "utils_utf8.h"
 #include "dbrm.h"
 
+#include "crashtrace.h"
+
 namespace
 {
 	void added_a_pm(int)
@@ -96,6 +98,9 @@ int main(int argc, char** argv)
     string systemLang = "C";
 	systemLang = funcexp::utf8::idb_setlocale();
 
+    // This is unset due to the way we start it
+    program_invocation_short_name = const_cast<char*>("WriteEngineServ");
+
     printf ("Locale is : %s\n", systemLang.c_str() );
 
 	//set BUSY_INIT state
@@ -119,6 +124,12 @@ int main(int argc, char** argv)
 	sigaction(SIGHUP, &sa, 0);
 	sa.sa_handler = SIG_IGN;
 	sigaction(SIGPIPE, &sa, 0);
+
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_handler = fatalHandler;
+    sigaction(SIGSEGV, &sa, 0);
+    sigaction(SIGABRT, &sa, 0);
+    sigaction(SIGFPE, &sa, 0);
 #endif
 
 	// Init WriteEngine Wrapper (including Config Columnstore.xml cache)