diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ff14cad0..44aed9bc8 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,9 @@ ENDIF("${isSystemDir}" STREQUAL "-1") INCLUDE (configureEngine.cmake) +# releasenum is used by external scripts for various tasks. Leave it alone. +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL_ENGINE} COMPONENT platform) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) diff --git a/build/postInstall_libs.sh b/build/postInstall_libs.sh index b4e8c04e4..6331d51f3 100644 --- a/build/postInstall_libs.sh +++ b/build/postInstall_libs.sh @@ -7,5 +7,4 @@ fi prefix=/usr/local -echo "MariaDB ColumnStore RPM install completed" diff --git a/build/postInstall_platform.sh b/build/postInstall_platform.sh index 78e7b0288..7f75fff0e 100644 --- a/build/postInstall_platform.sh +++ b/build/postInstall_platform.sh @@ -9,4 +9,3 @@ prefix=/usr/local test -x /usr/local/mariadb/columnstore/bin/post-install && /usr/local/mariadb/columnstore/bin/post-install --prefix=$prefix --rpmmode=$rpmmode -echo "MariaDB ColumnStore RPM install completed" diff --git a/build/postInstall_storage_engine.sh b/build/postInstall_storage_engine.sh index 3c346b537..3e88e4922 100644 --- a/build/postInstall_storage_engine.sh +++ b/build/postInstall_storage_engine.sh @@ -7,6 +7,5 @@ fi prefix=/usr/local -echo "MariaDB ColumnStore RPM install completed" diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 5c340adfc..be5df3462 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -87,7 +87,7 @@ IF (EXISTS "/etc/SuSE-release") set(SUSE_VERSION_NUMBER "${CMAKE_MATCH_1}") ENDIF () if (${REDHAT_VERSION_NUMBER} EQUAL 6) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "mariadb-columnstore-libs" "snappy") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "mariadb-columnstore-libs" "mariadb-columnstore-shared" "snappy") # Disable auto require as this will also try to pull Boost via RPM SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") elseif (${SUSE_VERSION_NUMBER} EQUAL 12) @@ -188,6 +188,7 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/etc/MessageFile.txt" "/usr/local/mariadb/columnstore/etc/ErrorMessage.txt" "/usr/local/mariadb/columnstore/local/module" +"/usr/local/mariadb/columnstore/releasenum" "/usr/local/mariadb/columnstore/bin/rollback" "/usr/local/mariadb/columnstore/bin/editem" "/usr/local/mariadb/columnstore/bin/getConfig" diff --git a/dbcon/joblist/joblist.cpp b/dbcon/joblist/joblist.cpp index fcb5c4faf..023b21bb6 100644 --- a/dbcon/joblist/joblist.cpp +++ b/dbcon/joblist/joblist.cpp @@ -1155,6 +1155,12 @@ void JobList::abort() for (i = 0; i < fProject.size(); i++) fProject[i]->abort(); + + for (i = 0; i < fQuery.size(); i++) + fQuery[i]->join(); + + for (i = 0; i < fProject.size(); i++) + fProject[i]->join(); } } diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index 394a78395..a5e0759fa 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -415,6 +415,7 @@ void SubQueryTransformer::updateCorrelateInfo() sc->schemaName(""); sc->tableName(fVtable.name()); sc->tableAlias(fVtable.alias()); + sc->viewName(fVtable.view()); sc->oid(fVtable.columnOid(k->second)); sc->columnName(fVtable.columns()[k->second]->columnName()); const CalpontSystemCatalog::ColType& ct = fVtable.columnType(k->second); diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index cddedf543..a77cbcdd1 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5063,22 +5063,6 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) if (more) { - // Trying out a ramp-up strategy for starting the - // first phase threads to cut overhead on big systems - // processing small result - // sets. On every non-zero read from the input FIFO, - // and if there is more data to read, the - // first thread will start another thread until the - // maximum number is reached. -#if 0 - if (threadID == 0 && fFirstPhaseThreadCount < fNumOfThreads && - dlIn->more(fInputIter)) - { - fFirstPhaseRunners.push_back(jobstepThreadPool.invoke(ThreadedAggregator(this, fFirstPhaseThreadCount))); - fFirstPhaseThreadCount++; - } - -#endif fRowGroupIns[threadID].setData(&rgData); fMemUsage[threadID] += fRowGroupIns[threadID].getSizeWithStrings(); @@ -5364,29 +5348,17 @@ uint64_t TupleAggregateStep::doThreadedAggregate(ByteStream& bs, RowGroupDL* dlp { initializeMultiThread(); -// This block of code starts all threads at the start - fFirstPhaseThreadCount = fNumOfThreads; - fFirstPhaseRunners.clear(); - fFirstPhaseRunners.reserve(fNumOfThreads); // to prevent a resize during use + vector runners; // thread pool handles + runners.reserve(fNumOfThreads); // to prevent a resize during use + // Start the aggregator threads for (i = 0; i < fNumOfThreads; i++) { - fFirstPhaseRunners.push_back(jobstepThreadPool.invoke(ThreadedAggregator(this, i))); + runners.push_back(jobstepThreadPool.invoke(ThreadedAggregator(this, i))); } -#if 0 -// This block of code starts one thread, relies on doThreadedAggregation() -// For reasons unknown, this doesn't work right with threadpool -// to start more as needed - fFirstPhaseRunners.clear(); - fFirstPhaseRunners.reserve(fNumOfThreads); // to prevent a resize during use - fFirstPhaseThreadCount = 1; - fFirstPhaseRunners.push_back(jobstepThreadPool.invoke(ThreadedAggregator(this, 0))); -#endif - - // Now wait for that thread plus all the threads it may have spawned - jobstepThreadPool.join(fFirstPhaseRunners); - fFirstPhaseRunners.clear(); + // Now wait for all those threads + jobstepThreadPool.join(runners); } if (dynamic_cast(fAggregator.get()) && fAggregator->aggMapKeyLength() > 0) diff --git a/dbcon/joblist/tupleaggregatestep.h b/dbcon/joblist/tupleaggregatestep.h index f6429d01a..13a12a8c8 100644 --- a/dbcon/joblist/tupleaggregatestep.h +++ b/dbcon/joblist/tupleaggregatestep.h @@ -199,8 +199,6 @@ private: bool fIsMultiThread; int fInputIter; // iterator boost::scoped_array fMemUsage; - std::vector fFirstPhaseRunners; // thread pool handles - uint32_t fFirstPhaseThreadCount; boost::shared_ptr fSessionMemLimit; }; diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql index 691ea754d..80b0f12b1 100644 --- a/dbcon/mysql/columnstore_info.sql +++ b/dbcon/mysql/columnstore_info.sql @@ -28,8 +28,8 @@ DROP PROCEDURE IF EXISTS `total_usage` // CREATE PROCEDURE total_usage () BEGIN SELECT - (SELECT format_filesize(sum(data_size)) TOTAL_DATA_SIZE FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS) TOTAL_DATA_SIZE, - (SELECT format_filesize(sum(file_size)) TOTAL_DISK_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_FILES) TOTAL_DISK_USAGE; + (SELECT columnstore_info.format_filesize(sum(data_size)) TOTAL_DATA_SIZE FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS) TOTAL_DATA_SIZE, + (SELECT columnstore_info.format_filesize(sum(file_size)) TOTAL_DISK_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_FILES) TOTAL_DISK_USAGE; END // DROP PROCEDURE IF EXISTS `table_usage` // @@ -52,21 +52,21 @@ CREATE PROCEDURE table_usage (IN t_schema char(64), IN t_name char(64)) CREATE TABLE columnstore_info.columnstore_files engine=myisam as (select * from information_schema.columnstore_files); ALTER TABLE columnstore_info.columnstore_files ADD INDEX `object_id` (`object_id`); IF t_name IS NOT NULL THEN -SELECT TABLE_SCHEMA, TABLE_NAME, format_filesize(data) as DATA_DISK_USAGE, format_filesize(dict) as DICT_DISK_USAGE, format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics where table_name = t_name and (table_schema = t_schema or t_schema IS NULL) group by table_schema, table_name ) q; ELSEIF t_schema IS NOT NULL THEN -SELECT TABLE_SCHEMA, TABLE_NAME, format_filesize(data) as DATA_DISK_USAGE, format_filesize(dict) as DICT_DISK_USAGE, format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics where table_schema = t_schema group by table_schema, table_name ) q; ELSE -SELECT TABLE_SCHEMA, TABLE_NAME, format_filesize(data) as DATA_DISK_USAGE, format_filesize(dict) as DICT_DISK_USAGE, format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index e960864f2..01a42ac5c 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -1870,7 +1870,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Use of the MySQL auto_increment syntax is not supported in Columnstore. If you wish to create an auto increment column in Columnstore, please consult the Columnstore SQL Syntax Guide for the correct usage."); } // MCOL-867. MariaDB RENAME TABLE statement supports WAIT|NOWAIT options since 10.3.0 but Columnstore isn't yet. - else if(ddlStatement.find("WAIT") != string::npos || ddlStatement.find("NOWAIT") != string::npos) + else if (ddlStatement.find("WAIT") != string::npos || ddlStatement.find("NOWAIT") != string::npos) { thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "WAIT and NOWAIT options are not supported in Columnstore. Please consult the Columnstore SQL Syntax Guide for the correct usage."); } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 652314097..8078d452e 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -2029,7 +2029,7 @@ extern "C" #ifdef _MSC_VER __declspec(dllexport) #endif -// Return 1 if system is read only; 0 if writeable +// Return non-zero if system is read only; 0 if writeable long long mcssystemreadonly(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) { @@ -2038,10 +2038,15 @@ extern "C" try { - if (dbrm.isReadWrite()) // Returns 0 for writable, 5 for read only + if (dbrm.getSystemSuspended()) { rtn = 1; } + + if (dbrm.isReadWrite() > 0) // Returns 0 for writable, 5 for read only + { + rtn = 2; + } } catch (...) { @@ -2067,47 +2072,6 @@ extern "C" { } -#ifdef _MSC_VER - __declspec(dllexport) -#endif -// Return 1 if system is read only; 0 if writeable - long long mcswritessuspended(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, char* error) - { - long long rtn = 0; - DBRM dbrm(true); - - try - { - if (dbrm.getSystemSuspended()) - { - rtn = 1; - } - } - catch (...) - { - *error = 1; - rtn = 1; - } - - return rtn; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - my_bool mcswritessuspended_init(UDF_INIT* initid, UDF_ARGS* args, char* message) - { - return 0; - } - -#ifdef _MSC_VER - __declspec(dllexport) -#endif - void mcswritessuspended_deinit(UDF_INIT* initid) - { - } - #define MAXSTRINGLENGTH 50 const char* PmSmallSideMaxMemory = "pmmaxmemorysmallside"; diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 17c6e1817..e8eb5b2b0 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -84,7 +84,6 @@ CREATE FUNCTION idbpartition RETURNS STRING soname 'libcalmysql.so'; CREATE FUNCTION idblocalpm RETURNS INTEGER soname 'libcalmysql.so'; CREATE FUNCTION mcssystemready RETURNS INTEGER soname 'libcalmysql.so'; CREATE FUNCTION mcssystemreadonly RETURNS INTEGER soname 'libcalmysql.so'; -CREATE FUNCTION mcswritessuspended RETURNS INTEGER soname 'libcalmysql.so'; CREATE DATABASE IF NOT EXISTS infinidb_vtable; CREATE DATABASE IF NOT EXISTS infinidb_querystats; diff --git a/exemgr/main.cpp b/exemgr/main.cpp index a4947ba41..ca0573788 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1242,10 +1242,12 @@ public: } }; +#ifdef _MSC_VER void exit_(int) { exit(0); } +#endif void added_a_pm(int) { @@ -1293,7 +1295,6 @@ void printTotalUmMemory(int sig) void setupSignalHandlers() { #ifdef _MSC_VER - signal(SIGSEGV, exit_); signal(SIGINT, exit_); signal(SIGTERM, exit_); #else @@ -1309,6 +1310,11 @@ void setupSignalHandlers() sigaction(SIGHUP, &ign, 0); ign.sa_handler = printTotalUmMemory; sigaction(SIGUSR1, &ign, 0); + memset(&ign, 0, sizeof(ign)); + ign.sa_handler = fatalHandler; + sigaction(SIGSEGV, &ign, 0); + sigaction(SIGABRT, &ign, 0); + sigaction(SIGFPE, &ign, 0); #endif } @@ -1389,6 +1395,9 @@ int main(int argc, char* argv[]) string systemLang = "C"; systemLang = funcexp::utf8::idb_setlocale(); + // This is unset due to the way we start it + program_invocation_short_name = const_cast("ExeMgr"); + gDebug = 0; bool eFlg = false; int c; diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index ccc5e30b2..70a0add6d 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -19,6 +19,10 @@ 0.0.0.0 8603 + + 127.0.0.1 + 8606 + 0.0.0.0 8604 diff --git a/oam/etc/Columnstore.xml.singleserver b/oam/etc/Columnstore.xml.singleserver index a316ea264..8768eb6ce 100644 --- a/oam/etc/Columnstore.xml.singleserver +++ b/oam/etc/Columnstore.xml.singleserver @@ -14,6 +14,10 @@ 127.0.0.1 8603 + + 127.0.0.1 + 8606 + 127.0.0.1 8604 diff --git a/oam/install_scripts/columnstore.conf b/oam/install_scripts/columnstore.conf index fb47efa80..e1b92ea84 100644 --- a/oam/install_scripts/columnstore.conf +++ b/oam/install_scripts/columnstore.conf @@ -1,2 +1,2 @@ /usr/local/mariadb/columnstore/lib - +/usr/local/mariadb/columnstore/mysql/lib \ No newline at end of file diff --git a/oam/install_scripts/columnstoreLogRotate b/oam/install_scripts/columnstoreLogRotate index efcf30fcf..8bb75ed6e 100644 --- a/oam/install_scripts/columnstoreLogRotate +++ b/oam/install_scripts/columnstoreLogRotate @@ -1,6 +1,6 @@ # -# MariaDb Columnstore Log Rotate file that gets installed in /etc/logrotate.d -# as part of the RPM installation +# MariaDB ColumnStore Log Rotate file that gets installed in /etc/logrotate.d +# as part of the package installation # /var/log/mariadb/columnstore/*.log { @@ -10,14 +10,7 @@ dateext copytruncate olddir /var/log/mariadb/columnstore/archive -} -/var/log/mariadb/columnstore/*.log1 { - missingok - rotate 7 - daily - nocreate - compress - olddir /var/log/mariadb/columnstore/archive + su root root } /usr/local/mariadb/columnstore/etc/Columnstore.xml { daily @@ -32,4 +25,5 @@ dateext copytruncate olddir /usr/local/mariadb/columnstore/mysql/db + su root root } diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index fc60e32e7..a295a088f 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -230,10 +230,6 @@ if [ $user = "root" ]; then fi fi -#setup MariaDB Columnstore system logging -sed -i -e s/groupname/$user/g $installdir/bin/columnstoreSyslog7 -sed -i -e s/username/$user/g $installdir/bin/columnstoreSyslog7 - if [ $user = "root" ]; then $installdir/bin/syslogSetup.sh install > /tmp/syslog_install.log 2>&1 rm -f /etc/default/columnstore @@ -255,10 +251,6 @@ else $SUDO chmod 666 /etc/fstab fi -# install Columnstore Log Rotate File -$SUDO cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1 -$SUDO chmod 644 /etc/logrotate.d/columnstore - #check if MariaDB Columnstore system logging was setup cat /tmp/syslog_install.log | grep 'No System Logging' >/dev/null 2>&1 if [ $? -eq 0 ]; then @@ -276,8 +268,11 @@ fi #log install message test -f $installdir/post/functions && . $installdir/post/functions -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" - +if [ $user = "root" ]; then + $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" +else + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" +fi #setup hadoop hadoop=`which hadoop 2>/dev/null` if [ -z "$hadoop" ]; then diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 1e481a109..1f4235a30 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -166,17 +166,21 @@ if [ ! -z "$syslog_conf" ] ; then #set the syslog for ColumnStore logging # remove older version incase it was installed by previous build $SUDO rm -rf /etc/rsyslog.d/columnstore.conf + if [ $rsyslog7 == 1 ]; then + sed -i -e s/groupname/adm/g ${columnstoreSyslogFile7} + sed -i -e s/username/syslog/g ${columnstoreSyslogFile7} + $SUDO rm -f /etc/rsyslog.d/49-columnstore.conf $SUDO cp ${columnstoreSyslogFile7} ${syslog_conf} - else + else $SUDO cp ${columnstoreSyslogFile} ${syslog_conf} fi fi - - if [ $rsyslog7 == 1 ]; then - $SUDO chown -R syslog:adm /var/log/mariadb/columnstore >/dev/null 2>&1 - fi + + # install Columnstore Log Rotate File + $SUDO cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1 + $SUDO chmod 644 /etc/logrotate.d/columnstore restartSyslog fi diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 5601c8573..b45cd7307 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -521,8 +521,7 @@ enum PROC_MGT_MSG_REQUEST ENABLEMYSQLREP, DISABLEMYSQLREP, GLUSTERASSIGN, - GLUSTERUNASSIGN, - PROCESSALARM + GLUSTERUNASSIGN }; /** @brief Process Management - Mgr to Mon request options diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index bae8110a8..6cea84596 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -28,10 +28,10 @@ #include #include -#include "messagequeue.h" #include "alarmglobal.h" #include "liboamcpp.h" #include "installdir.h" +#include "messagequeue.h" using namespace std; using namespace oam; @@ -409,7 +409,7 @@ void configAlarm (Alarm& calAlarm) /***************************************************************************************** * @brief sendAlarmReport API * -* purpose: Process Alarm Report +* purpose: Send Alarm Report * *****************************************************************************************/ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int state, @@ -484,16 +484,73 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st else processName = repProcessName; - Alarm calAlarm; + int returnStatus = API_SUCCESS; //default + ByteStream msg1; - calAlarm.setAlarmID (alarmID); - calAlarm.setComponentID (componentID); - calAlarm.setState (state); - calAlarm.setSname (ModuleName); - calAlarm.setPname (processName); - calAlarm.setPid (pid); - calAlarm.setTid (tid); + // setup message + msg1 << (ByteStream::byte) alarmID; + msg1 << (std::string) componentID; + msg1 << (ByteStream::byte) state; + msg1 << (std::string) ModuleName; + msg1 << (std::string) processName; + msg1 << (ByteStream::byte) pid; + msg1 << (ByteStream::byte) tid; + try + { + //send the msg to Process Manager + MessageQueueClient procmgr("ProcMgr_Alarm"); + procmgr.write(msg1); + + // shutdown connection + procmgr.shutdown(); + } + catch (std::runtime_error& e) + { + LoggingID lid(11); + MessageLog ml(lid); + Message msg; + Message::Args args; + args.add("sendAlarmReport error:"); + args.add(e.what()); + msg.format(args); + ml.logErrorMessage(msg); + } + catch (std::exception& e) + { + LoggingID lid(11); + MessageLog ml(lid); + Message msg; + Message::Args args; + args.add("sendAlarmReport error:"); + args.add(e.what()); + msg.format(args); + ml.logErrorMessage(msg); + } + catch (...) + { + LoggingID lid(11); + MessageLog ml(lid); + Message msg; + Message::Args args; + args.add("sendAlarmReport error:"); + args.add("general failure"); + msg.format(args); + ml.logErrorMessage(msg); + } + + return; +#endif //SKIP_ALARM +} + +/***************************************************************************************** +* @brief processAlarmReport API +* +* purpose: Process Alarm Report +* +*****************************************************************************************/ +void ALARMManager::processAlarmReport (Alarm& calAlarm) +{ // Get alarm configuration try { @@ -512,7 +569,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st } return; -#endif //SKIP_ALARM + } /***************************************************************************************** diff --git a/oamapps/alarmmanager/alarmmanager.h b/oamapps/alarmmanager/alarmmanager.h index 66bdc4cac..0ad5a4832 100644 --- a/oamapps/alarmmanager/alarmmanager.h +++ b/oamapps/alarmmanager/alarmmanager.h @@ -78,6 +78,8 @@ public: std::string repProcessName = ""); + EXPORT void processAlarmReport (Alarm& calAlarm); + /** @brief return active alarm list * * @param AlarmList the alarm map reference to store alarms diff --git a/oamapps/mcsadmin/mcsadmin.h b/oamapps/mcsadmin/mcsadmin.h index 005a4a17d..2843090b8 100644 --- a/oamapps/mcsadmin/mcsadmin.h +++ b/oamapps/mcsadmin/mcsadmin.h @@ -58,7 +58,7 @@ const int ArgNum = 10; const int DescNumMax = 10; const int cmdNum = 68; -const std::string DEFAULT_LOG_FILE = "/var/log/mariadb/columnstore/uiCommands.log"; +const std::string DEFAULT_LOG_FILE = "/var/log/mariadb/columnstore/mcsadmin.log"; std::ofstream logFile; /** diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 86a302f7d..9cc3bf5b2 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -2316,7 +2316,8 @@ int main(int argc, char* argv[]) sysConfig->setConfig(parentProcessMonitor, "IPAddr", parentOAMModuleIPAddr); sysConfig->setConfig(parentProcessMonitor, "Port", "8800"); sysConfig->setConfig("ProcMgr", "IPAddr", parentOAMModuleIPAddr); - //sysConfig->setConfig("ProcHeartbeatControl", "IPAddr", parentOAMModuleIPAddr); + sysConfig->setConfig("ProcHeartbeatControl", "IPAddr", parentOAMModuleIPAddr); + sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", parentOAMModuleIPAddr); sysConfig->setConfig("ProcStatusControl", "IPAddr", parentOAMModuleIPAddr); string parentServerMonitor = parentOAMModuleName + "_ServerMonitor"; sysConfig->setConfig(parentServerMonitor, "IPAddr", parentOAMModuleIPAddr); @@ -3227,7 +3228,7 @@ int main(int argc, char* argv[]) if ( EEPackageType == "rpm") columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.rpm"; else if ( EEPackageType == "deb") - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.deb"; + columnstorePackage = HOME + "/" + "mariadb-columnstore-*" + systemsoftware.Version + "*.deb"; else columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.bin.tar.gz"; diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 338cd9826..e24b15c86 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -78,6 +78,7 @@ extern bool startFailOver; extern bool gOAMParentModuleFlag; static void messageThread(Configuration config); +static void alarmMessageThread(Configuration config); static void sigUser1Handler(int sig); static void startMgrProcessThread(); static void hdfsActiveAlarmsPushingThread(); @@ -280,6 +281,13 @@ int main(int argc, char** argv) pthread_t MessageThread; int ret = pthread_create (&MessageThread, NULL, (void* (*)(void*)) &messageThread, &config); + if ( ret != 0 ) + log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); + + // create alarm message thread + pthread_t AlarmMessageThread; + ret = pthread_create (&AlarmMessageThread, NULL, (void* (*)(void*)) &alarmMessageThread, &config); + if ( ret != 0 ) log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); @@ -377,6 +385,7 @@ int main(int argc, char** argv) string IPaddr = (*pt1).IPAddr; sysConfig->setConfig("ProcMgr", "IPAddr", IPaddr); + sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", IPaddr); log.writeLog(__LINE__, "set ProcMgr IPaddr to " + IPaddr, LOG_TYPE_DEBUG); @@ -406,6 +415,13 @@ int main(int argc, char** argv) pthread_t MessageThread; int ret = pthread_create (&MessageThread, NULL, (void* (*)(void*)) &messageThread, &config); + if ( ret != 0 ) + log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); + + // create alarm message thread + pthread_t AlarmMessageThread; + ret = pthread_create (&AlarmMessageThread, NULL, (void* (*)(void*)) &alarmMessageThread, &config); + if ( ret != 0 ) log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); } @@ -513,8 +529,143 @@ static void messageThread(Configuration config) sleep(60); } } +} - return; +/****************************************************************************************** +* @brief alarmMesssageThread +* +* purpose: Read incoming alarm messages +* +******************************************************************************************/ +static void alarmMessageThread(Configuration config) +{ + ProcessLog log; + ProcessManager processManager(config, log); + Oam oam; + + ByteStream msg; + + //check for running active, then launch + while (true) + { + if ( !runStandby) + break; + + sleep (1); + } + + log.writeLog(__LINE__, "Alarm Message Thread started ..", LOG_TYPE_DEBUG); + + //read and cleanup port before trying to use + try + { + Config* sysConfig = Config::makeConfig(); + string port = sysConfig->getConfig("ProcMgr_Alarm", "Port"); + string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1"; + + if ( !rootUser) + cmd = "sudo fuser -k " + port + "/tcp >/dev/null 2>&1"; + + system(cmd.c_str()); + } + catch (...) + { + } + + // + //waiting for request + // + IOSocket fIos; + + for (;;) + { + try + { + MessageQueueServer procmgr("ProcMgr_Alarm"); + + for (;;) + { + try + { + fIos = procmgr.accept(); + + try + { + msg = fIos.read(); + + if (msg.length() <= 0) + continue; + + //log.writeLog(__LINE__, "MSG RECEIVED: Process Alarm Message"); + + ByteStream::byte alarmID; + std::string componentID; + ByteStream::byte state; + std::string ModuleName; + std::string processName; + ByteStream::byte pid; + ByteStream::byte tid; + + msg >> alarmID; + msg >> componentID; + msg >> state; + msg >> ModuleName; + msg >> processName; + msg >> pid; + msg >> tid; + + Alarm calAlarm; + + calAlarm.setAlarmID (alarmID); + calAlarm.setComponentID (componentID); + calAlarm.setState (state); + calAlarm.setSname (ModuleName); + calAlarm.setPname (processName); + calAlarm.setPid (pid); + calAlarm.setTid (tid); + + ALARMManager aManager; + aManager.processAlarmReport(calAlarm); + } + catch (exception& ex) + { + string error = ex.what(); + log.writeLog(__LINE__, "EXCEPTION ERROR on read for ProcMgr_Alarm:" + error, LOG_TYPE_ERROR); + continue; + } + catch (...) + { + log.writeLog(__LINE__, "EXCEPTION ERROR on read for ProcMgr_Alarm: Caught unknown exception!", LOG_TYPE_ERROR); + continue; + } + } + catch (exception& ex) + { + string error = ex.what(); + log.writeLog(__LINE__, "EXCEPTION ERROR on accept for ProcMgr_Alarm:" + error, LOG_TYPE_ERROR); + continue; + } + catch (...) + { + log.writeLog(__LINE__, "EXCEPTION ERROR on accept for ProcMgr_Alarm: Caught unknown exception!", LOG_TYPE_ERROR); + continue; + } + } + } + catch (exception& ex) + { + string error = ex.what(); + log.writeLog(__LINE__, "EXCEPTION ERROR on MessageQueueServer for ProcMgr_Alarm:" + error, LOG_TYPE_ERROR); + + sleep(1); + } + catch (...) + { + log.writeLog(__LINE__, "EXCEPTION ERROR on MessageQueueServer for ProcMgr_Alarm: Caught unknown exception!", LOG_TYPE_ERROR); + + sleep(1); + } + } } /****************************************************************************************** diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 9fb9e399f..6a87495ba 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2909,44 +2909,6 @@ void processMSG(messageqcpp::IOSocket* cfIos) break; } - /* - case PROCESSALARM: - { - log.writeLog(__LINE__, "MSG RECEIVED: Process Alarm Message"); - - ByteStream::byte alarmID; - std::string componentID; - ByteStream::byte state; - std::string ModuleName; - std::string processName; - ByteStream::byte pid; - ByteStream::byte tid; - - msg >> alarmID; - msg >> componentID; - msg >> state; - msg >> ModuleName; - msg >> processName; - msg >> pid; - msg >> tid; - - Alarm calAlarm; - - calAlarm.setAlarmID (alarmID); - calAlarm.setComponentID (componentID); - calAlarm.setState (state); - calAlarm.setSname (ModuleName); - calAlarm.setPname (processName); - calAlarm.setPid (pid); - calAlarm.setTid (tid); - - ALARMManager aManager; - aManager.processAlarmReport(calAlarm); - - break; - } - - */ default: log.writeLog(__LINE__, "MSG RECEIVED: Invalid type" ); break; @@ -9524,6 +9486,7 @@ int ProcessManager::switchParentOAMModule(std::string newActiveModuleName) newActiveIPaddr = (*pt2).IPAddr; sysConfig4->setConfig("ProcMgr", "IPAddr", newActiveIPaddr); + sysConfig4->setConfig("ProcMgr_Alarm", "IPAddr", newActiveIPaddr); sysConfig4->setConfig("ProcStatusControl", "IPAddr", newActiveIPaddr); sysConfig4->setConfig("DBRM_Controller", "IPAddr", newActiveIPaddr); @@ -10159,6 +10122,7 @@ int ProcessManager::OAMParentModuleChange() localIPaddr = (*pt1).IPAddr; sysConfig4->setConfig("ProcMgr", "IPAddr", localIPaddr); + sysConfig4->setConfig("ProcMgr_Alarm", "IPAddr", localIPaddr); sysConfig4->setConfig("ProcStatusControl", "IPAddr", localIPaddr); sysConfig4->setConfig("DBRM_Controller", "IPAddr", localIPaddr); diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 4bbeca586..61233ce28 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -80,6 +80,7 @@ void stopSystemThread(oam::DeviceNetworkList devicenetworklist); void startModuleThread(std::string moduleName); void stopModuleThread(std::string moduleName); void processMSG(messageqcpp::IOSocket* fIos); + void sendUpgradeRequest(); /** @brief Timeset for Milleseconds diff --git a/procmon/main.cpp b/procmon/main.cpp index 8cd131455..d6b14abf4 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -373,6 +373,7 @@ int main(int argc, char** argv) string IPaddr = (*pt1).IPAddr; sysConfig->setConfig("ProcMgr", "IPAddr", IPaddr); + sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", IPaddr); log.writeLog(__LINE__, "set ProcMgr IPaddr to Old Standby Module: " + IPaddr, LOG_TYPE_DEBUG); @@ -781,7 +782,7 @@ int main(int argc, char** argv) log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); //mysql status monitor thread - if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) || + if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) || (PMwithUM == "y") ) { diff --git a/tools/configMgt/autoConfigure.cpp b/tools/configMgt/autoConfigure.cpp index 5458865dd..9947b9161 100644 --- a/tools/configMgt/autoConfigure.cpp +++ b/tools/configMgt/autoConfigure.cpp @@ -1060,6 +1060,7 @@ int main(int argc, char* argv[]) sysConfigNew->setConfig(parentProcessMonitor, "IPAddr", parentOAMModuleIPAddr); sysConfigNew->setConfig(parentProcessMonitor, "Port", "8800"); sysConfigNew->setConfig("ProcMgr", "IPAddr", parentOAMModuleIPAddr); + sysConfigNew->setConfig("ProcMgr_Alarm", "IPAddr", parentOAMModuleIPAddr); sysConfigNew->setConfig("ProcStatusControl", "IPAddr", parentOAMModuleIPAddr); string parentServerMonitor = systemParentOAMModuleName + "_ServerMonitor"; sysConfigNew->setConfig(parentServerMonitor, "IPAddr", parentOAMModuleIPAddr); diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index b0decfda2..3318a9ac8 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -362,6 +362,20 @@ CalpontSystemCatalog::ColType Func_simple_case::operationType(FunctionParm& fp, } +bool Func_simple_case::getBoolVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, operationColType); + + if (isNull) + return joblist::BIGINTNULL; + + return parm[i + 1]->data()->getBoolVal(row, isNull); +} + + int64_t Func_simple_case::getIntVal(Row& row, FunctionParm& parm, bool& isNull, @@ -465,6 +479,18 @@ CalpontSystemCatalog::ColType Func_searched_case::operationType(FunctionParm& fp return caseOperationType(fp, resultType, false); } +bool Func_searched_case::getBoolVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + uint64_t i = searched_case_cmp(row, parm, isNull); + + if (isNull) + return joblist::BIGINTNULL; + + return parm[i + 1]->data()->getBoolVal(row, isNull); +} int64_t Func_searched_case::getIntVal(Row& row, FunctionParm& parm, diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index 3daf279fd..6300084cd 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -70,6 +70,11 @@ public: execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + bool getBoolVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -112,6 +117,11 @@ public: execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + bool getBoolVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/winport/win_setup_mysql_part2.sql b/utils/winport/win_setup_mysql_part2.sql index 0a2a972fb..3ac04dcad 100644 --- a/utils/winport/win_setup_mysql_part2.sql +++ b/utils/winport/win_setup_mysql_part2.sql @@ -11,7 +11,6 @@ CREATE FUNCTION calcleartablelock RETURNS STRING SONAME 'libcalmysql.dll'; CREATE FUNCTION calgetsqlcount RETURNS STRING SONAME 'libcalmysql.dll'; CREATE FUNCTION mcssystemready RETURNS INTEGER SONAME 'libcalmysql.dll'; CREATE FUNCTION mcssystemreadonly RETURNS INTEGER SONAME 'libcalmysql.dll'; -CREATE FUNCTION mcswritessuspended RETURNS INTEGER SONAME 'libcalmysql.dll'; create database if not exists calpontsys; use calpontsys;