diff --git a/CMakeLists.txt b/CMakeLists.txt index 69f8cdb77..b4fb6e1e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,3 +90,8 @@ set(COMPONENTS foreach(component ${COMPONENTS}) add_subdirectory(${component}) endforeach() + +add_dependencies(udf_mysql GenError) +add_dependencies(funcexp GenError) +add_dependencies(oamcpp GenError) +add_dependencies(regr_mysql GenError) diff --git a/cmake/ColumnstoreLibrary.cmake b/cmake/ColumnstoreLibrary.cmake index 6e0b97b55..e9f35bbf7 100644 --- a/cmake/ColumnstoreLibrary.cmake +++ b/cmake/ColumnstoreLibrary.cmake @@ -82,16 +82,6 @@ macro(columnstore_shared_library libname) columnstore_install_target(${libname} ${ENGINE_LIBDIR}) endmacro() -macro(columnstore_static_library libname) - add_definitions(-fPIC -DPIC) - add_library(${libname} STATIC ${ARGN}) -endmacro() - -macro(columnstore_shared_library libname) - add_library(${libname} SHARED ${ARGN}) - columnstore_install_target(${libname} ${ENGINE_LIBDIR}) -endmacro() - macro(columnstore_library libname) if(COLUMNSTORE_STATIC_LIBRARIES) columnstore_static_library(${libname} ${ARGN}) @@ -109,15 +99,6 @@ macro(columnstore_link libname) target_link_libraries(${libname} ${ARGN}) endmacro() -macro(columnstore_mysql_plugin_library libname) - add_library(${libname} SHARED ${ARGN}) - columnstore_install_target(${libname} ${MARIADB_PLUGINDIR}) -endmacro() - -macro(columnstore_link libname) - target_link_libraries(${libname} ${ARGN}) -endmacro() - macro(columnstore_executable executable_name) if(WITH_COLUMNSTORE_ASAN) add_executable(${executable_name} ${ARGN} ${CMAKE_BINARY_DIR}/asan_options.cpp) diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 0d8706c91..20a78c763 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -20,7 +20,7 @@ #include #include #include "exceptclasses.h" -#include "basic/conststring.h" +#include "conststring.h" #include "mcs_datatype_basic.h" #include "mcs_numeric_limits.h" #include "mcs_data_condition.h" @@ -794,7 +794,7 @@ class MinMaxPartitionInfo : public MinMaxInfo uint64_t m_status; public: - MinMaxPartitionInfo() : m_status(0) {}; + MinMaxPartitionInfo() : m_status(0){}; explicit MinMaxPartitionInfo(const BRM::EMEntry& entry); void set_invalid() { @@ -1100,8 +1100,7 @@ class TypeHandlerBit : public TypeHandler idbassert(0); // QQ return 1; } - std::string format(const SimpleValue& /*v*/, - const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } @@ -1134,7 +1133,7 @@ class TypeHandlerBit : public TypeHandler const ConvertFromStringParam& /*prm*/, const std::string& /*str*/, bool& /*pushWarning*/) const override; - const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& /*attr*/) const override +const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& /*attr*/) const override { idbassert(0); return nullptr; @@ -2033,8 +2032,7 @@ class TypeHandlerReal : public TypeHandler { return {}; // QQ: real types were not handled in IDB_format() } - std::string format(const SimpleValue& /*v*/, - const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } @@ -2330,8 +2328,7 @@ class TypeHandlerBlob : public TypeHandlerStr { return storeValueToFieldBlobText(row, pos, f); } - std::string format(const SimpleValue& /*v*/, - const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } @@ -2357,8 +2354,7 @@ class TypeHandlerText : public TypeHandlerStr { return storeValueToFieldBlobText(row, pos, f); } - std::string format(const SimpleValue& /*v*/, - const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } @@ -2389,8 +2385,7 @@ class TypeHandlerClob : public TypeHandlerStr idbassert(0); // QQ return 1; } - std::string format(const SimpleValue& /*v*/, - const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } diff --git a/datatypes/mcs_string.h b/datatypes/mcs_string.h index 0714945aa..dbd8feb92 100644 --- a/datatypes/mcs_string.h +++ b/datatypes/mcs_string.h @@ -17,8 +17,8 @@ #pragma once -#include "basic/conststring.h" -#include "mariadb_charset/collation.h" // class Charset +#include "conststring.h" +#include "collation.h" // class Charset namespace datatypes { diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index d4fdf6280..4ed7d69cc 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -42,4 +42,4 @@ columnstore_library( ${FLEX_ddl_scan_OUTPUTS} ) -columnstore_link(ddlpackage loggingcpp messageqcpp) +columnstore_link(ddlpackage loggingcpp) diff --git a/dbcon/ddlpackage/sqlparser.h b/dbcon/ddlpackage/sqlparser.h index c45b903aa..460433fc6 100644 --- a/dbcon/ddlpackage/sqlparser.h +++ b/dbcon/ddlpackage/sqlparser.h @@ -27,7 +27,7 @@ */ #include -#include "mariadb_charset/collation.h" // CHARSET_INFO +#include "collation.h" // CHARSET_INFO #include "ddlpkg.h" #include "mariadb_my_sys.h" // myf, MYF() @@ -87,7 +87,7 @@ struct pass_to_bison myf utf8_flag; pass_to_bison(ParseTree* pt) - : fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)) {}; + : fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)){}; }; class SqlParser diff --git a/dbcon/ddlpackageproc/CMakeLists.txt b/dbcon/ddlpackageproc/CMakeLists.txt index 151d039fb..3eed93356 100644 --- a/dbcon/ddlpackageproc/CMakeLists.txt +++ b/dbcon/ddlpackageproc/CMakeLists.txt @@ -13,4 +13,4 @@ set(ddlpackageproc_LIB_SRCS ) columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS}) -columnstore_link(ddlpackageproc loggingcpp oamcpp messageqcpp) +columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/dbcon/dmlpackage/CMakeLists.txt b/dbcon/dmlpackage/CMakeLists.txt index 0a138cca5..a0e4ddffa 100644 --- a/dbcon/dmlpackage/CMakeLists.txt +++ b/dbcon/dmlpackage/CMakeLists.txt @@ -41,4 +41,4 @@ columnstore_library( ${FLEX_dml_scan_OUTPUTS} ) -columnstore_link(dmlpackage loggingcpp messageqcpp) +columnstore_link(dmlpackage loggingcpp) diff --git a/dbcon/dmlpackageproc/CMakeLists.txt b/dbcon/dmlpackageproc/CMakeLists.txt index 8a862ccb5..177bb799b 100644 --- a/dbcon/dmlpackageproc/CMakeLists.txt +++ b/dbcon/dmlpackageproc/CMakeLists.txt @@ -13,4 +13,4 @@ set(dmlpackageproc_LIB_SRCS ) columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS}) -columnstore_link(dmlpackageproc loggingcpp oamcpp messageqcpp) +columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/dbcon/execplan/CMakeLists.txt b/dbcon/execplan/CMakeLists.txt index 4edaf97d0..6e1acbb98 100755 --- a/dbcon/execplan/CMakeLists.txt +++ b/dbcon/execplan/CMakeLists.txt @@ -46,12 +46,4 @@ set(execplan_LIB_SRCS ) columnstore_library(execplan ${execplan_LIB_SRCS}) -columnstore_link( - execplan - messageqcpp - ${ENGINE_DT_LIB} - pron - oamcpp - loggingcpp - querytele -) +columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp) diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index cfb748c54..dee640c3e 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -49,7 +49,7 @@ #undef max #include "mcs_datatype.h" -#include "mariadb_charset/collation.h" // CHARSET_INFO, class Charset +#include "collation.h" // CHARSET_INFO, class Charset #include "nullstring.h" class ExecPlanTest; @@ -75,8 +75,7 @@ const int32_t IDB_VTABLE_ID = CNX_VTABLE_ID; /** * A struct to hold a list of table partitions. */ -struct Partitions -{ +struct Partitions { std::vector fPartNames; void serialize(messageqcpp::ByteStream& b) const { @@ -97,11 +96,12 @@ struct Partitions b >> t; fPartNames.push_back(t); } + } }; -bool operator<(const Partitions& a, const Partitions& b); -bool operator==(const Partitions& a, const Partitions& b); -bool operator!=(const Partitions& a, const Partitions& b); // for GCC9 +bool operator <(const Partitions& a, const Partitions& b); +bool operator ==(const Partitions& a, const Partitions& b); +bool operator !=(const Partitions& a, const Partitions& b); // for GCC9 /** The CalpontSystemCatalog class * @@ -259,9 +259,9 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog public: ColType() = default; ColType(const ColType& rhs); - ColType(int32_t colWidth_, int32_t scale_, int32_t precision_, const ConstraintType& constraintType_, - const DictOID& ddn_, int32_t colPosition_, int32_t compressionType_, OID columnOID_, - const ColDataType& colDataType_); + ColType(int32_t colWidth_, int32_t scale_, int32_t precision_, + const ConstraintType& constraintType_, const DictOID& ddn_, int32_t colPosition_, + int32_t compressionType_, OID columnOID_, const ColDataType& colDataType_); ColType& operator=(const ColType& rhs); CHARSET_INFO* getCharset() const; @@ -1301,8 +1301,8 @@ const int OID_SYSCOLUMN_MINVALUE = SYSCOLUMN_BASE + 19; /** @brief min va const int OID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_BASE + 20; /** @brief max value col */ const int OID_SYSCOLUMN_COMPRESSIONTYPE = SYSCOLUMN_BASE + 21; /** @brief compression type */ const int OID_SYSCOLUMN_NEXTVALUE = SYSCOLUMN_BASE + 22; /** @brief next value */ -const int OID_SYSCOLUMN_CHARSETNUM = SYSCOLUMN_BASE + 23; /** @brief character set number for the column */ -const int SYSCOLUMN_MAX = SYSCOLUMN_BASE + 24; // be sure this is one more than the highest # +const int OID_SYSCOLUMN_CHARSETNUM = SYSCOLUMN_BASE + 23; /** @brief character set number for the column */ +const int SYSCOLUMN_MAX = SYSCOLUMN_BASE + 24; // be sure this is one more than the highest # /***************************************************** * SYSTABLE columns dictionary OID definition diff --git a/dbcon/execplan/clientrotator.cpp b/dbcon/execplan/clientrotator.cpp index 4e0d1303a..395d76482 100644 --- a/dbcon/execplan/clientrotator.cpp +++ b/dbcon/execplan/clientrotator.cpp @@ -47,6 +47,8 @@ using namespace logging; #include "clientrotator.h" +//#include "idb_mysql.h" + /** Debug macro */ #ifdef INFINIDB_DEBUG #define IDEBUG(x) \ diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index b6e976d46..434c2c768 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -39,7 +39,7 @@ #include "returnedcolumn.h" #include "dataconvert.h" -#include "mariadb_charset/collation.h" // CHARSET_INFO +#include "collation.h" // CHARSET_INFO namespace messageqcpp { diff --git a/dbcon/execplan/pseudocolumn.cpp b/dbcon/execplan/pseudocolumn.cpp index 38f2be10f..a6055489e 100644 --- a/dbcon/execplan/pseudocolumn.cpp +++ b/dbcon/execplan/pseudocolumn.cpp @@ -28,7 +28,7 @@ using namespace std; #include "basic/string_utils.h" #include "bytestream.h" -#include "mariadb_charset/collation.h" +#include "collation.h" using namespace messageqcpp; diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index a63e5c82c..3783ccf14 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -68,15 +68,7 @@ columnstore_library(joblist ${joblist_LIB_SRCS}) target_include_directories( joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR} ) -columnstore_link( - joblist - loggingcpp - boost_thread - oamcpp - querytele - messageqcpp - statistics_manager -) +columnstore_link(joblist loggingcpp) if(WITH_ORDERBY_UT) columnstore_executable(job_orderby_tests orderby-tests.cpp) diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 7ba090460..96add786b 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -71,7 +71,7 @@ using namespace dataconvert; #include "jlf_tuplejoblist.h" using namespace joblist; -#include "statistics_manager/statistics.h" +#include "statistics.h" #ifdef __clang__ #pragma clang diagnostic push diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 168732a07..4000d1c96 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -64,7 +64,7 @@ if(COMMAND mysql_add_plugin) ${PLUGIN_EXEC_LIBS} ${PLUGIN_WRITE_LIBS} joblist_server - statistics_manager + ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${S3API_DEPS} threadpool @@ -90,10 +90,10 @@ else() ${S3API_DEPS} ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} + ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool loggingcpp - statistics_manager marias3 ) # define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index e6619f7c0..906ae3631 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -127,9 +127,9 @@ using namespace funcexp; #include "ha_mcs_sysvars.h" #include "ha_mcs_datatype.h" +#include "statistics.h" #include "ha_mcs_logging.h" #include "ha_subquery.h" -#include "statistics_manager/statistics.h" using namespace std; diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index 204988d60..ae96bc807 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -14,17 +14,10 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - -/* One include file to deal with all the MySQL pollution of the - global namespace - - Don't include ANY mysql headers anywhere except here! - - WARN: if any cmake build target uses this include file, - GenError from server must be added to the target dependencies - to generate mysqld_error.h used below -*/ - +// One include file to deal with all the MySQL pollution of the +// global namespace +// +// Don't include ANY mysql headers anywhere except here! #pragma once #ifdef TEST_MCSCONFIG_H diff --git a/ddlproc/CMakeLists.txt b/ddlproc/CMakeLists.txt index 0da0d740b..64924fb79 100644 --- a/ddlproc/CMakeLists.txt +++ b/ddlproc/CMakeLists.txt @@ -6,4 +6,4 @@ set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp) columnstore_executable(DDLProc ${DDLProc_SRCS}) -columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool loggingcpp) +columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp) diff --git a/dmlproc/CMakeLists.txt b/dmlproc/CMakeLists.txt index 036d7b0ea..af9ed6492 100644 --- a/dmlproc/CMakeLists.txt +++ b/dmlproc/CMakeLists.txt @@ -12,6 +12,7 @@ columnstore_link( DMLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} + ${NETSNMP_LIBRARIES} threadpool ddlcleanuputil batchloader diff --git a/oam/oamcpp/CMakeLists.txt b/oam/oamcpp/CMakeLists.txt index cdce7a55b..d5b4ac307 100644 --- a/oam/oamcpp/CMakeLists.txt +++ b/oam/oamcpp/CMakeLists.txt @@ -5,6 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp) columnstore_library(oamcpp ${oamcpp_LIB_SRCS}) -columnstore_link(oamcpp loggingcpp idbboot) +columnstore_link(oamcpp loggingcpp) target_compile_options(oamcpp PRIVATE -Wno-unused-result) diff --git a/oamapps/columnstoreSupport/CMakeLists.txt b/oamapps/columnstoreSupport/CMakeLists.txt new file mode 100644 index 000000000..feaf17aa2 --- /dev/null +++ b/oamapps/columnstoreSupport/CMakeLists.txt @@ -0,0 +1,16 @@ +include_directories(${ENGINE_COMMON_INCLUDES}) + +# ########## next target ############### + +set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp) + +columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS}) +target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result) +columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS}) + +columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(configReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(logReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR}) diff --git a/oamapps/columnstoreSupport/alarmReport.sh b/oamapps/columnstoreSupport/alarmReport.sh new file mode 100755 index 000000000..ec81e2228 --- /dev/null +++ b/oamapps/columnstoreSupport/alarmReport.sh @@ -0,0 +1,39 @@ +#! /bin/sh +# +# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $ +# +if [ $1 ] ; then + SERVER=$1 +else + SERVER="localhost" +fi + +if [ $2 ] ; then + DATE=$2 +else + DATE=" " +fi + +#get temp directory +tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/logReport.log + +{ +echo " " +echo "******************** Alarm Report for $SERVER ********************" +echo " " + +echo "-- Today's Alarms --" +echo " " +cat /var/log/mariadb/columnstore/alarm.log 2>/dev/null + +if test -f /var/log/mariadb/columnstore/archive/alarm.log-$DATE ; then + echo "-- Archived Alarms --" + echo " " + cat /var/log/mariadb/columnstore/archive/alarm.log-$DATE 2>/dev/null +fi + +} > ${tmpDir}/logReport.log + +exit 0 diff --git a/oamapps/columnstoreSupport/approximateRowCount.sh b/oamapps/columnstoreSupport/approximateRowCount.sh new file mode 100755 index 000000000..26f3e41aa --- /dev/null +++ b/oamapps/columnstoreSupport/approximateRowCount.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# +# Estimates the row count for a given table. Uses number of extents * 8M for the estimate. +# + +# +# Initialize variables. +# + +if [ -z "$MYSQLCMD" ]; then + MYSQLCMD="mysql -u root" +fi + +# +# Validate that there are two parameters - schema and table. +# +if [ $# -ne 2 ]; then + echo "" + echo "Reports the approximate row count for the given table." + echo "" + echo "Parameters:" + echo " Schema" + echo " Table" +fi +db=$1 +table=$2 + +# +# Validate that the table exists. +# +sql="select count(*) from systable where \`schema\`='$db' and tablename='$table';" +count=`$MYSQLCMD calpontsys --skip-column-names -e "$sql;"` +if [ $count -le 0 ]; then + echo "" + echo "$db.$table does not exist in Columnstore." + echo "" + exit 1 +fi + +# +# Grab the objectid and column width for a column in the table. +# +sql="select objectid from syscolumn where \`schema\`='$db' and tablename='$table' limit 1;" +objectid=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"` +sql="select columnlength from syscolumn where objectid=$objectid;" +colWidth=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"` + +# +# Use editem to count the extents. +# +extentCount=`editem -o $objectid | wc -l` +let extentCount-=2 # Take out the 2 extra rows for header and blank line at end. +let approximateRowCount=$extentCount*8192*1024; + +echo "" +echo "Approximate row count for $db.$table is $approximateRowCount." +echo "" + +exit 0 diff --git a/oamapps/columnstoreSupport/bulklogReport.sh b/oamapps/columnstoreSupport/bulklogReport.sh new file mode 100755 index 000000000..83076f4eb --- /dev/null +++ b/oamapps/columnstoreSupport/bulklogReport.sh @@ -0,0 +1,30 @@ +#! /bin/sh +# +# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $ +# +if [ $1 ] ; then + MODULE=$1 +else + MODULE="pm1" +fi + +if [ $2 ] ; then + OUT_FILE=$2 +else + OUT_FILE=${MODULE}_logReport.txt +fi + +{ + +if test -d /var/lib/columnstore/data/bulk ; then + echo " " + echo "-- Check for Errors in Bulk Logs --" + echo " " + echo "################# egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err #################" + echo " " + egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err 2>/dev/null +fi + +} >> $OUT_FILE + +exit 0 diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp new file mode 100644 index 000000000..51bd670b9 --- /dev/null +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -0,0 +1,897 @@ +/* Copyright (C) 2013 Calpont Corp. */ +/* Copyright (C) 2016 MariaDB Corporation */ + +/****************************************************************************************** + * $Id: columnstoreSupport.cpp 64 2006-10-12 22:21:51Z dhill $ + * + * + * + ******************************************************************************************/ +/** + * @file + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "stdio.h" +#include "ctype.h" +#include +#include +#include + +#include "mcsconfig.h" +#include "liboamcpp.h" +#include "configcpp.h" +#include "installdir.h" +#include "mcsSupportUtil.h" +#include "columnstoreversion.h" + +using namespace std; +using namespace oam; +using namespace config; + +typedef struct Child_Module_struct +{ + std::string moduleName; + std::string moduleIP; + std::string hostName; +} ChildModule; + +typedef std::vector ChildModuleList; + +string currentDate; +string systemName; +string localModule; +string localModuleHostName; +ChildModuleList childmodulelist; +ChildModuleList parentmodulelist; +ChildModule childmodule; + +string rootPassword = ""; +string debug_flag = "0"; +string mysqlpw = " "; +string tmpDir; + +int runningThreads = 0; +pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; + +typedef boost::tuple threadInfo_t; + +bool LOCAL = false; + +void* childReportThread(threadInfo_t* st) +{ + assert(st); + ChildModuleList::iterator& list = boost::get<0>(*st); + string reportType = boost::get<1>(*st); + + string remoteModuleName = (*list).moduleName; + string remoteModuleIP = (*list).moduleIP; + string remoteHostName = (*list).hostName; + + pthread_mutex_lock(&mutex1); + runningThreads++; + // cout << "++ " << runningThreads << endl; + pthread_mutex_unlock(&mutex1); + + string outputFile; + + if (reportType == "log") + { + outputFile = remoteModuleName + "_" + reportType + "Report.tar.gz"; + } + else + { + outputFile = remoteModuleName + "_" + reportType + "Report.txt"; + + FILE* pOutputFile = fopen(outputFile.c_str(), "a"); + if (pOutputFile == NULL) + { + printf("Could not open file: %s", outputFile.c_str()); + exit(1); + } + + fprintf(pOutputFile, + "********************************************************************************\n" + "\n" + " System %s\n" + " columnstoreSupportReport script ran from Module %s on %s\n" + " SoftwareVersion = %s-%s" + "\n" + "********************************************************************************\n" + "\n" + " %s report\n" + "\n" + "********************************************************************************\n", + systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), + columnstore_release.c_str(), reportType.c_str()); + } + + cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl; + + string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + ";" + reportType + "Report.sh " + + remoteModuleName + "' " + debug_flag + " - forcetty"; + int rtnCode = system(cmd.c_str()); + + if (WEXITSTATUS(rtnCode) != 0) + { + cout << "Error with running remote_command.sh, exiting..." << endl; + } + + cmd = "remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile + + " > /dev/null 2>&1"; + rtnCode = system(cmd.c_str()); + + if (WEXITSTATUS(rtnCode) != 0) + cout << "ERROR: failed to retrieve " << tmpDir << "/" << outputFile << " from " + remoteHostName << endl; + + pthread_mutex_lock(&mutex1); + runningThreads--; + // cout << "-- " << runningThreads << endl; + pthread_mutex_unlock(&mutex1); + + // exit thread + pthread_exit(0); +} + +void* reportThread(string* reporttype) +{ + assert(reporttype); + string reportType = *reporttype; + + Oam oam; + + pthread_mutex_lock(&mutex1); + runningThreads++; + // cout << "++ " << runningThreads << endl; + pthread_mutex_unlock(&mutex1); + + string outputFile = localModule + "_" + reportType + "Report.txt"; + + FILE* pOutputFile = fopen(outputFile.c_str(), "a"); + if (pOutputFile == NULL) + { + printf("Could not open file: %s", outputFile.c_str()); + exit(1); + } + // get local report + fprintf(pOutputFile, + "********************************************************************************\n" + "\n" + " System %s\n" + " columnstoreSupportReport script ran from Module %s on %s\n" + " SoftwareVersion = %s-%s" + "\n" + "********************************************************************************\n" + "\n" + " %s report\n" + "\n" + "********************************************************************************\n", + systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), + columnstore_release.c_str(), reportType.c_str()); + + fclose(pOutputFile); + // run on child servers and get report + if (!LOCAL) + { + ChildModuleList::iterator list1 = childmodulelist.begin(); + + for (; list1 != childmodulelist.end(); list1++) + { + threadInfo_t* st = new threadInfo_t; + *st = boost::make_tuple(list1, reportType); + + pthread_t childreportthread; + int status = pthread_create(&childreportthread, NULL, (void* (*)(void*)) & childReportThread, st); + + if (status != 0) + { + cout << "ERROR: childreportthread: pthread_create failed, return status = " + oam.itoa(status) + << endl; + } + + sleep(1); + } + } + + if (reportType == "log") + { + // run log config on local server + cout << "Get log config data for " + localModule << endl; + + string cmd = "logReport.sh " + localModule + " " + outputFile; + system(cmd.c_str()); + } + else + { + string cmd = reportType + "Report.sh " + localModule + " " + outputFile; + system(cmd.c_str()); + + if (reportType == "config") + { + pOutputFile = fopen(outputFile.c_str(), "a"); + if (pOutputFile == NULL) + { + printf("Could not open file: %s", outputFile.c_str()); + exit(1); + } + + fprintf(pOutputFile, + "\n******************** System Network Configuration ******************************\n\n"); + getSystemNetworkConfig(pOutputFile); + + fprintf(pOutputFile, + "\n******************** System Module Configure **********************************\n\n"); + getModuleTypeConfig(pOutputFile); + + fprintf(pOutputFile, + "\n******************** System Storage Configuration *****************************\n\n"); + getStorageConfig(pOutputFile); + + fprintf(pOutputFile, + "\n******************** System Storage Status ************************************\n\n"); + getStorageStatus(pOutputFile); + + // BT: most of this is tedious to collect and can be manually looked up in the debug.log file + // fprintf(pOutputFile,"\n******************** System Status + // ********************************************\n\n"); printSystemStatus(pOutputFile); + // printProcessStatus(pOutputFile); + // printAlarmSummary(pOutputFile); + // + // fprintf(pOutputFile,"\n******************** System Directories + // ***************************************\n\n"); getSystemDirectories(pOutputFile); + + boost::filesystem::path configFile = + std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml"); + boost::filesystem::copy_file(configFile, "./Columnstore.xml", + boost::filesystem::copy_options::overwrite_existing); + boost::filesystem::path SMconfigFile = + std::string(MCSSYSCONFDIR) + std::string("/columnstore/storagemanager.cnf"); + boost::filesystem::copy_file(SMconfigFile, "./storagemanager.cnf", + boost::filesystem::copy_options::overwrite_existing); + system("sed -i 's/.*aws_access_key_id.*/aws_access_key_id={PRIVATE}/' ./storagemanager.cnf"); + system("sed -i 's/.*aws_secret_access_key.*/aws_secret_access_key={PRIVATE}/' ./storagemanager.cnf"); + fclose(pOutputFile); + } + + /* + // TODO: This can be ported from mcsadmin if needed most info included does not seem useful at this time + if (reportType == "resource" ) + { + if (LOCAL) + { + fprintf(pOutputFile,"\n******************** mcsadmin getModuleResourceUsage + **************************\n\n"); string cmd = "mcsadmin getModuleResourceUsage " + localModule + " >> " + + outputFile; system(cmd.c_str()); + } + else + { + fprintf(pOutputFile,"\n******************** mcsadmin getSystemResourceUsage + **************************\n\n"); string cmd = "mcsadmin getSystemResourceUsage >> " + outputFile; + system(cmd.c_str()); + } + }*/ + } + + // exit thread + pthread_mutex_lock(&mutex1); + runningThreads--; + // cout << "-- " << runningThreads << endl; + pthread_mutex_unlock(&mutex1); + + pthread_exit(0); +} + +int main(int argc, char* argv[]) +{ + Oam oam; + + Config* sysConfig = Config::makeConfig(); + string SystemSection = "SystemConfig"; + string InstallSection = "Installation"; + + bool HARDWARE = false; + bool CONFIG = false; + bool DBMS = false; + bool RESOURCE = false; + bool LOG = false; + bool BULKLOG = false; + bool HADOOP = false; + + // get current time and date + time_t now; + now = time(NULL); + struct tm tm; + localtime_r(&now, &tm); + char timestamp[200]; + strftime(timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm); + currentDate = timestamp; + + char helpArg[3] = "-h"; + + // Get System Name + try + { + oam.getSystemConfig("SystemName", systemName); + } + catch (...) + { + systemName = "unassigned"; + } + + // get Local Module Name and Server Install Indicator + string singleServerInstall = "n"; + + oamModuleInfo_t st; + + try + { + st = oam.getModuleInfo(); + localModule = boost::get<0>(st); + } + catch (...) + { + cout << endl << "**** Failed : Failed to read Local Module Name" << endl; + exit(-1); + } + + if (argc == 1) + { + argv[1] = &helpArg[0]; + argc = 2; + } + + string DataFilePlugin; + + try + { + DataFilePlugin = sysConfig->getConfig(SystemSection, "DataFilePlugin"); + } + catch (...) + { + cout << "ERROR: Problem accessing Columnstore configuration file" << endl; + exit(-1); + } + + tmpDir = startup::StartUp::tmpDir(); + + for (int i = 1; i < argc; i++) + { + if (string("-h") == argv[i]) + { + cout << endl; + cout << "'columnstoreSupport' generates a Set of System Support Report Files in a tar file" << endl; + cout << "called columnstoreSupportReport.'system-name'.tar.gz in the local directory." << endl; + cout << "It should be run on the server with the DBRM front-end." << endl; + cout << "Check the Admin Guide for additional information." << endl; + cout << endl; + cout << "Usage: columnstoreSupport [-h][-a][-hw][-s][-c][-db][-r][-l][-bl][-lc][-p " + "'root-password'][-de]"; + + cout << endl; + cout << " -h help" << endl; + cout << " -a Output all Reports (excluding Bulk Logs Reports)" << endl; + cout << " -hw Output Hardware Reports only" << endl; + cout << " -c Output Configuration/Status Reports only" << endl; + cout << " -db Output DBMS Reports only" << endl; + cout << " -r Output Resource Reports only" << endl; + cout << " -l Output Columnstore Log/Alarms Reports only" << endl; + cout << " -bl Output Columnstore Bulk Log Reports only" << endl; + cout << " -lc Output Reports for Local Server only" << endl; + cout << " -p password (multi-server systems), root-password or 'ssh' to use 'ssh keys'" + << endl; + cout << " -de Debug Flag" << endl; + + exit(0); + } + else + { + if (string("-a") == argv[i]) + { + HARDWARE = true; + CONFIG = true; + DBMS = true; + RESOURCE = true; + LOG = true; + HADOOP = (DataFilePlugin.empty() ? false : true); + } + else if (string("-hw") == argv[i]) + HARDWARE = true; + else if (string("-c") == argv[i]) + CONFIG = true; + else if (string("-db") == argv[i]) + DBMS = true; + else if (string("-r") == argv[i]) + RESOURCE = true; + else if (string("-l") == argv[i]) + LOG = true; + else if (string("-bl") == argv[i]) + BULKLOG = true; + else if (string("-lc") == argv[i]) + LOCAL = true; + else if (string("-p") == argv[i]) + { + i++; + + if (argc == i) + { + cout << "ERROR: missing root password argument" << endl; + exit(-1); + } + + rootPassword = argv[i]; + + // add single quote for special characters + if (rootPassword != "ssh") + { + rootPassword = "'" + rootPassword + "'"; + } + } + else if (string("-mp") == argv[i]) + { + i++; + + if (argc == i) + { + cout << "ERROR: missing MariaDB Columnstore root user password argument" << endl; + exit(-1); + } + + mysqlpw = argv[i]; + mysqlpw = "'" + mysqlpw + "'"; + } + else if (string("-de") == argv[i]) + debug_flag = "1"; + else if (string("-hd") == argv[i]) + { + HADOOP = (DataFilePlugin.empty() ? false : true); + } + else + { + cout << "Invalid Option of '" << argv[i] << "', run with '-h' for help" << endl; + exit(1); + } + } + } + + // default to -a if nothing is set + if (!HARDWARE && !CONFIG && !DBMS && !RESOURCE && !LOG && !BULKLOG && !HADOOP) + { + HARDWARE = true; + CONFIG = true; + DBMS = true; + RESOURCE = true; + LOG = true; + HADOOP = (DataFilePlugin.empty() ? false : true); + } + + // get Parent OAM Module Name and setup of it's Custom OS files + string PrimaryUMModuleName; + + try + { + PrimaryUMModuleName = sysConfig->getConfig(SystemSection, "PrimaryUMModuleName"); + } + catch (...) + { + cout << "ERROR: Problem getting Parent OAM Module Name" << endl; + exit(-1); + } + + if (PrimaryUMModuleName == "unassigned") + PrimaryUMModuleName = localModule; + + if ((localModule != PrimaryUMModuleName) && DBMS) + { + char* pcommand = 0; + char* p; + string argument = "n"; + + while (true) + { + cout << endl << "You selected to get the DBMS data." << endl; + cout << "You need to run the columnstoreSupport command on module '" << PrimaryUMModuleName + << "' to get that information." << endl; + cout << "Or you can proceed on to get all data except the DBMS." << endl; + + pcommand = readline(" Do you want to proceed: (y or n) [n]: "); + + if (pcommand && *pcommand) + { + p = strtok(pcommand, " "); + argument = p; + free(pcommand); + pcommand = 0; + } + + if (pcommand) + { + free(pcommand); + pcommand = 0; + } + + if (argument == "y") + { + cout << endl; + break; + } + else if (argument == "n") + exit(1); + } + } + + // get number of worker-nodes, will tell us if a single server system + // get Parent OAM Module Name and setup of it's Custom OS files + try + { + string NumWorkers = sysConfig->getConfig("DBRM_Controller", "NumWorkers"); + + if (NumWorkers == "1") + singleServerInstall = "y"; + } + catch (...) + { + } + + if (singleServerInstall == "n" && !LOCAL) + if (HARDWARE || CONFIG || RESOURCE || LOG || HADOOP) + if (rootPassword.empty()) + { + cout << "ERROR: Multi-Module System, Password Argument required or use '-lc' option, check help for " + "more information" + << endl; + exit(-1); + } + + // get Parent OAM Module Name and setup of it's Custom OS files + // string parentOAMModuleName; + ChildModule parentOAMModule; + + try + { + parentOAMModule.moduleName = sysConfig->getConfig(SystemSection, "ParentOAMModuleName"); + } + catch (...) + { + cout << "ERROR: Problem getting Parent OAM Module Name" << endl; + exit(-1); + } + + // Get list of configured system modules + SystemModuleTypeConfig sysModuleTypeConfig; + + try + { + oam.getSystemConfig(sysModuleTypeConfig); + } + catch (...) + { + cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl; + exit(-1); + } + + string ModuleSection = "SystemModuleConfig"; + + for (unsigned int i = 0; i < sysModuleTypeConfig.moduletypeconfig.size(); i++) + { + string moduleType = sysModuleTypeConfig.moduletypeconfig[i].ModuleType; + int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount; + + if (moduleCount == 0) + // no modules equipped for this Module Type, skip + continue; + + // get IP addresses and Host Names + DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end(); listPT++) + { + string moduleName = (*listPT).DeviceName; + HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); + string moduleIPAddr = (*pt1).IPAddr; + string moduleHostName = (*pt1).HostName; + + if (moduleName == localModule) + { + localModuleHostName = moduleHostName; + } + + // save Child modules + if (moduleName != localModule && moduleType != "xm") + { + childmodule.moduleName = moduleName; + childmodule.moduleIP = moduleIPAddr; + childmodule.hostName = moduleHostName; + childmodulelist.push_back(childmodule); + } + + if (moduleName == parentOAMModule.moduleName) + { + parentOAMModule.moduleIP = moduleIPAddr; + parentOAMModule.hostName = moduleHostName; + parentOAMModule.moduleName = moduleName; + } + } + } // end of i for loop + + // create a clean Columnstore Support Report + system("rm -f *_configReport.txt"); + system("rm -f *_dbmsReport.txt"); + system("rm -f *_hardwareReport.txt"); + system("rm -f *_logReport.txt"); + system("rm -f *_bulklogReport.txt"); + system("rm -f *_resourceReport.txt"); + + // + // Configuration + // + if (CONFIG) + { + string reportType = "config"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // Alarms and Columnstore Logs + // + if (LOG) + { + string reportType = "log"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // Bulk Logs + // + if (BULKLOG) + { + string reportType = "bulklog"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // Hardware + // + if (HARDWARE) + { + string reportType = "hardware"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // Resources + // + if (RESOURCE) + { + string reportType = "resource"; + cout << "Get " + reportType + " report data for " + localModule << endl; + pthread_t reportthread; + int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); + if (status != 0) + { + cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); + } + sleep(1); + } + + // + // DBMS + // + if (DBMS) + { + cout << "Get dbms report data for " << localModule << endl; + + string outputFile = localModule + "_dbmsReport.txt"; + + FILE* pOutputFile = fopen(outputFile.c_str(), "w"); + if (pOutputFile == NULL) + { + cout << "Could not open file: " + outputFile << endl; + exit(1); + } + + fprintf(pOutputFile, + "********************************************************************************\n" + "\n" + " System %s\n" + " columnstoreSupportReport script ran from Module %s on %s\n" + " SoftwareVersion = %s-%s" + "\n" + "********************************************************************************\n" + "\n" + " DBMS report\n" + "\n" + "********************************************************************************\n", + systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), + columnstore_release.c_str()); + + fclose(pOutputFile); + + // run DBMS report on local server + bool FAILED = false; + + if (localModule != PrimaryUMModuleName) + { + cout << " FAILED: run columnstoreSupport on '" << PrimaryUMModuleName << "' to get the dbrm report" + << endl; + FAILED = true; + } + else + { + // check if mysql is supported and get info + string logFile = tmpDir + "/idbmysql.log"; + string columnstoreMysql = "mysql -u root "; + string cmd = columnstoreMysql + " -e 'status' > " + logFile + " 2>&1"; + system(cmd.c_str()); + + // check for mysql password set + string pwprompt = " "; + + if (checkLogStatus(logFile, "ERROR 1045")) + { + cout << "NOTE: MariaDB Columnstore root user password is set" << endl; + + // needs a password, was password entered on command line + if (mysqlpw == " ") + { + // go check columnstore.cnf + string file = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; + ifstream oldFile(file.c_str()); + + vector lines; + char line[200]; + string buf; + + while (oldFile.getline(line, 200)) + { + buf = line; + string::size_type pos = buf.find("password", 0); + + if (pos != string::npos) + { + string::size_type pos1 = buf.find("=", 0); + + if (pos1 != string::npos) + { + pos = buf.find("#", 0); + + if (pos == string::npos) + { + // password arg in columnstore.cnf, go get password + cout << "NOTE: Using password from columnstore.cnf" << endl; + mysqlpw = buf.substr(pos1 + 1, 80); + cout << mysqlpw << endl; + break; + } + } + } + } + + oldFile.close(); + + if (mysqlpw == " ") + { + cout << "NOTE: No password provide on command line or found uncommented in columnstore.cnf" + << endl; + cout << endl; + string prompt = " *** Enter MariaDB Columnstore password > "; + mysqlpw = getpass(prompt.c_str()); + } + } + + // check for mysql password set + pwprompt = "--password=" + mysqlpw; + + string cmd = columnstoreMysql + pwprompt + " -e 'status' > " + logFile + " 2>&1"; + system(cmd.c_str()); + + if (checkLogStatus(logFile, "ERROR 1045")) + { + cout << "FAILED: Failed login using MariaDB Columnstore root user password '" << mysqlpw << "'" + << endl; + FAILED = true; + } + } + + if (!FAILED) + { + string cmd = "dbmsReport.sh " + localModule + " " + outputFile + " " + std::string(MCSSUPPORTDIR) + + " " + pwprompt; + system(cmd.c_str()); + } + } + + /* + BT: This doesn't appear to do anything + fprintf(pOutputFile,"\n******************** Database Size Report + *************************************\n\n"); getStorageStatus(pOutputFile); + + string file = "databaseSizeReport"; + ifstream File (file.c_str()); + + if (File) + { + string cmd = "databaseSizeReport >> " + outputFile; + system(cmd.c_str()); + } + */ + + boost::filesystem::path configFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; + boost::filesystem::copy_file(configFile, "./columnstore.cnf", + boost::filesystem::copy_options::overwrite_existing); + } + + int wait = 0; + + while (true) + { + // cout << "check " << runningThreads << endl; + if (runningThreads < 1) + break; + + sleep(2); + wait++; + + // give it 60 minutes to complete + if (wait >= 3600 * 5) + { + cout << "Timed out (60 minutes) waiting for Requests to complete" << endl; + } + } + + system("unix2dos *Report.txt > /dev/null 2>&1"); + system( + "rm -rf columnstoreSupportReport;" + "mkdir columnstoreSupportReport;" + "mv *Report.txt columnstoreSupportReport/. > /dev/null 2>&1;" + "mv Columnstore.xml columnstoreSupportReport/. > /dev/null 2>&1;" + "mv columnstore.cnf columnstoreSupportReport/. > /dev/null 2>&1;" + "mv storagemanager.cnf columnstoreSupportReport/. > /dev/null 2>&1;" + "mv *Report.tar.gz columnstoreSupportReport/. > /dev/null 2>&1"); + string cmd = "tar -zcf columnstoreSupportReport." + systemName + ".tar.gz columnstoreSupportReport/*"; + system(cmd.c_str()); + + cout << endl + << "Columnstore Support Script Successfully completed, files located in columnstoreSupportReport." + + systemName + ".tar.gz" + << endl; +} diff --git a/oamapps/columnstoreSupport/configReport.sh b/oamapps/columnstoreSupport/configReport.sh new file mode 100755 index 000000000..cb642f137 --- /dev/null +++ b/oamapps/columnstoreSupport/configReport.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# +# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $ +# +if [ $1 ] ; then + MODULE=$1 +else + MODULE="pm1" +fi + +if [ $2 ] ; then + OUT_FILE=$2 +else + OUT_FILE=${MODULE}_logReport.txt +fi + +{ +echo " " +echo "******************** Configuration/Status Report for ${MODULE} ********************" +echo " " + +chkconfig=`which chkconfig 2>/dev/null` +if [ -n "$chkconfig" ]; then + echo "-- chkconfig configuration --" + echo " " + echo "################# chkconfig --list | grep columnstore #################" + echo " " + chkconfig --list | grep columnstore 2>/dev/null +fi + +systemctl=`which systemctl 2>/dev/null` +if [ -n "$systemctl" ]; then + echo "-- systemctl configuration --" + echo " " + echo "################# systemctl list-unit-files --type=service | grep columnstore #################" + echo " " + systemctl list-unit-files --type=service | grep columnstore 2>/dev/null + echo "################# systemctl list-unit-files --type=service | grep mariadb #################" + echo " " + systemctl list-unit-files --type=service | grep mariadb 2>/dev/null +fi + +updaterc=`which update-rc.d 2>/dev/null` +if [ -n "$updaterc" ]; then + echo "-- services configuration --" + echo " " + echo "################# service --status-all | grep columnstore #################" + echo " " + service --status-all | grep columnstore 2>/dev/null +fi + + +echo " " +echo "-- fstab Configuration --" +echo " " +echo "################# cat /etc/fstab #################" +echo " " +cat /etc/fstab 2>/dev/null + +echo " " +echo "-- Server Processes --" +echo " " +echo "################# ps axu #################" +echo " " +ps axu + +echo " " +echo "-- Server Processes with resource usage --" +echo " " +echo "################# top -b -n 1 #################" +echo " " +top -b -n 1 + +} >> $OUT_FILE + +exit 0 diff --git a/oamapps/columnstoreSupport/dbmsReport.sh b/oamapps/columnstoreSupport/dbmsReport.sh new file mode 100755 index 000000000..caac0cfd9 --- /dev/null +++ b/oamapps/columnstoreSupport/dbmsReport.sh @@ -0,0 +1,64 @@ +#! /bin/sh +# +# $Id: dbmsReport.sh +# +if [ $1 ] ; then + MODULE=$1 +else + MODULE="pm1" +fi + +if [ $2 ] ; then + OUT_FILE=$2 +else + OUT_FILE=${MODULE}_logReport.txt +fi + +if [ $3 ] ; then + MCSSUPPORTDIR=$3 +else + MCSSUPPORTDIR="/usr/share/columnstore" +fi + +if [ $4 ] ; then + PW_PROMPT=$4 +else + PW_PROMPT="" +fi + +{ + +columnstoreMysql="mysql -u root ${PW_PROMPT} " + +if ${columnstoreMysql} -V > /dev/null 2>&1; then + echo " " + echo "******************** DBMS Columnstore Version *********************************" + echo " " + ${columnstoreMysql} -e 'status' + echo " " + echo "******************** DBMS Columnstore System Column ***************************" + echo " " + ${columnstoreMysql} -e 'desc calpontsys.syscolumn;' + echo " " + echo "******************** DBMS Columnstore System Table ****************************" + echo " " + ${columnstoreMysql} -e 'desc calpontsys.systable;' + echo " " + echo "******************** DBMS Columnstore System Catalog Data *********************" + echo " " + ${columnstoreMysql} calpontsys < $MCSSUPPORTDIR/dumpcat_mysql.sql + echo " " + echo "******************** DBMS Columnstore System Table Data ***********************" + echo "******************** DBMS Columnstore Databases *******************************" + echo " " + ${columnstoreMysql} -e 'show databases;' + echo " " + echo "******************** DBMS Columnstore variables *******************************" + echo " " + ${columnstoreMysql} -e 'show variables;' + echo " " +fi +} >> $OUT_FILE + +exit 0 + diff --git a/oamapps/columnstoreSupport/getMinMax.sh b/oamapps/columnstoreSupport/getMinMax.sh new file mode 100755 index 000000000..0e4a0aa2f --- /dev/null +++ b/oamapps/columnstoreSupport/getMinMax.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# +# Reports the max value from the extent map for the given column. +# + +# +# Initialize variables. +# + +if [ -z "$MYSQLCMD" ]; then + MYSQLCMD="mysql -u root" +fi + +# +# Validate that there are three parameters - schema and table and columnname. +# +if [ $# -ne 3 ]; then + echo "" + echo "Reports the max value for the given column." + echo "" + echo "Parameters:" + echo " Schema" + echo " Table" + echo " Column" + exit 1 +fi +db=$1 +table=$2 +column=$3 + +# +# Validate that the column exists. +# +sql="select count(*) from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';" +count=`$MYSQLCMD calpontsys --skip-column-names -e "$sql;"` +if [ $count -le 0 ]; then + echo "" + echo "$db.$table.$column does not exist in Columnstore." + echo "" + exit 1 +fi + +# +# Validate that the column type is one that this script supports. +# Supported Types: +# 6 int +# 8 date +# 9 bigint +# 11 datetime +sql="select datatype from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';" +dataType=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"` +if [ $dataType -ne 6 ] && [ $dataType -ne 8 ] && [ $dataType -ne 9 ] && [ $dataType -ne 11 ]; then + echo "" + echo "The column data type must be an int, bigint, date, or datetime." + echo "" + exit 1 +fi + +# +# Grab the objectid for the column. +# +sql="select objectid from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';" +objectid=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"` + +# +# Set the editem specific parameter if the column is a date or datetime. +# +if [ $dataType -eq 8 ]; then + parm="-t" +elif [ $dataType -eq 11 ]; then + parm="-s" +fi + +# +# Use the editem utility to get the min and max value. +# +editem -o $objectid $parm | grep max | awk -v dataType=$dataType ' + BEGIN { + allValid=1; + foundValidExtent=0; + } + { + if(dataType == 11) { + state=substr($14, 1, length($14)-1); # Datetime has date and time as two fields. + thisMin=$6 " " substr($7, 1, length($7)-1); + thisMax=$9 " " substr($10, 1, length($10)-1); + } + else { + state=substr($12, 1, length($12)-1); + thisMin=substr($6, 1, length($6)-1); + thisMax=substr($8, 1, length($8)-1); + } + if(state == "valid") { + if(!foundValidExtent) { + min=thisMin; + max=thisMax; + foundValidExtent=1; + } + else { + if(thisMin < min) { + min=thisMin; + } + if(thisMax > max) { + max=thisMax; + } + } + } + else { + allValid=0; + } + } + END { + if(foundValidExtent == 1) { + print ""; + print "Min=" min; + print "Max=" max; + print ""; + if(allValid == 0) { + print "Not all extents had min and max values set. Answer is incomplete." + } + } + else { + print ""; + print "There were not any extents with valid min/max values. Unable to provide answer."; + print ""; + } + }' + +exit 0 diff --git a/oamapps/columnstoreSupport/hardwareReport.sh b/oamapps/columnstoreSupport/hardwareReport.sh new file mode 100755 index 000000000..d0bc92d5c --- /dev/null +++ b/oamapps/columnstoreSupport/hardwareReport.sh @@ -0,0 +1,79 @@ +#! /bin/sh +# +# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $ +# +if [ $1 ] ; then + MODULE=$1 +else + MODULE="pm1" +fi + +if [ $2 ] ; then + OUT_FILE=$2 +else + OUT_FILE=${MODULE}_logReport.txt +fi + +{ +echo " " +echo "******************** Hardware Report for ${MODULE} ********************" +echo " " + +echo "-- Server OS Version --" +echo " " +echo "################# cat /proc/version #################" +echo " " +cat /proc/version 2>/dev/null +echo " " +echo "################# uname -a #################" +echo " " +uname -a +echo " " +echo "################# cat /etc/issue #################" +echo " " +cat /etc/issue 2>/dev/null +echo " " +echo "run columnstore_os_check.sh" +echo " " +echo "################# /bin/columnstore_os_check.sh #################" +echo " " +columnstore_os_check.sh 2>/dev/null + +echo " " +echo "-- Server Uptime --" +echo " " +echo "################# uptime #################" +echo " " +uptime + +echo " " +echo "-- Server cpu-info --" +echo " " +echo "################# cat /proc/cpuinfo #################" +echo " " +cat /proc/cpuinfo 2>/dev/null + +echo " " +echo "-- Server memory-info --" +echo " " +echo "################# cat /proc/meminfo #################" +echo " " +$cat /proc/meminfo 2>/dev/null + +echo " " +echo "-- Server mounts --" +echo " " +echo "################# cat /proc/mounts #################" +echo " " +cat /proc/mounts 2>/dev/null + +echo " " +echo "-- Server Ethernet Configuration --" +echo " " +echo "################# ifconfig -a #################" +echo " " +ifconfig -a 2>/dev/null + +} >> $OUT_FILE + +exit 0 diff --git a/oamapps/columnstoreSupport/logReport.sh b/oamapps/columnstoreSupport/logReport.sh new file mode 100755 index 000000000..35b3e9a5f --- /dev/null +++ b/oamapps/columnstoreSupport/logReport.sh @@ -0,0 +1,52 @@ +#! /bin/sh +# +# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $ +# +if [ $1 ] ; then + MODULE=$1 +else + MODULE="pm1" +fi + +if [ $2 ] ; then + OUT_FILE=$2 +else + OUT_FILE=${MODULE}_logReport.txt +fi + +#get temp directory +tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir` + +rm -f ${tmpDir}/${MODULE}_logReport.tar.gz +tar -zcf ${tmpDir}/${MODULE}_logReport.tar.gz /var/log/mariadb/columnstore > /dev/null 2>&1 +cp ${tmpDir}/${MODULE}_logReport.tar.gz . +tar -zcf ${MODULE}_mysqllogReport.tar.gz /var/log/mysql/*.err 2>/dev/null + +echo '******************** Log Configuration ********************' >> $OUT_FILE +echo '' >> $OUT_FILE +echo 'MariaDB ColumnStore System Log Configuration Data' >> $OUT_FILE +echo '' >> $OUT_FILE +configFileName=`mcsGetConfig Installation SystemLogConfigFile` +echo 'System Logging Configuration File being used: '${configFileName} >> $OUT_FILE +echo '' >> $OUT_FILE +echo -e 'Module\tConfigured Log Levels' >> $OUT_FILE +echo -e '------\t---------------------------------------' >> $OUT_FILE +moduleConfig='' +if grep -q '/var/log/mariadb/columnstore/crit.log' ${configFileName}; then + moduleConfig=${moduleConfig}' CRITICAL' +fi +if grep -q '/var/log/mariadb/columnstore/err.log' ${configFileName}; then + moduleConfig=${moduleConfig}' ERROR' +fi +if grep -q '/var/log/mariadb/columnstore/warning.log' ${configFileName}; then + moduleConfig=${moduleConfig}' WARNING' +fi +if grep -q '/var/log/mariadb/columnstore/info.log' ${configFileName}; then + moduleConfig=${moduleConfig}' INFO' +fi +if grep -q '/var/log/mariadb/columnstore/debug.log' ${configFileName}; then + moduleConfig=${moduleConfig}' DEBUG' +fi +echo -e ${MODULE}'\t'${moduleConfig} >> $OUT_FILE +exit 0 + diff --git a/oamapps/columnstoreSupport/mcsSupportUtil.cpp b/oamapps/columnstoreSupport/mcsSupportUtil.cpp new file mode 100644 index 000000000..586a342e5 --- /dev/null +++ b/oamapps/columnstoreSupport/mcsSupportUtil.cpp @@ -0,0 +1,621 @@ +/* Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "mcsSupportUtil.h" + +using namespace std; +using namespace oam; +using namespace config; + +void getSystemNetworkConfig(FILE* pOutputFile) +{ + Oam oam; + // get and display Module Network Config + SystemModuleTypeConfig systemmoduletypeconfig; + systemmoduletypeconfig.moduletypeconfig.clear(); + + // get max length of a host name for header formatting + + int maxSize = 9; + + try + { + oam.getSystemConfig(systemmoduletypeconfig); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + { + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; + + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc; + + if (moduleCount > 0) + { + DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) + { + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for (; pt1 != (*pt).hostConfigList.end(); pt1++) + { + if (maxSize < (int)(*pt1).HostName.size()) + maxSize = (*pt1).HostName.size(); + } + } + } + } + } + catch (exception& e) + { + fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what()); + } + + fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "Module Name", "Module Description", "NIC ID", + "Host Name", "IP Address"); + fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "-----------", "-------------------------", "------", + "---------", "---------------"); + + try + { + oam.getSystemConfig(systemmoduletypeconfig); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + { + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; + + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc; + + if (moduleCount > 0) + { + DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) + { + string modulename = (*pt).DeviceName; + string moduleID = modulename.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE); + string modulenamedesc = moduletypedesc + " #" + moduleID; + + fprintf(pOutputFile, "%-15s%-30s", modulename.c_str(), modulenamedesc.c_str()); + + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for (; pt1 != (*pt).hostConfigList.end(); pt1++) + { + /* MCOL-1607. IPAddr may be a host name here b/c it is read straight + from the config file. */ + string tmphost = getIPAddress(pt1->IPAddr); + string ipAddr; + if (tmphost.empty()) + ipAddr = pt1->IPAddr; + else + ipAddr = tmphost; + string hostname = (*pt1).HostName; + string nicID = oam.itoa((*pt1).NicID); + + if (nicID != "1") + { + fprintf(pOutputFile, "%-45s", ""); + } + fprintf(pOutputFile, "%-13s%-14s%-20s\n", nicID.c_str(), hostname.c_str(), ipAddr.c_str()); + } + } + } + } + } + catch (exception& e) + { + fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what()); + } +} + +void getModuleTypeConfig(FILE* pOutputFile) +{ + Oam oam; + SystemModuleTypeConfig systemmoduletypeconfig; + ModuleTypeConfig moduletypeconfig; + ModuleConfig moduleconfig; + systemmoduletypeconfig.moduletypeconfig.clear(); + + try + { + oam.getSystemConfig(systemmoduletypeconfig); + + fprintf(pOutputFile, "Module Type Configuration\n\n"); + + for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + { + if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) + // end of list + break; + + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + + if (moduleCount < 1) + continue; + + string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; + + fprintf(pOutputFile, "ModuleType '%s' Configuration information\n", moduletype.c_str()); + fprintf(pOutputFile, "ModuleDesc = %s\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc.c_str()); + fprintf(pOutputFile, "ModuleCount = %i\n", moduleCount); + + if (moduleCount > 0) + { + DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) + { + string modulename = (*pt).DeviceName; + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for (; pt1 != (*pt).hostConfigList.end(); pt1++) + { + string ipAddr = (*pt1).IPAddr; + string servername = (*pt1).HostName; + fprintf(pOutputFile, "ModuleHostName and ModuleIPAddr for NIC ID %u on module '%s' = %s , %s\n", + (*pt1).NicID, modulename.c_str(), servername.c_str(), ipAddr.c_str()); + } + } + } + + DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin(); + + for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end(); pt++) + { + if ((*pt).dbrootConfigList.size() > 0) + { + fprintf(pOutputFile, "DBRootIDs assigned to module 'pm%u' = ", (*pt).DeviceID); + DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); + + for (; pt1 != (*pt).dbrootConfigList.end();) + { + fprintf(pOutputFile, "%u", *pt1); + pt1++; + + if (pt1 != (*pt).dbrootConfigList.end()) + fprintf(pOutputFile, ", "); + } + } + fprintf(pOutputFile, "\n"); + } + + fprintf(pOutputFile, "ModuleCPUCriticalThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUCriticalThreshold); + fprintf(pOutputFile, "ModuleCPUMajorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMajorThreshold); + fprintf(pOutputFile, "ModuleCPUMinorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorThreshold); + fprintf(pOutputFile, "ModuleCPUMinorClearThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorClearThreshold); + fprintf(pOutputFile, "ModuleDiskCriticalThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskCriticalThreshold); + fprintf(pOutputFile, "ModuleDiskMajorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMajorThreshold); + fprintf(pOutputFile, "ModuleDiskMinorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMinorThreshold); + fprintf(pOutputFile, "ModuleMemCriticalThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleMemCriticalThreshold); + fprintf(pOutputFile, "ModuleMemMajorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMajorThreshold); + fprintf(pOutputFile, "ModuleMemMinorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMinorThreshold); + fprintf(pOutputFile, "ModuleSwapCriticalThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapCriticalThreshold); + fprintf(pOutputFile, "ModuleSwapMajorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMajorThreshold); + fprintf(pOutputFile, "ModuleSwapMinorThreshold = %u%%\n", + systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMinorThreshold); + + DiskMonitorFileSystems::iterator pt2 = systemmoduletypeconfig.moduletypeconfig[i].FileSystems.begin(); + int id = 1; + + for (; pt2 != systemmoduletypeconfig.moduletypeconfig[i].FileSystems.end(); pt2++) + { + string fs = *pt2; + fprintf(pOutputFile, "ModuleDiskMonitorFileSystem#%i = %s\n", id, fs.c_str()); + ++id; + } + fprintf(pOutputFile, "\n"); + } + } + catch (exception& e) + { + cout << endl << "**** getModuleTypeConfig Failed = " << e.what() << endl; + } +} + +void getStorageConfig(FILE* pOutputFile) +{ + Oam oam; + try + { + systemStorageInfo_t t; + t = oam.getStorageConfig(); + + string cloud; + + try + { + oam.getSystemConfig("Cloud", cloud); + } + catch (...) + { + } + + string::size_type pos = cloud.find("amazon", 0); + + if (pos != string::npos) + cloud = "amazon"; + + fprintf(pOutputFile, "System Storage Configuration\n"); + + fprintf(pOutputFile, "Performance Module (DBRoot) Storage Type = %s\n", boost::get<0>(t).c_str()); + + if (cloud == "amazon") + fprintf(pOutputFile, "User Module Storage Type = %s\n", boost::get<3>(t).c_str()); + + fprintf(pOutputFile, "System Assigned DBRoot Count = %i\n", boost::get<1>(t)); + + DeviceDBRootList moduledbrootlist = boost::get<2>(t); + + typedef std::vector dbrootList; + dbrootList dbrootlist; + + DeviceDBRootList::iterator pt = moduledbrootlist.begin(); + + for (; pt != moduledbrootlist.end(); pt++) + { + fprintf(pOutputFile, "DBRoot IDs assigned to 'pm%u' = ", (*pt).DeviceID); + DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); + + for (; pt1 != (*pt).dbrootConfigList.end();) + { + fprintf(pOutputFile, "%u", *pt1); + dbrootlist.push_back(*pt1); + pt1++; + + if (pt1 != (*pt).dbrootConfigList.end()) + fprintf(pOutputFile, ", "); + } + + fprintf(pOutputFile, "\n"); + } + + // get any unassigned DBRoots + /*DBRootConfigList undbrootlist; + + try + { + oam.getUnassignedDbroot(undbrootlist); + } + catch (...) {} + + if ( !undbrootlist.empty() ) + { + fprintf(pOutputFile,"DBRoot IDs unassigned = "); + DBRootConfigList::iterator pt1 = undbrootlist.begin(); + + for ( ; pt1 != undbrootlist.end() ;) + { + fprintf(pOutputFile,"%u",*pt1); + pt1++; + + if (pt1 != undbrootlist.end()) + fprintf(pOutputFile,", "); + } + + fprintf(pOutputFile,"\n"); + }*/ + + fprintf(pOutputFile, "\n"); + + // um volumes + if (cloud == "amazon" && boost::get<3>(t) == "external") + { + ModuleTypeConfig moduletypeconfig; + oam.getSystemConfig("um", moduletypeconfig); + + for (int id = 1; id < moduletypeconfig.ModuleCount + 1; id++) + { + string volumeNameID = "UMVolumeName" + oam.itoa(id); + string volumeName = oam::UnassignedName; + string deviceNameID = "UMVolumeDeviceName" + oam.itoa(id); + string deviceName = oam::UnassignedName; + + try + { + oam.getSystemConfig(volumeNameID, volumeName); + oam.getSystemConfig(deviceNameID, deviceName); + } + catch (...) + { + } + + fprintf(pOutputFile, "Amazon EC2 Volume Name/Device Name for 'um%i': %s, %s", id, volumeName.c_str(), + deviceName.c_str()); + } + } + + // pm volumes + if (cloud == "amazon" && boost::get<0>(t) == "external") + { + fprintf(pOutputFile, "\n"); + + DBRootConfigList dbrootConfigList; + + try + { + oam.getSystemDbrootConfig(dbrootConfigList); + + DBRootConfigList::iterator pt = dbrootConfigList.begin(); + + for (; pt != dbrootConfigList.end(); pt++) + { + string volumeNameID = "PMVolumeName" + oam.itoa(*pt); + string volumeName = oam::UnassignedName; + string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt); + string deviceName = oam::UnassignedName; + + try + { + oam.getSystemConfig(volumeNameID, volumeName); + oam.getSystemConfig(deviceNameID, deviceName); + } + catch (...) + { + continue; + } + } + } + catch (exception& e) + { + cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; + } + + // print un-assigned dbroots + /*DBRootConfigList::iterator pt1 = undbrootlist.begin(); + + for ( ; pt1 != undbrootlist.end() ; pt1++) + { + string volumeNameID = "PMVolumeName" + oam.itoa(*pt1); + string volumeName = oam::UnassignedName; + string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt1); + string deviceName = oam::UnassignedName; + + try + { + oam.getSystemConfig( volumeNameID, volumeName); + oam.getSystemConfig( deviceNameID, deviceName); + } + catch (...) + { + continue; + } + }*/ + } + + string DataRedundancyConfig; + int DataRedundancyCopies; + string DataRedundancyStorageType; + + try + { + oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); + oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); + oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); + } + catch (...) + { + } + + if (DataRedundancyConfig == "y") + { + fprintf(pOutputFile, "\nData Redundant Configuration\n\n"); + fprintf(pOutputFile, "Copies Per DBroot = %i", DataRedundancyCopies); + + oamModuleInfo_t st; + string moduleType; + + try + { + st = oam.getModuleInfo(); + moduleType = boost::get<1>(st); + } + catch (...) + { + } + + if (moduleType != "pm") + return; + + try + { + DBRootConfigList dbrootConfigList; + oam.getSystemDbrootConfig(dbrootConfigList); + + DBRootConfigList::iterator pt = dbrootConfigList.begin(); + + for (; pt != dbrootConfigList.end(); pt++) + { + fprintf(pOutputFile, "DBRoot #%u has copies on PMs = ", *pt); + + string pmList = ""; + + try + { + string errmsg; + // oam.glusterctl(oam::GLUSTER_WHOHAS, oam.itoa(*pt), pmList, errmsg); + } + catch (...) + { + } + + boost::char_separator sep(" "); + boost::tokenizer > tokens(pmList, sep); + + for (boost::tokenizer >::iterator it = tokens.begin(); + it != tokens.end(); ++it) + { + fprintf(pOutputFile, "%s ", (*it).c_str()); + } + + fprintf(pOutputFile, "\n"); + } + + fprintf(pOutputFile, "\n"); + } + catch (exception& e) + { + cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; + } + } + } + catch (exception& e) + { + cout << endl << "**** getStorageConfig Failed : " << e.what() << endl; + } +} + +void getStorageStatus(FILE* pOutputFile) +{ + Oam oam; + + fprintf(pOutputFile, "System External DBRoot Storage Statuses\n\n"); + fprintf(pOutputFile, "Component Status Last Status Change\n"); + fprintf(pOutputFile, "------------ -------------------------- ------------------------\n"); + + /*try + { + oam.getSystemStatus(systemstatus, false); + + if ( systemstatus.systemdbrootstatus.dbrootstatus.size() == 0 ) + { + fprintf(pOutputFile," No External DBRoot Storage Configured\n\n"); + return; + } + + for ( unsigned int i = 0 ; i < systemstatus.systemdbrootstatus.dbrootstatus.size(); i++) + { + if ( systemstatus.systemdbrootstatus.dbrootstatus[i].Name.empty() ) + // end of list + break; + + int state = systemstatus.systemdbrootstatus.dbrootstatus[i].OpState; + string stime = systemstatus.systemdbrootstatus.dbrootstatus[i].StateChangeDate ; + stime = stime.substr (0, 24); + fprintf(pOutputFile,"DBRoot%s%-29s%-24s\n", + systemstatus.systemdbrootstatus.dbrootstatus[i].Name.c_str(), + oamState[state].c_str(), + stime.c_str()); + } + fprintf(pOutputFile,"\n"); + } + catch (exception& e) + { + cout << endl << "**** getSystemStatus Failed = " << e.what() << endl; + }*/ + + string DataRedundancyConfig; + int DataRedundancyCopies; + string DataRedundancyStorageType; + + try + { + oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); + oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); + oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); + } + catch (...) + { + } +} + +/******************************************************************** + * + * checkLogStatus - Check for a phrase in a log file and return status + * + ********************************************************************/ +bool checkLogStatus(std::string fileName, std::string phrase) +{ + ifstream file(fileName.c_str()); + + if (!file.is_open()) + { + return false; + } + + string buf; + + while (getline(file, buf)) + { + string::size_type pos = buf.find(phrase, 0); + + if (pos != string::npos) + // found phrase + return true; + } + + if (file.bad()) + { + return false; + } + + file.close(); + return false; +} + +/****************************************************************************************** + * @brief Get Network IP Address for Host Name + * + * purpose: Get Network IP Address for Host Name + * + ******************************************************************************************/ +string getIPAddress(string hostName) +{ + static uint32_t my_bind_addr; + struct hostent* ent; + string IPAddr = ""; + Oam oam; + + ent = gethostbyname(hostName.c_str()); + + if (ent != 0) + { + my_bind_addr = (uint32_t)((in_addr*)ent->h_addr_list[0])->s_addr; + + uint8_t split[4]; + uint32_t ip = my_bind_addr; + split[0] = (ip & 0xff000000) >> 24; + split[1] = (ip & 0x00ff0000) >> 16; + split[2] = (ip & 0x0000ff00) >> 8; + split[3] = (ip & 0x000000ff); + + IPAddr = + oam.itoa(split[3]) + "." + oam.itoa(split[2]) + "." + oam.itoa(split[1]) + "." + oam.itoa(split[0]); + } + + return IPAddr; +} diff --git a/oamapps/columnstoreSupport/mcsSupportUtil.h b/oamapps/columnstoreSupport/mcsSupportUtil.h new file mode 100644 index 000000000..dda881255 --- /dev/null +++ b/oamapps/columnstoreSupport/mcsSupportUtil.h @@ -0,0 +1,34 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include "configcpp.h" +#include "liboamcpp.h" + +void getSystemNetworkConfig(FILE* pOutputFile); +void getModuleTypeConfig(FILE* pOutputFile); +void getStorageConfig(FILE* pOutputFile); +void getStorageStatus(FILE* pOutputFile); +bool checkLogStatus(std::string filename, std::string phase); +std::string getIPAddress(std::string hostName); diff --git a/oamapps/columnstoreSupport/resourceReport.sh b/oamapps/columnstoreSupport/resourceReport.sh new file mode 100755 index 000000000..b0a092563 --- /dev/null +++ b/oamapps/columnstoreSupport/resourceReport.sh @@ -0,0 +1,66 @@ +#! /bin/sh +# +# $Id: resourceReport.sh 421 2007-04-05 15:46:55Z dhill $ +# +if [ $1 ] ; then + MODULE=$1 +else + MODULE="pm1" +fi + +if [ $2 ] ; then + OUT_FILE=$2 +else + OUT_FILE=${MODULE}_logReport.txt +fi + +{ +echo " " +echo "******************** Resource Usage Report for ${MODULE} ********************" +echo " " + +echo " " +echo "-- Shared Memory --" +echo " " +echo "################# ipcs -l #################" +echo " " +ipcs -l + +echo "################# clearShm -n #################" +echo " " +clearShm -n + +echo " " +echo "-- Disk Usage --" +echo " " +echo "################# df -k #################" +echo " " +df -k + +echo " " +echo "-- Disk BRM Data files --" +echo " " +ls -l /var/lib/columnstore/data1/systemFiles/dbrm 2> /dev/null +ls -l /var/lib/columnstore/dbrm 2> /dev/null + +echo "################# cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current #################" +echo " " +cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current 2> /dev/null + +echo " " +echo "-- View Table Locks --" +echo " " +echo "################# cat bin/viewtablelock #################" +echo " " +viewtablelock 2> /dev/null + +echo " " +echo "-- BRM Extent Map --" +echo " " +echo "################# bin/editem -i #################" +echo " " +editem -i 2>/dev/null + +} >> $OUT_FILE + +exit 0 diff --git a/oamapps/columnstoreSupport/softwareReport.sh b/oamapps/columnstoreSupport/softwareReport.sh new file mode 100755 index 000000000..24a93274c --- /dev/null +++ b/oamapps/columnstoreSupport/softwareReport.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $ +# +if [ $1 ] ; then + MODULE=$1 +else + MODULE="pm1" +fi + +if [ $2 ] ; then + OUT_FILE=$2 +else + OUT_FILE=${MODULE}_logReport.txt +fi + +{ +echo " " +echo "******************** Software Report for ${MODULE} ********************" +echo " " + +echo " " +echo "-- Columnstore Package Details --" +echo " " +rpm -qi MariaDB-columnstore-engine +echo " " + +} >> $OUT_FILE + +exit 0 diff --git a/oamapps/replayTransactionLog/CMakeLists.txt b/oamapps/replayTransactionLog/CMakeLists.txt new file mode 100644 index 000000000..678ebba7a --- /dev/null +++ b/oamapps/replayTransactionLog/CMakeLists.txt @@ -0,0 +1,34 @@ +# +# Not used +# + +# original Makefile.am contents follow: + +# Copyright (C) 2014 InfiniDB, Inc. +# +# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in +# +# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) +# bin_PROGRAMS = ReplayTransactionLog ReplayTransactionLog_SOURCES = replaytransactionlog.cpp +# ReplayTransactionLog_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) ReplayTransactionLog_LDFLAGS = +# @idb_common_ldflags@ @idb_exec_libs@ -lreplaytxnlog $(AM_LDFLAGS) +# +# test: +# +# coverage: +# +# leakcheck: +# +# docs: +# +# bootstrap: install-data-am +# diff --git a/oamapps/replayTransactionLog/replaytransactionlog.cpp b/oamapps/replayTransactionLog/replaytransactionlog.cpp new file mode 100644 index 000000000..0a662510b --- /dev/null +++ b/oamapps/replayTransactionLog/replaytransactionlog.cpp @@ -0,0 +1,156 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +// WWW - Add header comment. +#include +#include +#include +#include "liboamcpp.h" + +using namespace std; +using namespace oam; + +#include "replaytxnlog.h" +namespace +{ +void usage(char* prog) +{ + cout << endl; + cout << "Usage: " << prog << " [options]" << endl; + + cout << endl; + cout << "This utility can be used after a backup is restored to report transactions that " << endl; + cout << "occurred after the backup. It begins with the first transaction that was committed " << endl; + cout << "after the backup and reports DDL and DML statements as well as imports." << endl; + cout << endl; + + cout << "Options:" << endl; + /* + cout << "-u Database user id." << endl << endl; + + cout << "-p Password." << endl << endl; + */ + cout << "-d Stop date and time as mm/dd/yy@hh:mm:ss or 'Now'." << endl; + cout << " Only transactions committed before this date and time will be reported." << endl; + cout << " The current date and time will be used if 'Now'." << endl << endl; + + /* + cout << "-i Ignore bulk load log entries." << endl; + cout << " The program will pause and prompt at bulk load entries by default." << endl << + endl; + + cout << "-e Report mode. The sql statements will be displayed to the console only. No" << + endl; cout << " transactions will be processed. The user and password will be ignored." << + endl << endl; + */ + + cout << "-h Display this help." << endl << endl; +} + +bool isRunningOnPm() +{ + Oam oam; + oamModuleInfo_t t; + string moduleType; + int installType = -1; + + char* csc_ident = getenv("CALPONT_CSC_IDENT"); + + if (csc_ident == 0 || *csc_ident == 0) + { + // get local module info valdiate running on a pm + try + { + t = oam.getModuleInfo(); + moduleType = boost::get<1>(t); + installType = boost::get<5>(t); + } + catch (exception& e) + { + moduleType = "pm"; + } + } + else + moduleType = csc_ident; + + if (installType != oam::INSTALL_COMBINE_DM_UM_PM) + { + if (moduleType != "pm") + { + cerr << "Exiting, ReplayTransactionLog can only be run on a performance module (pm)" << endl; + return false; + } + } + + return true; +} +} // namespace + +int main(int argc, char** argv) +{ + string user; + string password; + string stopDate; + bool ignoreBulk = false; + bool reportMode = false; + char c; + + // Invokes member function `int operator ()(void);' + while ((c = getopt(argc, argv, "u:p:d:ihe")) != -1) + { + switch (c) + { + /* + case 'u': + user = optarg; + break; + case 'p': + password = optarg; + break; + */ + case 'd': stopDate = optarg; break; + + /* + case 'i': + ignoreBulk = true; + break; + case 'e': + reportMode = true; + break; + */ + case 'h': + usage(argv[0]); + return 0; + break; + + default: + usage(argv[0]); + return 1; + break; + } + } + + if (!isRunningOnPm()) + { + return 0; + } + + ReplayTxnLog replayTxnLog(user, password, stopDate, ignoreBulk, reportMode); + replayTxnLog.process(); + + return 0; +} diff --git a/oamapps/replayTransactionLog/tdriver.cpp b/oamapps/replayTransactionLog/tdriver.cpp new file mode 100644 index 000000000..809786d7d --- /dev/null +++ b/oamapps/replayTransactionLog/tdriver.cpp @@ -0,0 +1,66 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/*************************************************************************** + * wweeks@calpont.com * + * * + ***************************************************************************/ + +using namespace std; +#include +#include +#include +#include +#include "sessionmanager.h" + +#include + +using namespace execplan; + +int maxNewTxns = 1000; +int maxTxns = 1000; + +class ExecPlanTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(ExecPlanTest); + + CPPUNIT_TEST_SUITE_END(); + + private: + public: + void setUp() + { + } + + void tearDown() + { + } +}; // test suite + +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); + +#include +#include + +int main(int argc, char* argv[]) +{ + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); +} diff --git a/oamapps/sessionWalker/CMakeLists.txt b/oamapps/sessionWalker/CMakeLists.txt new file mode 100644 index 000000000..fe6fe4fea --- /dev/null +++ b/oamapps/sessionWalker/CMakeLists.txt @@ -0,0 +1,34 @@ +# +# Not used +# + +# original Makefile.am contents follow: + +# Copyright (C) 2014 InfiniDB, Inc. +# +# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in +# +# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) +# bin_PROGRAMS = sessionWalker sessionWalker_SOURCES = sessionwalker.cpp sessionWalker_CPPFLAGS = @idb_common_includes@ +# $(AM_CPPFLAGS) sessionWalker_LDFLAGS = @idb_common_ldflags@ @idb_common_libs@ @idb_write_libs@ @netsnmp_libs@ +# $(AM_LDFLAGS) +# +# test: +# +# coverage: +# +# leakcheck: +# +# docs: +# +# bootstrap: install-data-am +# diff --git a/oamapps/sessionWalker/sessionwalker.cpp b/oamapps/sessionWalker/sessionwalker.cpp new file mode 100644 index 000000000..06cf08bd0 --- /dev/null +++ b/oamapps/sessionWalker/sessionwalker.cpp @@ -0,0 +1,135 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/*************************************************************************** + * $Id: sessionwalker.cpp 3072 2013-04-04 19:04:45Z rdempsey $ + * + * jrodriguez@calpont.com + * * + ***************************************************************************/ + +#include +using namespace std; + +#include "sessionmonitor.h" +using namespace execplan; + +#include "vendordmlstatement.h" +#include "calpontdmlpackage.h" +#include "calpontdmlfactory.h" +using namespace dmlpackage; + +#include "bytestream.h" +#include "messagequeue.h" +using namespace messageqcpp; + +namespace +{ +void usage() +{ + cout << "sessionwalker [-d|-h]" << endl + << " -r rollback all transactions found" << endl + << " -h display this help" << endl; +} + +void rollback(const SessionMonitor::MonSIDTIDEntry& txn) +{ + VendorDMLStatement dmlStmt("ROLLBACK;", txn.sessionid); + CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); + + if (pDMLPackage == 0) + { + return; + } + + ByteStream bytestream; + pDMLPackage->write(bytestream); + delete pDMLPackage; + MessageQueueClient mq("DMLProc"); + + try + { + cout << "sending ROLLBACK for sessionID " << txn.sessionid << endl; + mq.write(bytestream); + bytestream = mq.read(); + } + catch (...) + { + } +} + +} // namespace + +int main(int argc, char** argv) +{ + bool rflg = false; + opterr = 0; + int c; + + while ((c = getopt(argc, argv, "rh")) != EOF) + switch (c) + { + case 'r': rflg = true; break; + + case 'h': + usage(); + return 0; + break; + + default: + usage(); + return 1; + break; + } + + vector toTxns; + SessionMonitor* monitor = new SessionMonitor(); + + toTxns.clear(); + toTxns = monitor->timedOutTxns(); // get timed out txns + + vector::iterator iter = toTxns.begin(); + vector::iterator end = toTxns.end(); + + vector tmp; + + while (iter != end) + { + if ((*iter)->sessionid > 0) + tmp.push_back(*iter); + + ++iter; + } + + toTxns.swap(tmp); + + cout << toTxns.size() << " timed out transactions." << endl; + + for (unsigned idx = 0; idx < toTxns.size(); idx++) + { + monitor->printTxns(*toTxns[idx]); + + if (rflg) + { + rollback(*toTxns[idx]); + } + } + + delete monitor; + + return 0; +} diff --git a/oamapps/sessionWalker/tdriver.cpp b/oamapps/sessionWalker/tdriver.cpp new file mode 100644 index 000000000..29e1771fd --- /dev/null +++ b/oamapps/sessionWalker/tdriver.cpp @@ -0,0 +1,189 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/*************************************************************************** + * jrodriguez@calpont.com * + * * + ***************************************************************************/ + +using namespace std; +#include +#include +#include +#include +#include "sessionmonitor.h" +#include "sessionmanager.h" + +#include + +using namespace execplan; + +int maxNewTxns = 1000; +int maxTxns = 1000; + +class ExecPlanTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(ExecPlanTest); + + CPPUNIT_TEST(MonitorTestPlan_1); + CPPUNIT_TEST_SUITE_END(); + + private: + public: + void setUp() + { + } + + void tearDown() + { + } + + int verifyLen; + SessionManager* manager; + SessionManager::TxnID managerTxns[1000]; + + int createTxns(const int& start, const int& end) + { + int first = start; + int last = end; + int newTxns = 0; + + verifyLen = manager->verifySize(); + + for (int idx = first; idx < last && verifyLen < maxNewTxns; idx++) + { + managerTxns[idx] = manager->newTxnID((uint32_t)idx + 1000); + CPPUNIT_ASSERT(managerTxns[idx].id > 0); + CPPUNIT_ASSERT(managerTxns[idx].valid == true); + verifyLen = manager->verifySize(); + CPPUNIT_ASSERT(verifyLen > 0); + newTxns++; + } + + CPPUNIT_ASSERT(newTxns == last - first); + return newTxns; + } + + int closeTxns(const int& start, const int& end) + { + int first = start; + int last = end; + int totalClosed = 0; + + for (int idx = first; idx < last; idx++) + { + try + { + SessionManager::TxnID tmp = manager->getTxnID(idx + 1000); + + if (tmp.valid == true) + { + manager->committed(tmp); + CPPUNIT_ASSERT(tmp.valid == false); + totalClosed++; + } + } + catch (exception& e) + { + cerr << e.what() << endl; + continue; + } + } + + return totalClosed; + + } // closeTxns + + void MonitorTestPlan_1() + { + int currStartTxn = 0; + int currEndTxn = 5; + int txnCntIncr = 5; + const int sleepTime = 1; + const int iterMax = 1; + vector toTxns; + + manager = new SessionManager(); + // CPPUNIT_ASSERT(manager->verifySize()==0); + + SessionMonitor* monitor = NULL; + + for (int jdx = 0; jdx < iterMax; jdx++) + { + // store the current state of the SessionManager + monitor = new SessionMonitor(); + monitor->AgeLimit(sleepTime); + delete monitor; + int idx = 0; + int grpStart = currStartTxn; + + for (idx = 0; idx < 3; idx++) + { + createTxns(currStartTxn, currEndTxn); + // CPPUNIT_ASSERT(manager->verifySize()==(idx+1)*txnCntIncr); + + currStartTxn += txnCntIncr; + currEndTxn += txnCntIncr; + sleep(sleepTime + 1); // make sessions time out + + monitor = new SessionMonitor(); // read Monitor data + monitor->AgeLimit(sleepTime); + toTxns.clear(); + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == (uint32_t)txnCntIncr * idx); + + delete monitor; + } + + int grpEnd = currEndTxn; + monitor = new SessionMonitor(); + monitor->AgeLimit(sleepTime); + closeTxns(grpStart, grpEnd); // close this iteration of txns + // CPPUNIT_ASSERT(manager->verifySize()==0); + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == 0); + + delete monitor; + } + + monitor = new SessionMonitor(); // readload Monitor data + monitor->AgeLimit(sleepTime - 1); + + toTxns.clear(); + toTxns = monitor->timedOutTxns(); // get timed out txns + CPPUNIT_ASSERT(toTxns.size() == 0); + delete monitor; + + // CPPUNIT_ASSERT(manager->verifySize()==0); + delete manager; + } + +}; // test suite + +CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); + +#include +#include + +int main(int argc, char* argv[]) +{ + CppUnit::TextUi::TestRunner runner; + CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); + runner.addTest(registry.makeTest()); + bool wasSuccessful = runner.run("", false); + return (wasSuccessful ? 0 : 1); +} diff --git a/primitives/blockcache/CMakeLists.txt b/primitives/blockcache/CMakeLists.txt index 60deb6c52..9ca0c950f 100644 --- a/primitives/blockcache/CMakeLists.txt +++ b/primitives/blockcache/CMakeLists.txt @@ -14,4 +14,4 @@ set(dbbc_STAT_SRCS fsutils.cpp ) columnstore_static_library(dbbc ${dbbc_STAT_SRCS}) -columnstore_link(dbbc loggingcpp) +columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/primitives/linux-port/CMakeLists.txt b/primitives/linux-port/CMakeLists.txt index 254a2d786..1a0032707 100644 --- a/primitives/linux-port/CMakeLists.txt +++ b/primitives/linux-port/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc) set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp) columnstore_static_library(processor ${processor_STAT_SRCS}) -columnstore_link(processor loggingcpp) +columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index b8f1df2fb..dbb82f54e 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -40,6 +40,7 @@ using namespace boost; #include "simd_sse.h" #include "simd_arm.h" #include "utils/common/columnwidth.h" +#include "utils/common/bit_cast.h" #include "exceptclasses.h" diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index 017f8bea3..cd1232812 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -3,26 +3,25 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../linux-port) # ########## next target ############### set(PrimProc_SRCS - activestatementcounter.cpp + primproc.cpp batchprimitiveprocessor.cpp bppseeder.cpp bppsendthread.cpp columncommand.cpp command.cpp dictstep.cpp - femsghandler.cpp filtercommand.cpp logger.cpp passthrucommand.cpp primitiveserver.cpp - primproc.cpp pseudocc.cpp - rssmonfcn.cpp rtscommand.cpp - samenodepseudosocket.cpp + umsocketselector.cpp serviceexemgr.cpp sqlfrontsessionthread.cpp - umsocketselector.cpp + rssmonfcn.cpp + activestatementcounter.cpp + femsghandler.cpp ../../utils/common/crashtrace.cpp ) @@ -32,11 +31,11 @@ target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS}) columnstore_link( PrimProc ${ENGINE_LDFLAGS} + ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} threadpool cacheutils dbbc processor loggingcpp - statistics_manager ) diff --git a/primitives/primproc/serviceexemgr.cpp b/primitives/primproc/serviceexemgr.cpp index 6e759cb78..2de371f21 100644 --- a/primitives/primproc/serviceexemgr.cpp +++ b/primitives/primproc/serviceexemgr.cpp @@ -78,7 +78,7 @@ #include "dbrm.h" #include "mariadb_my_sys.h" -#include "statistics_manager/statistics.h" +#include "statistics.h" #include "serviceexemgr.h" #include "sqlfrontsessionthread.h" diff --git a/primitives/primproc/serviceexemgr.h b/primitives/primproc/serviceexemgr.h index 3f1eb6545..d775a135c 100644 --- a/primitives/primproc/serviceexemgr.h +++ b/primitives/primproc/serviceexemgr.h @@ -58,7 +58,7 @@ #include "dbrm.h" #include "mariadb_my_sys.h" -#include "statistics_manager/statistics.h" +#include "statistics.h" namespace exemgr { @@ -69,7 +69,7 @@ class Opt int m_debug; bool m_e; bool m_fg; - Opt() : m_debug(0), m_e(false), m_fg(false) {}; + Opt() : m_debug(0), m_e(false), m_fg(false){}; Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) { int c; diff --git a/primitives/primproc/sqlfrontsessionthread.h b/primitives/primproc/sqlfrontsessionthread.h index 4e07ac3fb..92464a733 100644 --- a/primitives/primproc/sqlfrontsessionthread.h +++ b/primitives/primproc/sqlfrontsessionthread.h @@ -56,77 +56,76 @@ #include "dbrm.h" #include "mariadb_my_sys.h" -#include "statistics_manager/statistics.h" +#include "statistics.h" #include "serviceexemgr.h" namespace exemgr { -class SQLFrontSessionThread -{ - public: - SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, - joblist::ResourceManager* rm) - : fIos(ios) - , fEc(ec) - , fRm(rm) - , fStatsRetrieved(false) - , fTeleClient(globServiceExeMgr->getTeleServerParms()) - , fOamCachePtr(oam::OamCache::makeOamCache()) + class SQLFrontSessionThread { - } + public: + SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, + joblist::ResourceManager* rm) + : fIos(ios) + , fEc(ec) + , fRm(rm) + , fStatsRetrieved(false) + , fTeleClient(globServiceExeMgr->getTeleServerParms()) + , fOamCachePtr(oam::OamCache::makeOamCache()) + { + } - private: - messageqcpp::IOSocket fIos; - joblist::DistributedEngineComm* fEc; - joblist::ResourceManager* fRm; - querystats::QueryStats fStats; + private: + messageqcpp::IOSocket fIos; + joblist::DistributedEngineComm* fEc; + joblist::ResourceManager* fRm; + querystats::QueryStats fStats; - // Variables used to store return stats - bool fStatsRetrieved; + // Variables used to store return stats + bool fStatsRetrieved; - querytele::QueryTeleClient fTeleClient; + querytele::QueryTeleClient fTeleClient; - oam::OamCache* fOamCachePtr; // this ptr is copyable... + oam::OamCache* fOamCachePtr; // this ptr is copyable... - //...Reinitialize stats for start of a new query - void initStats(uint32_t sessionId, std::string& sqlText) - { - initMaxMemPct(sessionId); + //...Reinitialize stats for start of a new query + void initStats(uint32_t sessionId, std::string& sqlText) + { + initMaxMemPct(sessionId); - fStats.reset(); - fStats.setStartTime(); - fStats.fSessionID = sessionId; - fStats.fQuery = sqlText; - fStatsRetrieved = false; - } - //...Get % memory usage during latest query for sesssionId. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static uint64_t getMaxMemPct(uint32_t sessionId); - //...Delete sessionMemMap entry for the specified session's memory % use. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void deleteMaxMemPct(uint32_t sessionId); - //...Get and log query stats to specified output stream - const std::string formatQueryStats( - joblist::SJLP& jl, // joblist associated with query - const std::string& label, // header label to print in front of log output - bool includeNewLine, // include line breaks in query stats std::string - bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); - static void incThreadCntPerSession(uint32_t sessionId); - static void decThreadCntPerSession(uint32_t sessionId); - //...Init sessionMemMap entry for specified session to 0 memory %. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void initMaxMemPct(uint32_t sessionId); - //... Round off to human readable format (KB, MB, or GB). - const std::string roundBytes(uint64_t value) const; - void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); - void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, - boost::shared_ptr csc); - void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); - void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); - void analyzeTableHandleStats(messageqcpp::ByteStream& bs); - uint64_t roundMB(uint64_t value) const; - - public: - void operator()(); -}; -} // namespace exemgr + fStats.reset(); + fStats.setStartTime(); + fStats.fSessionID = sessionId; + fStats.fQuery = sqlText; + fStatsRetrieved = false; + } + //...Get % memory usage during latest query for sesssionId. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static uint64_t getMaxMemPct(uint32_t sessionId); + //...Delete sessionMemMap entry for the specified session's memory % use. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void deleteMaxMemPct(uint32_t sessionId); + //...Get and log query stats to specified output stream + const std::string formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); + static void incThreadCntPerSession(uint32_t sessionId); + static void decThreadCntPerSession(uint32_t sessionId); + //...Init sessionMemMap entry for specified session to 0 memory %. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void initMaxMemPct(uint32_t sessionId); + //... Round off to human readable format (KB, MB, or GB). + const std::string roundBytes(uint64_t value) const; + void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); + void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc); + void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); + void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); + void analyzeTableHandleStats(messageqcpp::ByteStream& bs); + uint64_t roundMB(uint64_t value) const; + public: + void operator()(); + }; +} diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 1b01b403f..c72171cbd 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -64,7 +64,7 @@ configure_file( link_directories(${CMAKE_BINARY_DIR}/lib) set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) -columnstore_library(storagemanager ${storagemanager_SRCS}) +columnstore_library(storagemanager SHARED ${storagemanager_SRCS}) add_dependencies(storagemanager marias3 external_boost) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) diff --git a/tools/cleartablelock/CMakeLists.txt b/tools/cleartablelock/CMakeLists.txt index 01dcc211e..30dc54edd 100644 --- a/tools/cleartablelock/CMakeLists.txt +++ b/tools/cleartablelock/CMakeLists.txt @@ -6,4 +6,4 @@ set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp) columnstore_executable(cleartablelock ${cleartablelock_SRCS}) -columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS}) +columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) diff --git a/tools/configMgt/CMakeLists.txt b/tools/configMgt/CMakeLists.txt index 15d6d7693..9e6b3f3e9 100644 --- a/tools/configMgt/CMakeLists.txt +++ b/tools/configMgt/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(autoConfigure_SRCS autoConfigure.cpp) add_executable(autoConfigure ${autoConfigure_SRCS}) -columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) diff --git a/tools/dbbuilder/CMakeLists.txt b/tools/dbbuilder/CMakeLists.txt index d6bceb767..9d687f912 100644 --- a/tools/dbbuilder/CMakeLists.txt +++ b/tools/dbbuilder/CMakeLists.txt @@ -6,4 +6,4 @@ set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp) columnstore_executable(dbbuilder ${dbbuilder_SRCS}) -columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS}) +columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) diff --git a/tools/ddlcleanup/CMakeLists.txt b/tools/ddlcleanup/CMakeLists.txt index 4e6016c04..9ba55097e 100644 --- a/tools/ddlcleanup/CMakeLists.txt +++ b/tools/ddlcleanup/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(ddlcleanup_SRCS ddlcleanup.cpp) columnstore_executable(ddlcleanup ${ddlcleanup_SRCS}) -columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ddlcleanuputil) +columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil) diff --git a/tools/editem/CMakeLists.txt b/tools/editem/CMakeLists.txt index ba6dde613..b0dc993c6 100644 --- a/tools/editem/CMakeLists.txt +++ b/tools/editem/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(editem_SRCS editem.cpp) columnstore_executable(editem ${editem_SRCS}) -columnstore_link(editem ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) diff --git a/tools/rgprint/CMakeLists.txt b/tools/rgprint/CMakeLists.txt new file mode 100644 index 000000000..788079df9 --- /dev/null +++ b/tools/rgprint/CMakeLists.txt @@ -0,0 +1,9 @@ +include_directories(${ENGINE_COMMON_INCLUDES}) + +# ########## next target ############### + +set(rgprint_SRCS rgprint.cpp) + +columnstore_executable(rgprint ${rgprint_SRCS}) + +columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) diff --git a/tools/setConfig/CMakeLists.txt b/tools/setConfig/CMakeLists.txt index 85c16b0a8..b43566721 100644 --- a/tools/setConfig/CMakeLists.txt +++ b/tools/setConfig/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(setConfig_SRCS main.cpp) columnstore_executable(mcsSetConfig ${setConfig_SRCS}) -columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index cae710dea..9e7b74f4c 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,28 +1,26 @@ # add_subdirectory(boost_idb) -add_subdirectory(batchloader) -add_subdirectory(cacheutils) -add_subdirectory(cloudio) -add_subdirectory(common) -add_subdirectory(compress) -add_subdirectory(configcpp) -add_subdirectory(dataconvert) -add_subdirectory(ddlcleanup) -add_subdirectory(funcexp) -add_subdirectory(idbdatafile) -add_subdirectory(joiner) -add_subdirectory(libmarias3) -add_subdirectory(libmysql_client) -add_subdirectory(loggingcpp) -add_subdirectory(mariadb_charset) -add_subdirectory(messageqcpp) -add_subdirectory(pron) -add_subdirectory(querystats) -add_subdirectory(querytele) -add_subdirectory(regr) -add_subdirectory(rowgroup) -add_subdirectory(rwlock) add_subdirectory(startup) -add_subdirectory(statistics_manager) +add_subdirectory(common) +add_subdirectory(configcpp) +add_subdirectory(loggingcpp) +add_subdirectory(messageqcpp) add_subdirectory(threadpool) +add_subdirectory(rwlock) +add_subdirectory(dataconvert) +add_subdirectory(joiner) +add_subdirectory(rowgroup) +add_subdirectory(cacheutils) +add_subdirectory(funcexp) add_subdirectory(udfsdk) +add_subdirectory(compress) +add_subdirectory(batchloader) +add_subdirectory(ddlcleanup) +add_subdirectory(querystats) add_subdirectory(windowfunction) +add_subdirectory(idbdatafile) +add_subdirectory(querytele) +add_subdirectory(libmysql_client) +add_subdirectory(regr) +add_subdirectory(cloudio) +add_subdirectory(libmarias3) +add_subdirectory(pron) diff --git a/utils/batchloader/CMakeLists.txt b/utils/batchloader/CMakeLists.txt index 694e88ef0..25f58197e 100644 --- a/utils/batchloader/CMakeLists.txt +++ b/utils/batchloader/CMakeLists.txt @@ -6,4 +6,4 @@ set(batchloader_LIB_SRCS batchloader.cpp) columnstore_library(batchloader ${batchloader_LIB_SRCS}) -columnstore_link(batchloader oamcpp loggingcpp) +columnstore_link(batchloader ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/utils/cacheutils/CMakeLists.txt b/utils/cacheutils/CMakeLists.txt index 666834835..0c9c5d63c 100644 --- a/utils/cacheutils/CMakeLists.txt +++ b/utils/cacheutils/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(cacheutils_LIB_SRCS cacheutils.cpp) columnstore_library(cacheutils ${cacheutils_LIB_SRCS}) -columnstore_link(cacheutils messageqcpp boost_thread) +columnstore_link(cacheutils messageqcpp) diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt index 131030ace..c5bb954d0 100755 --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -14,7 +14,7 @@ columnstore_library(cloudio ${cloudio_LIB_SRCS}) # IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for # idbdatafile, we should be able to reverse the dependency like so: -columnstore_link(cloudio idbdatafile messageqcpp boost_thread) +columnstore_link(cloudio idbdatafile messageqcpp) add_executable(cloudio_component_test component_test.cpp) add_executable(cloudio_end_to_end_test end_to_end_test.cpp) diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index d99ee7590..dc19cd77d 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -9,17 +9,10 @@ set(common_LIB_SRCS MonitorProcMem.cpp nullvaluemanip.cpp threadnaming.cpp + utils_utf8.cpp + statistics.cpp string_prefixes.cpp ) columnstore_library(common ${common_LIB_SRCS}) -columnstore_link( - common - PRIVATE - boost_filesystem - configcpp - loggingcpp - messageqcpp - idbdatafile - mariadb_charset -) +columnstore_link(common boost_filesystem configcpp loggingcpp messageqcpp) diff --git a/utils/common/bit_cast.h b/utils/common/bit_cast.h new file mode 100644 index 000000000..e2a99e21a --- /dev/null +++ b/utils/common/bit_cast.h @@ -0,0 +1,39 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +#include +#include + +namespace utils +{ +template +std::enable_if_t< + sizeof(To) == sizeof(From) && std::is_trivially_copyable_v && std::is_trivially_copyable_v, To> +// constexpr support needs compiler magic +bitCast(const From& src) noexcept +{ + static_assert(std::is_trivially_constructible_v, + "This implementation additionally requires " + "destination type to be trivially constructible"); + + To dst; + std::memcpy(&dst, &src, sizeof(To)); + return dst; +} +} // namespace utils diff --git a/utils/common/branchpred.h b/utils/common/branchpred.h index 2553c1390..b0c4feb08 100644 --- a/utils/common/branchpred.h +++ b/utils/common/branchpred.h @@ -19,6 +19,12 @@ #pragma once +#if !defined(__GNUC__) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) +#ifndef __builtin_expect +#define __builtin_expect(x, expected_value) (x) +#endif +#endif + #ifndef LIKELY #define LIKELY(x) __builtin_expect((x), 1) #define UNLIKELY(x) __builtin_expect((x), 0) diff --git a/utils/mariadb_charset/collation.h b/utils/common/collation.h similarity index 99% rename from utils/mariadb_charset/collation.h rename to utils/common/collation.h index 411eca6aa..e5781b8d9 100644 --- a/utils/mariadb_charset/collation.h +++ b/utils/common/collation.h @@ -35,7 +35,7 @@ #include "mcsconfig.h" #include "exceptclasses.h" -#include "basic/conststring.h" +#include "conststring.h" /* Redefine definitions used by MariaDB m_ctype.h. diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 1a9ca6635..0c060ae89 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -17,9 +17,6 @@ #pragma once -#include -#include - #include "branchpred.h" namespace utils diff --git a/utils/basic/conststring.h b/utils/common/conststring.h similarity index 100% rename from utils/basic/conststring.h rename to utils/common/conststring.h diff --git a/utils/common/genericparser.h b/utils/common/genericparser.h index 8f4c67101..d2bce2d26 100644 --- a/utils/common/genericparser.h +++ b/utils/common/genericparser.h @@ -17,7 +17,7 @@ #pragma once -#include "basic/conststring.h" +#include "conststring.h" namespace genericparser { diff --git a/utils/common/hashfamily.h b/utils/common/hashfamily.h index 954779e40..4b09e01a8 100644 --- a/utils/common/hashfamily.h +++ b/utils/common/hashfamily.h @@ -18,7 +18,7 @@ #pragma once #include "hasher.h" -#include "mariadb_charset/collation.h" +#include "collation.h" namespace utils { diff --git a/utils/common/nullstring.h b/utils/common/nullstring.h index 0b4bd62a9..313a5bcb9 100644 --- a/utils/common/nullstring.h +++ b/utils/common/nullstring.h @@ -25,7 +25,7 @@ #include #include #include "exceptclasses.h" -#include "basic/conststring.h" +#include "conststring.h" #include "mcs_datatype_basic.h" namespace utils diff --git a/utils/statistics_manager/statistics.cpp b/utils/common/statistics.cpp similarity index 99% rename from utils/statistics_manager/statistics.cpp rename to utils/common/statistics.cpp index b32007855..b5a2b07ac 100644 --- a/utils/statistics_manager/statistics.cpp +++ b/utils/common/statistics.cpp @@ -116,8 +116,9 @@ void StatisticsManager::analyzeSample(bool traceOn) // MCV statistics. std::vector> mcvList(columnMCV.begin(), columnMCV.end()); std::sort(mcvList.begin(), mcvList.end(), - [](const std::pair& a, const std::pair& b) - { return a.second > b.second; }); + [](const std::pair& a, const std::pair& b) { + return a.second > b.second; + }); // 200 buckets as Microsoft does. const auto mcvSize = std::min(columnMCV.size(), static_cast(200)); diff --git a/utils/statistics_manager/statistics.h b/utils/common/statistics.h similarity index 100% rename from utils/statistics_manager/statistics.h rename to utils/common/statistics.h diff --git a/utils/common/string_prefixes.cpp b/utils/common/string_prefixes.cpp index a8c45c84e..9f9c5c2e7 100644 --- a/utils/common/string_prefixes.cpp +++ b/utils/common/string_prefixes.cpp @@ -18,7 +18,7 @@ /* handling of the conversion of string prefixes to int64_t for quick range checking */ -#include "mariadb_charset/collation.h" +#include "collation.h" #include "joblisttypes.h" #include "string_prefixes.h" diff --git a/utils/common/syncstream.h b/utils/common/syncstream.h new file mode 100644 index 000000000..d00b41150 --- /dev/null +++ b/utils/common/syncstream.h @@ -0,0 +1,157 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/** @file */ + +/* + * classes isyncstream and osyncstream provide a C++ iostream interface + * for C stdio FILE* streams. The current implementation does not provide + * the necessary methods to support seeking. The I/O buffering of the + * input FILE* is used. The C++ iostream library calls syncbuf::sync() + * for every line, so output buffering is line-by-line. + * */ + +/* +#include "syncstream.h" + +void copyStream(istream& iss, ostream& oss) +{ + string line; + getline(iss, line); + while (iss.good()) + { + oss << line << endl; + getline(iss, line); + } +} + +main() +{ + FILE* ifp; + FILE* ofp; + + ... + + isyncstream iss(ifp); + osyncstream oss(ofp); + + copyStream(iss, oss); + + ... +} +*/ + +#pragma once + +#include +#include + +namespace syncstream +{ +/** A streambuf implementation for C stdio FILE* streams. + * + * Adapted from http://www.drdobbs.com/184401305 + */ +class syncbuf : public std::streambuf +{ + public: + /** ctor */ + syncbuf(FILE* f) : std::streambuf(), fptr(f) + { + } + + protected: + /** Write character in the case of overflow */ + virtual int overflow(int c = EOF) + { + return (c != EOF ? fputc(c, fptr) : EOF); + } + /** Get character in the case of overflow */ + virtual int underflow() + { + int c = getc(fptr); + + if (c != EOF) + ungetc(c, fptr); + + return c; + } + /** Get character in the case of overflow and advance get pointer */ + virtual int uflow() + { + return getc(fptr); + } + /** put character back in the case of backup underflow */ + virtual int pbackfail(int c = EOF) + { + return (c != EOF ? ungetc(c, fptr) : EOF); + } + /** Synchronize stream buffer */ + virtual int sync() + { + return fflush(fptr); + } + + private: + FILE* fptr; +}; + +/** An istream adaptor for input FILE* streams */ +class isyncstream : public std::istream +{ + public: + /** ctor */ + isyncstream() : istream(&buf), buf(0) + { + } + /** ctor */ + isyncstream(FILE* fptr) : istream(&buf), buf(fptr) + { + } + /** const streambuf accessor */ + const syncbuf* rdbuf() const + { + return &buf; + } + + private: + syncbuf buf; +}; + +/** An ostream adaptor for output FILE* streams */ +class osyncstream : public std::ostream +{ + public: + /** ctor */ + osyncstream() : ostream(&buf), buf(0) + { + } + /** ctor */ + osyncstream(FILE* fptr) : ostream(&buf), buf(fptr) + { + } + /** const streambuf accessor */ + const syncbuf* rdbuf() const + { + return &buf; + } + + private: + syncbuf buf; +}; + +} // namespace syncstream diff --git a/utils/mariadb_charset/charset.cpp b/utils/common/utils_utf8.cpp similarity index 72% rename from utils/mariadb_charset/charset.cpp rename to utils/common/utils_utf8.cpp index 9a101592b..f0471f7d3 100644 --- a/utils/mariadb_charset/charset.cpp +++ b/utils/common/utils_utf8.cpp @@ -1,9 +1,8 @@ -/* Copyright (C) 2025 MariaDB Corporation +/* Copyright (C) 2020 MariaDB Corporation. - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -12,12 +11,11 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "utils_utf8.h" -#include "mariadb_charset/collation.h" #include "mariadb_my_sys.h" -#include "mcs_datatype.h" namespace datatypes { @@ -37,3 +35,4 @@ void Charset::setCharset(uint32_t charsetNumber) } } // namespace datatypes + diff --git a/utils/common/utils_utf8.h b/utils/common/utils_utf8.h index 49f856bb5..352bbb4f7 100644 --- a/utils/common/utils_utf8.h +++ b/utils/common/utils_utf8.h @@ -20,10 +20,16 @@ #pragma once -#include #include +#if defined(__FreeBSD__) +//#include +#else +#include +#endif #include + #include +#include "liboamcpp.h" // Change the name from utf8. Even change the file name to something resembling char helper namespace utf8 diff --git a/utils/ddlcleanup/CMakeLists.txt b/utils/ddlcleanup/CMakeLists.txt index 600e73e84..c813fe428 100644 --- a/utils/ddlcleanup/CMakeLists.txt +++ b/utils/ddlcleanup/CMakeLists.txt @@ -2,4 +2,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(ddlcleanuputil_LIB_SRCS ddlcleanuputil.cpp) columnstore_library(ddlcleanuputil ${ddlcleanuputil_LIB_SRCS}) -columnstore_link(ddlcleanuputil PRIVATE loggingcpp) +columnstore_link(ddlcleanuputil PRIVATE loggingcpp ${NETSNMP_LIBRARIES}) diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index 752221fed..6f1c758a7 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -150,7 +150,6 @@ columnstore_link( pron loggingcpp dataconvert - mariadb_charset + ${MARIADB_STRING_LIBS} + ${NETSNMP_LIBRARIES} ) - -columnstore_link(funcexp PRIVATE ${MARIADB_STRING_LIBS}) diff --git a/utils/funcexp/functor_json.h b/utils/funcexp/functor_json.h index f6939a4c5..55c4d1b5f 100644 --- a/utils/funcexp/functor_json.h +++ b/utils/funcexp/functor_json.h @@ -7,7 +7,7 @@ #include #include -#include "mariadb_charset/collation.h" +#include "collation.h" #include "functor_bool.h" #include "functor_int.h" #include "functor_str.h" diff --git a/utils/funcexp/jsonhelpers.h b/utils/funcexp/jsonhelpers.h index 4f77c05a1..7047677c9 100644 --- a/utils/funcexp/jsonhelpers.h +++ b/utils/funcexp/jsonhelpers.h @@ -10,10 +10,10 @@ #include // #include -#include "mariadb_charset/collation.h" +#include "collation.h" #include "functor_json.h" #include "functor_str.h" -#include "mariadb_charset/collation.h" +#include "collation.h" #include "rowgroup.h" #include "treenode.h" #include "functioncolumn.h" diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index 8e8d9ab4d..06a1ebf44 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -14,4 +14,4 @@ set(idbdatafile_LIB_SRCS ) columnstore_library(idbdatafile ${idbdatafile_LIB_SRCS}) -columnstore_link(idbdatafile PRIVATE ${ENGINE_OAM_LIBS} boost_filesystem boost_system compress) +columnstore_link(idbdatafile PRIVATE ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} boost_filesystem boost_system) diff --git a/utils/joiner/CMakeLists.txt b/utils/joiner/CMakeLists.txt index 1d636162c..0df5c60f7 100644 --- a/utils/joiner/CMakeLists.txt +++ b/utils/joiner/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(joiner_LIB_SRCS tuplejoiner.cpp joinpartition.cpp) columnstore_library(joiner ${joiner_LIB_SRCS}) -columnstore_link(joiner PRIVATE loggingcpp rowgroup datatypes compress) +columnstore_link(joiner PRIVATE loggingcpp) diff --git a/utils/mariadb_charset/CMakeLists.txt b/utils/mariadb_charset/CMakeLists.txt deleted file mode 100644 index 9734bf1c5..000000000 --- a/utils/mariadb_charset/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -include_directories(${ENGINE_COMMON_INCLUDES}) - -# ########## next target ############### - -columnstore_static_library(mariadb_charset charset.cpp) -columnstore_link(mariadb_charset PRIVATE ${MARIADB_STRING_LIBS}) -add_dependencies(mariadb_charset loggingcpp) diff --git a/utils/messageqcpp/CMakeLists.txt b/utils/messageqcpp/CMakeLists.txt index c41a88168..527e9395a 100644 --- a/utils/messageqcpp/CMakeLists.txt +++ b/utils/messageqcpp/CMakeLists.txt @@ -8,6 +8,7 @@ set(messageqcpp_LIB_SRCS iosocket.cpp messagequeue.cpp messagequeuepool.cpp + samenodepseudosocket.cpp socketparms.cpp ) diff --git a/utils/messageqcpp/messagequeuepool.cpp b/utils/messageqcpp/messagequeuepool.cpp index 2debf46cd..25af41a5c 100644 --- a/utils/messageqcpp/messagequeuepool.cpp +++ b/utils/messageqcpp/messagequeuepool.cpp @@ -25,58 +25,41 @@ #include #include + namespace messageqcpp { using ClientMapType = std::multimap>; -template -struct Immortal +struct LockedClientMap { - template - Immortal(Args&&... args) - { - ::new (space) T(std::forward(args)...); - } - - operator T&() & noexcept - { - return reinterpret_cast(space); - } - - private: - alignas(T) unsigned char space[sizeof(T)]; -}; - -class LockedClientMap -{ - struct KeyToUsePrivateCtor - { - explicit KeyToUsePrivateCtor() = default; - }; - - public: - LockedClientMap(const LockedClientMap&) = delete; - LockedClientMap& operator=(const LockedClientMap&) = delete; - ~LockedClientMap() = delete; - - static LockedClientMap& getInstance() - { - static Immortal instance(KeyToUsePrivateCtor{}); - return instance; - } - - LockedClientMap(KeyToUsePrivateCtor) + LockedClientMap() + { + } + ~LockedClientMap() { } - - private: ClientMapType clientMap; std::mutex queueMutex; - - friend class MessageQueueClientPool; }; +static int clientMapNiftyCounter; + +static typename std::aligned_storage::type clientMapBuf; + +auto& lockedMap = reinterpret_cast(clientMapBuf); + + +LockedClientMapInitilizer::LockedClientMapInitilizer () +{ + if (clientMapNiftyCounter++ == 0) new (&lockedMap) LockedClientMap (); // placement new +} +LockedClientMapInitilizer::~LockedClientMapInitilizer () +{ + if (--clientMapNiftyCounter == 0) (&lockedMap)->~LockedClientMap(); +} + + // 300 seconds idle until cleanup #define MAX_IDLE_TIME 300 @@ -87,7 +70,7 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts) MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrIp, uint64_t port) { - auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex); + auto lock = std::scoped_lock(lockedMap.queueMutex); std::ostringstream oss; oss << dnOrIp << "_" << port; @@ -110,13 +93,14 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrI newClientObject->client.reset(new MessageQueueClient(dnOrIp, port)); newClientObject->inUse = true; newClientObject->lastUsed = nowSeconds; - LockedClientMap::getInstance().clientMap.emplace(std::move(searchString), std::move(newClientObject)); + lockedMap.clientMap.emplace(std::move(searchString), std::move(newClientObject)); return newClientObject->client.get(); } MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& module) { - auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex); + auto lock = std::scoped_lock(lockedMap.queueMutex); + MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module); @@ -132,11 +116,13 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& modul clock_gettime(CLOCK_MONOTONIC, &now); uint64_t nowSeconds = TimeSpecToSeconds(&now); + + newClientObject->client.reset(new MessageQueueClient(module)); newClientObject->inUse = true; newClientObject->lastUsed = nowSeconds; auto result = newClientObject->client.get(); - LockedClientMap::getInstance().clientMap.emplace(std::move(module), std::move(newClientObject)); + lockedMap.clientMap.emplace(std::move(module), std::move(newClientObject)); return result; } @@ -147,10 +133,11 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search uint64_t nowSeconds = TimeSpecToSeconds(&now); MessageQueueClient* returnClient = NULL; - auto it = LockedClientMap::getInstance().clientMap.begin(); + auto it = lockedMap.clientMap.begin(); + // Scan pool - while (it != LockedClientMap::getInstance().clientMap.end()) + while (it != lockedMap.clientMap.end()) { ClientObject* clientObject = it->second.get(); uint64_t elapsedTime = nowSeconds - clientObject->lastUsed; @@ -162,7 +149,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search // Do this so we don't invalidate current interator auto toDelete = it; it++; - LockedClientMap::getInstance().clientMap.erase(toDelete); + lockedMap.clientMap.erase(toDelete); continue; } @@ -176,7 +163,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search // Do this so we don't invalidate current interator auto toDelete = it; it++; - LockedClientMap::getInstance().clientMap.erase(toDelete); + lockedMap.clientMap.erase(toDelete); continue; } } @@ -206,10 +193,10 @@ void MessageQueueClientPool::releaseInstance(MessageQueueClient* client) if (client == NULL) return; - auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex); - auto it = LockedClientMap::getInstance().clientMap.begin(); + auto lock = std::scoped_lock(lockedMap.queueMutex); + auto it = lockedMap.clientMap.begin(); - while (it != LockedClientMap::getInstance().clientMap.end()) + while (it != lockedMap.clientMap.end()) { if (it->second->client.get() == client) { @@ -234,14 +221,15 @@ void MessageQueueClientPool::deleteInstance(MessageQueueClient* client) if (client == NULL) return; - auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex); - auto it = LockedClientMap::getInstance().clientMap.begin(); - while (it != LockedClientMap::getInstance().clientMap.end()) + auto lock = std::scoped_lock(lockedMap.queueMutex); + auto it = lockedMap.clientMap.begin(); + + while (it != lockedMap.clientMap.end()) { if (it->second->client.get() == client) { - LockedClientMap::getInstance().clientMap.erase(it); + lockedMap.clientMap.erase(it); return; } diff --git a/utils/messageqcpp/messagequeuepool.h b/utils/messageqcpp/messagequeuepool.h index 6af5e1cc6..17899b76a 100644 --- a/utils/messageqcpp/messagequeuepool.h +++ b/utils/messageqcpp/messagequeuepool.h @@ -26,6 +26,12 @@ namespace messageqcpp { + +static struct LockedClientMapInitilizer { + LockedClientMapInitilizer (); + ~LockedClientMapInitilizer (); +} clientMapInitilizer; // static initializer for every translation unit + struct ClientObject { std::unique_ptr client; @@ -43,8 +49,8 @@ class MessageQueueClientPool static MessageQueueClient* findInPool(const std::string& search); private: - MessageQueueClientPool() {}; - ~MessageQueueClientPool() {}; + MessageQueueClientPool(){}; + ~MessageQueueClientPool(){}; }; } // namespace messageqcpp diff --git a/primitives/primproc/samenodepseudosocket.cpp b/utils/messageqcpp/samenodepseudosocket.cpp similarity index 100% rename from primitives/primproc/samenodepseudosocket.cpp rename to utils/messageqcpp/samenodepseudosocket.cpp diff --git a/primitives/primproc/samenodepseudosocket.h b/utils/messageqcpp/samenodepseudosocket.h similarity index 100% rename from primitives/primproc/samenodepseudosocket.h rename to utils/messageqcpp/samenodepseudosocket.h diff --git a/utils/querystats/CMakeLists.txt b/utils/querystats/CMakeLists.txt index ff6b13f91..0d91ead9e 100644 --- a/utils/querystats/CMakeLists.txt +++ b/utils/querystats/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(querystats_LIB_SRCS querystats.cpp) columnstore_library(querystats ${querystats_LIB_SRCS}) -columnstore_link(querystats PRIVATE loggingcpp messageqcpp) +columnstore_link(querystats PRIVATE loggingcpp) diff --git a/utils/querytele/CMakeLists.txt b/utils/querytele/CMakeLists.txt index 9fd9af6a9..f5e58ae60 100644 --- a/utils/querytele/CMakeLists.txt +++ b/utils/querytele/CMakeLists.txt @@ -5,7 +5,7 @@ set(querytele_LIB_SRCS querytele.cpp queryteleclient.cpp querytele_constants.cpp ) columnstore_library(querytele ${querytele_LIB_SRCS}) -columnstore_link(querytele ${THRIFT_LIBRARY} boost_thread) +columnstore_link(querytele ${THRIFT_LIBRARY}) target_include_directories(querytele PRIVATE ${THRIFT_INCLUDE_DIRS}) -add_dependencies(querytele external_thrift) +add_dependencies(querytele external_boost external_thrift) diff --git a/utils/regr/CMakeLists.txt b/utils/regr/CMakeLists.txt index e4ce77d95..f2f378f34 100755 --- a/utils/regr/CMakeLists.txt +++ b/utils/regr/CMakeLists.txt @@ -21,12 +21,11 @@ set(regr_LIB_SRCS add_definitions(-DMYSQL_DYNAMIC_PLUGIN) columnstore_library(regr ${regr_LIB_SRCS}) -columnstore_link(regr PRIVATE loggingcpp messageqcpp) +columnstore_link(regr PRIVATE loggingcpp) set(regr_mysql_LIB_SRCS regrmysql.cpp modamysql.cpp) -# Do anyone use it? columnstore_mysql_plugin_library(regr_mysql SHARED ${regr_mysql_LIB_SRCS}) -add_dependencies(regr_mysql external_boost GenError) # for "idb_mysql.h" that uses generated mysqld_error.h +add_dependencies(regr_mysql external_boost) set_target_properties(regr_mysql PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../) diff --git a/utils/regr/moda.h b/utils/regr/moda.h index e709d264e..4cbdb0a3e 100644 --- a/utils/regr/moda.h +++ b/utils/regr/moda.h @@ -45,7 +45,7 @@ #include "calpontsystemcatalog.h" #include "windowfunctioncolumn.h" #include "hasher.h" -#include "mariadb_charset/collation.h" +#include "collation.h" #define EXPORT @@ -277,7 +277,7 @@ class Moda_impl_T : public mcsv1_UDAF { public: // Defaults OK - Moda_impl_T() : cs(8) {}; + Moda_impl_T() : cs(8){}; ~Moda_impl_T() override = default; mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; @@ -305,7 +305,7 @@ class moda : public mcsv1_UDAF { public: // Defaults OK - moda() : mcsv1_UDAF() {}; + moda() : mcsv1_UDAF(){}; ~moda() override = default; mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index b22e93b43..2ba4c2d46 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -5,5 +5,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rowgroup_LIB_SRCS rowaggregation.cpp rowgroup.cpp rowstorage.cpp) columnstore_library(rowgroup ${rowgroup_LIB_SRCS}) -columnstore_link(rowgroup PRIVATE funcexp loggingcpp compress) +columnstore_link(rowgroup PRIVATE ${NETSNMP_LIBRARIES} funcexp loggingcpp) add_dependencies(rowgroup external_boost) diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 5917885b3..50baf681c 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -53,7 +53,7 @@ #include "branchpred.h" #include "datatypes/mcs_int128.h" -#include "mariadb_charset/collation.h" +#include "collation.h" #include "common/hashfamily.h" #include "buffertypes.h" diff --git a/utils/statistics_manager/CMakeLists.txt b/utils/statistics_manager/CMakeLists.txt deleted file mode 100644 index 8bcdfd8ca..000000000 --- a/utils/statistics_manager/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -include_directories(${ENGINE_COMMON_INCLUDES}) - -# ########## next target ############### - -columnstore_static_library(statistics_manager statistics.cpp) -columnstore_link(statistics_manager PRIVATE rowgroup) diff --git a/utils/threadpool/CMakeLists.txt b/utils/threadpool/CMakeLists.txt index 496fba57a..1502bb4b3 100644 --- a/utils/threadpool/CMakeLists.txt +++ b/utils/threadpool/CMakeLists.txt @@ -2,4 +2,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp fair_threadpool.cpp) columnstore_library(threadpool ${threadpool_LIB_SRCS}) -columnstore_link(threadpool PRIVATE boost_chrono boost_thread loggingcpp messageqcpp) +columnstore_link(threadpool PRIVATE boost_chrono loggingcpp) + +add_dependencies(threadpool external_boost) diff --git a/utils/udfsdk/CMakeLists.txt b/utils/udfsdk/CMakeLists.txt index 25e5020ff..e13f07ab2 100755 --- a/utils/udfsdk/CMakeLists.txt +++ b/utils/udfsdk/CMakeLists.txt @@ -11,11 +11,9 @@ set(udfsdk_LIB_SRCS ) columnstore_library(udfsdk ${udfsdk_LIB_SRCS}) -columnstore_link(udfsdk PRIVATE loggingcpp messageqcpp) +columnstore_link(udfsdk PRIVATE loggingcpp) # Do anyone use it? add_definitions(-DMYSQL_DYNAMIC_PLUGIN) set(udf_mysql_LIB_SRCS udfmysql.cpp) columnstore_mysql_plugin_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS}) - -add_dependencies(udf_mysql GenError) # for "idb_mysql.h" that uses generated mysqld_error.h diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 8b62dc2b2..4468c1c05 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -34,7 +34,7 @@ set(brm_LIB_SRCS ) columnstore_library(brm ${brm_LIB_SRCS}) -columnstore_link(brm loggingcpp datatypes oamcpp boost_thread messageqcpp) +columnstore_link(brm loggingcpp datatypes) # ########## next target ############### @@ -62,41 +62,53 @@ columnstore_link(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS set(reset_locks_SRCS reset_locks.cpp) columnstore_executable(reset_locks ${reset_locks_SRCS}) -columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) +columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) # ########## next target ############### set(rollback_SRCS rollback.cpp) columnstore_executable(rollback ${rollback_SRCS}) -columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) +columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) # ########## next target ############### set(save_brm_SRCS save_brm.cpp) columnstore_executable(save_brm ${save_brm_SRCS}) -columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) +columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) # ########## next target ############### set(load_brm_SRCS load_brm.cpp) columnstore_executable(load_brm ${load_brm_SRCS}) -columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) +columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) columnstore_executable(mcs-load-em load_em.cpp) -columnstore_link(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) +columnstore_link( + mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} +) columnstore_executable(mcs-load-brm-from-file load_brm_from_file.cpp) columnstore_link( - mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} + mcs-load-brm-from-file + ${ENGINE_LDFLAGS} + ${MARIADB_CLIENT_LIBS} + ${ENGINE_OAM_LIBS} + ${ENGINE_EXEC_LIBS} + ${NETSNMP_LIBRARIES} boost_program_options ) columnstore_executable(mcs-shmem-locks shmem_locks.cpp) columnstore_link( - mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} + mcs-shmem-locks + ${ENGINE_LDFLAGS} + ${MARIADB_CLIENT_LIBS} + ${ENGINE_OAM_LIBS} + ${ENGINE_EXEC_LIBS} + ${NETSNMP_LIBRARIES} boost_program_options ) diff --git a/writeengine/bulk/CMakeLists.txt b/writeengine/bulk/CMakeLists.txt index 229fa19c2..0bb90a586 100644 --- a/writeengine/bulk/CMakeLists.txt +++ b/writeengine/bulk/CMakeLists.txt @@ -29,7 +29,7 @@ set(we_bulk_STAT_SRCS add_definitions(-D_FILE_OFFSET_BITS=64) columnstore_static_library(we_bulk ${we_bulk_STAT_SRCS}) -columnstore_link(we_bulk loggingcpp boost_program_options) +columnstore_link(we_bulk ${NETSNMP_LIBRARIES} loggingcpp boost_program_options) remove_definitions(-D_FILE_OFFSET_BITS=64) @@ -38,13 +38,13 @@ remove_definitions(-D_FILE_OFFSET_BITS=64) set(cpimport.bin_SRCS cpimport.cpp) columnstore_executable(cpimport.bin ${cpimport.bin_SRCS}) - +add_dependencies(cpimport.bin marias3) columnstore_link( cpimport.bin ${ENGINE_LDFLAGS} + ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ${S3API_DEPS} we_bulk we_xml - marias3 ) diff --git a/writeengine/client/CMakeLists.txt b/writeengine/client/CMakeLists.txt index 910680f2c..52633ceca 100644 --- a/writeengine/client/CMakeLists.txt +++ b/writeengine/client/CMakeLists.txt @@ -5,4 +5,7 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(writeengineclient_LIB_SRCS we_clients.cpp we_ddlcommandclient.cpp we_dmlcommandclient.cpp) columnstore_library(writeengineclient ${writeengineclient_LIB_SRCS}) -columnstore_link(writeengineclient boost_thread oamcpp messageqcpp loggingcpp) + +add_dependencies(writeengineclient loggingcpp) + +columnstore_link(writeengineclient ${NETSNMP_LIBRARIES}) diff --git a/writeengine/redistribute/CMakeLists.txt b/writeengine/redistribute/CMakeLists.txt index 5ca49b172..29b8b94ee 100644 --- a/writeengine/redistribute/CMakeLists.txt +++ b/writeengine/redistribute/CMakeLists.txt @@ -7,6 +7,9 @@ set(writeengineredistribute_LIB_SRCS we_redistribute.cpp we_redistributecontrol. ) columnstore_library(writeengineredistribute ${writeengineredistribute_LIB_SRCS}) -columnstore_link(writeengineredistribute loggingcpp oamcpp boost_thread messageqcpp) + +add_dependencies(writeengineredistribute loggingcpp) + +columnstore_link(writeengineredistribute ${NETSNMP_LIBRARIES}) target_compile_definitions(writeengineredistribute PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index 0843f246f..9ebc5cd72 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -18,4 +18,9 @@ set(WriteEngineServer_SRCS ) columnstore_executable(WriteEngineServer ${WriteEngineServer_SRCS}) -columnstore_link(WriteEngineServer ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool writeengineredistribute loggingcpp) + +add_dependencies(WriteEngineServer loggingcpp) + +columnstore_link( + WriteEngineServer ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} threadpool writeengineredistribute +) diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 158011e6e..7772351ab 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -40,7 +40,7 @@ #include "IDBDataFile.h" #include "IDBPolicy.h" #include "nullstring.h" -#include "mariadb_charset/collation.h" // For CHARSET_INFO struct +#include "collation.h" // For CHARSET_INFO struct #undef EXPORT #undef DELETE @@ -408,8 +408,8 @@ struct JobColumn /** @brief Job Column Structure */ int compressionType; /** @brief compression type */ bool autoIncFlag; /** @brief auto increment flag */ DctnryStruct dctnry; /** @brief dictionary structure */ - int128_t fMinIntSat; /** @brief For integer type, the min saturation value */ - uint128_t fMaxIntSat; /** @brief For integer type, the max saturation value */ + int128_t fMinIntSat; /** @brief For integer type, the min saturation value */ + uint128_t fMaxIntSat; /** @brief For integer type, the max saturation value */ double fMinDblSat; /** @brief for float/double, the min saturation value */ double fMaxDblSat; /** @brief for float/double, the max saturation value */ bool fWithDefault; /** @brief With default */ @@ -447,9 +447,10 @@ struct JobColumn /** @brief Job Column Structure */ , cs(nullptr) { } - JobColumn(const std::string& colName_, OID mapOid_, const std::string& typeName_, int width_, - int definedWidth_, int compressionType_, int dctnryCompressionType_, int64_t minIntSat_, - uint64_t maxIntSat_, bool withDefault_, unsigned long long defaultUInt_) + JobColumn(const std::string& colName_, OID mapOid_, const std::string& typeName_, + int width_, int definedWidth_, int compressionType_, int dctnryCompressionType_, + int64_t minIntSat_, uint64_t maxIntSat_, bool withDefault_, + unsigned long long defaultUInt_) : colName(colName_) , mapOid(mapOid_) , dataType(execplan::CalpontSystemCatalog::INT) diff --git a/writeengine/splitter/CMakeLists.txt b/writeengine/splitter/CMakeLists.txt index 8d2d12251..7b05928f6 100644 --- a/writeengine/splitter/CMakeLists.txt +++ b/writeengine/splitter/CMakeLists.txt @@ -16,13 +16,15 @@ set(cpimport_SRCS columnstore_executable(cpimport ${cpimport_SRCS}) +add_dependencies(cpimport loggingcpp) + columnstore_link( cpimport ${ENGINE_LDFLAGS} + ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} batchloader threadpool marias3 boost_program_options - loggingcpp ) diff --git a/writeengine/wrapper/CMakeLists.txt b/writeengine/wrapper/CMakeLists.txt index 2a1e71325..28a7b50f9 100644 --- a/writeengine/wrapper/CMakeLists.txt +++ b/writeengine/wrapper/CMakeLists.txt @@ -37,4 +37,7 @@ set(writeengine_LIB_SRCS add_definitions(-D_FILE_OFFSET_BITS=64) columnstore_library(writeengine ${writeengine_LIB_SRCS}) -columnstore_link(writeengine loggingcpp oamcpp boost_thread compress messageqcpp) + +add_dependencies(writeengine loggingcpp) + +columnstore_link(writeengine ${NETSNMP_LIBRARIES}) diff --git a/writeengine/xml/CMakeLists.txt b/writeengine/xml/CMakeLists.txt index 2ef5b0e51..98ab2fbb3 100644 --- a/writeengine/xml/CMakeLists.txt +++ b/writeengine/xml/CMakeLists.txt @@ -6,5 +6,6 @@ set(we_xml_STAT_SRCS we_xmlop.cpp we_xmljob.cpp we_xmlgendata.cpp we_xmlgenproc. add_definitions(-D_FILE_OFFSET_BITS=64) -columnstore_static_library(we_xml ${we_xml_STAT_SRCS}) -columnstore_link(we_xml loggingcpp) +add_library(we_xml STATIC ${we_xml_STAT_SRCS}) + +add_dependencies(we_xml loggingcpp)