diff --git a/CMakeLists.txt b/CMakeLists.txt index b86fc76e1..bf1da3a6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,8 +88,3 @@ set(COMPONENTS foreach(component ${COMPONENTS}) add_subdirectory(${component}) endforeach() - -add_dependencies(udf_mysql GenError) -add_dependencies(funcexp GenError) -add_dependencies(oamcpp GenError) -add_dependencies(regr_mysql GenError) diff --git a/cmake/ColumnstoreLibrary.cmake b/cmake/ColumnstoreLibrary.cmake index 36dbc6813..13a6097d6 100644 --- a/cmake/ColumnstoreLibrary.cmake +++ b/cmake/ColumnstoreLibrary.cmake @@ -82,6 +82,16 @@ macro(columnstore_shared_library libname) columnstore_install_target(${libname} ${ENGINE_LIBDIR}) endmacro() +macro(columnstore_static_library libname) + add_definitions(-fPIC -DPIC) + add_library(${libname} STATIC ${ARGN}) +endmacro() + +macro(columnstore_shared_library libname) + add_library(${libname} SHARED ${ARGN}) + columnstore_install_target(${libname} ${ENGINE_LIBDIR}) +endmacro() + macro(columnstore_library libname) if(COLUMNSTORE_STATIC_LIBRARIES) columnstore_static_library(${libname} ${ARGN}) @@ -99,6 +109,15 @@ macro(columnstore_link libname) target_link_libraries(${libname} ${ARGN}) endmacro() +macro(columnstore_mysql_plugin_library libname) + add_library(${libname} SHARED ${ARGN}) + columnstore_install_target(${libname} ${MARIADB_PLUGINDIR}) +endmacro() + +macro(columnstore_link libname) + target_link_libraries(${libname} ${ARGN}) +endmacro() + macro(columnstore_executable executable_name) add_executable(${executable_name} ${ARGN}) columnstore_install_target(${executable_name} ${ENGINE_BINDIR}) diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 2f01010f3..e5abe1112 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -20,7 +20,7 @@ #include #include #include "exceptclasses.h" -#include "conststring.h" +#include "basic/conststring.h" #include "mcs_datatype_basic.h" #include "mcs_numeric_limits.h" #include "mcs_data_condition.h" @@ -796,7 +796,7 @@ class MinMaxPartitionInfo : public MinMaxInfo uint64_t m_status; public: - MinMaxPartitionInfo() : m_status(0){}; + MinMaxPartitionInfo() : m_status(0) {}; explicit MinMaxPartitionInfo(const BRM::EMEntry& entry); void set_invalid() { @@ -1102,7 +1102,8 @@ class TypeHandlerBit : public TypeHandler idbassert(0); // QQ return 1; } - std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, + const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } @@ -1135,7 +1136,7 @@ class TypeHandlerBit : public TypeHandler const ConvertFromStringParam& /*prm*/, const std::string& /*str*/, bool& /*pushWarning*/) const override; -const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& /*attr*/) const override + const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& /*attr*/) const override { idbassert(0); return nullptr; @@ -2033,7 +2034,8 @@ class TypeHandlerReal : public TypeHandler { return {}; // QQ: real types were not handled in IDB_format() } - std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, + const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } @@ -2329,7 +2331,8 @@ class TypeHandlerBlob : public TypeHandlerStr { return storeValueToFieldBlobText(row, pos, f); } - std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, + const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } @@ -2355,7 +2358,8 @@ class TypeHandlerText : public TypeHandlerStr { return storeValueToFieldBlobText(row, pos, f); } - std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, + const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } @@ -2386,7 +2390,8 @@ class TypeHandlerClob : public TypeHandlerStr idbassert(0); // QQ return 1; } - std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override + std::string format(const SimpleValue& /*v*/, + const SystemCatalog::TypeAttributesStd& /*attr*/) const override { return "0"; // QQ } diff --git a/datatypes/mcs_string.h b/datatypes/mcs_string.h index dbd8feb92..0714945aa 100644 --- a/datatypes/mcs_string.h +++ b/datatypes/mcs_string.h @@ -17,8 +17,8 @@ #pragma once -#include "conststring.h" -#include "collation.h" // class Charset +#include "basic/conststring.h" +#include "mariadb_charset/collation.h" // class Charset namespace datatypes { diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 4ed7d69cc..d4fdf6280 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -42,4 +42,4 @@ columnstore_library( ${FLEX_ddl_scan_OUTPUTS} ) -columnstore_link(ddlpackage loggingcpp) +columnstore_link(ddlpackage loggingcpp messageqcpp) diff --git a/dbcon/ddlpackage/sqlparser.h b/dbcon/ddlpackage/sqlparser.h index 460433fc6..c45b903aa 100644 --- a/dbcon/ddlpackage/sqlparser.h +++ b/dbcon/ddlpackage/sqlparser.h @@ -27,7 +27,7 @@ */ #include -#include "collation.h" // CHARSET_INFO +#include "mariadb_charset/collation.h" // CHARSET_INFO #include "ddlpkg.h" #include "mariadb_my_sys.h" // myf, MYF() @@ -87,7 +87,7 @@ struct pass_to_bison myf utf8_flag; pass_to_bison(ParseTree* pt) - : fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)){}; + : fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)) {}; }; class SqlParser diff --git a/dbcon/ddlpackageproc/CMakeLists.txt b/dbcon/ddlpackageproc/CMakeLists.txt index 3eed93356..151d039fb 100644 --- a/dbcon/ddlpackageproc/CMakeLists.txt +++ b/dbcon/ddlpackageproc/CMakeLists.txt @@ -13,4 +13,4 @@ set(ddlpackageproc_LIB_SRCS ) columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS}) -columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp) +columnstore_link(ddlpackageproc loggingcpp oamcpp messageqcpp) diff --git a/dbcon/dmlpackage/CMakeLists.txt b/dbcon/dmlpackage/CMakeLists.txt index a0e4ddffa..0a138cca5 100644 --- a/dbcon/dmlpackage/CMakeLists.txt +++ b/dbcon/dmlpackage/CMakeLists.txt @@ -41,4 +41,4 @@ columnstore_library( ${FLEX_dml_scan_OUTPUTS} ) -columnstore_link(dmlpackage loggingcpp) +columnstore_link(dmlpackage loggingcpp messageqcpp) diff --git a/dbcon/dmlpackageproc/CMakeLists.txt b/dbcon/dmlpackageproc/CMakeLists.txt index 177bb799b..8a862ccb5 100644 --- a/dbcon/dmlpackageproc/CMakeLists.txt +++ b/dbcon/dmlpackageproc/CMakeLists.txt @@ -13,4 +13,4 @@ set(dmlpackageproc_LIB_SRCS ) columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS}) -columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp) +columnstore_link(dmlpackageproc loggingcpp oamcpp messageqcpp) diff --git a/dbcon/execplan/CMakeLists.txt b/dbcon/execplan/CMakeLists.txt index 6e1acbb98..4edaf97d0 100755 --- a/dbcon/execplan/CMakeLists.txt +++ b/dbcon/execplan/CMakeLists.txt @@ -46,4 +46,12 @@ set(execplan_LIB_SRCS ) columnstore_library(execplan ${execplan_LIB_SRCS}) -columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp) +columnstore_link( + execplan + messageqcpp + ${ENGINE_DT_LIB} + pron + oamcpp + loggingcpp + querytele +) diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index dee640c3e..cfb748c54 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -49,7 +49,7 @@ #undef max #include "mcs_datatype.h" -#include "collation.h" // CHARSET_INFO, class Charset +#include "mariadb_charset/collation.h" // CHARSET_INFO, class Charset #include "nullstring.h" class ExecPlanTest; @@ -75,7 +75,8 @@ const int32_t IDB_VTABLE_ID = CNX_VTABLE_ID; /** * A struct to hold a list of table partitions. */ -struct Partitions { +struct Partitions +{ std::vector fPartNames; void serialize(messageqcpp::ByteStream& b) const { @@ -96,12 +97,11 @@ struct Partitions { b >> t; fPartNames.push_back(t); } - } }; -bool operator <(const Partitions& a, const Partitions& b); -bool operator ==(const Partitions& a, const Partitions& b); -bool operator !=(const Partitions& a, const Partitions& b); // for GCC9 +bool operator<(const Partitions& a, const Partitions& b); +bool operator==(const Partitions& a, const Partitions& b); +bool operator!=(const Partitions& a, const Partitions& b); // for GCC9 /** The CalpontSystemCatalog class * @@ -259,9 +259,9 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog public: ColType() = default; ColType(const ColType& rhs); - ColType(int32_t colWidth_, int32_t scale_, int32_t precision_, - const ConstraintType& constraintType_, const DictOID& ddn_, int32_t colPosition_, - int32_t compressionType_, OID columnOID_, const ColDataType& colDataType_); + ColType(int32_t colWidth_, int32_t scale_, int32_t precision_, const ConstraintType& constraintType_, + const DictOID& ddn_, int32_t colPosition_, int32_t compressionType_, OID columnOID_, + const ColDataType& colDataType_); ColType& operator=(const ColType& rhs); CHARSET_INFO* getCharset() const; @@ -1301,8 +1301,8 @@ const int OID_SYSCOLUMN_MINVALUE = SYSCOLUMN_BASE + 19; /** @brief min va const int OID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_BASE + 20; /** @brief max value col */ const int OID_SYSCOLUMN_COMPRESSIONTYPE = SYSCOLUMN_BASE + 21; /** @brief compression type */ const int OID_SYSCOLUMN_NEXTVALUE = SYSCOLUMN_BASE + 22; /** @brief next value */ -const int OID_SYSCOLUMN_CHARSETNUM = SYSCOLUMN_BASE + 23; /** @brief character set number for the column */ -const int SYSCOLUMN_MAX = SYSCOLUMN_BASE + 24; // be sure this is one more than the highest # +const int OID_SYSCOLUMN_CHARSETNUM = SYSCOLUMN_BASE + 23; /** @brief character set number for the column */ +const int SYSCOLUMN_MAX = SYSCOLUMN_BASE + 24; // be sure this is one more than the highest # /***************************************************** * SYSTABLE columns dictionary OID definition diff --git a/dbcon/execplan/clientrotator.cpp b/dbcon/execplan/clientrotator.cpp index 395d76482..4e0d1303a 100644 --- a/dbcon/execplan/clientrotator.cpp +++ b/dbcon/execplan/clientrotator.cpp @@ -47,8 +47,6 @@ using namespace logging; #include "clientrotator.h" -//#include "idb_mysql.h" - /** Debug macro */ #ifdef INFINIDB_DEBUG #define IDEBUG(x) \ diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 434c2c768..b6e976d46 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -39,7 +39,7 @@ #include "returnedcolumn.h" #include "dataconvert.h" -#include "collation.h" // CHARSET_INFO +#include "mariadb_charset/collation.h" // CHARSET_INFO namespace messageqcpp { diff --git a/dbcon/execplan/pseudocolumn.cpp b/dbcon/execplan/pseudocolumn.cpp index a6055489e..38f2be10f 100644 --- a/dbcon/execplan/pseudocolumn.cpp +++ b/dbcon/execplan/pseudocolumn.cpp @@ -28,7 +28,7 @@ using namespace std; #include "basic/string_utils.h" #include "bytestream.h" -#include "collation.h" +#include "mariadb_charset/collation.h" using namespace messageqcpp; diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 3783ccf14..a63e5c82c 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -68,7 +68,15 @@ columnstore_library(joblist ${joblist_LIB_SRCS}) target_include_directories( joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR} ) -columnstore_link(joblist loggingcpp) +columnstore_link( + joblist + loggingcpp + boost_thread + oamcpp + querytele + messageqcpp + statistics_manager +) if(WITH_ORDERBY_UT) columnstore_executable(job_orderby_tests orderby-tests.cpp) diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 96add786b..7ba090460 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -71,7 +71,7 @@ using namespace dataconvert; #include "jlf_tuplejoblist.h" using namespace joblist; -#include "statistics.h" +#include "statistics_manager/statistics.h" #ifdef __clang__ #pragma clang diagnostic push diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 711c0a188..488943704 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -63,7 +63,7 @@ if(COMMAND mysql_add_plugin) ${PLUGIN_EXEC_LIBS} ${PLUGIN_WRITE_LIBS} joblist_server - ${NETSNMP_LIBRARIES} + statistics_manager ${MARIADB_CLIENT_LIBS} ${S3API_DEPS} threadpool @@ -89,10 +89,10 @@ else() ${S3API_DEPS} ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} - ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool loggingcpp + statistics_manager marias3 ) # define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 797ac40c4..a7e974159 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -128,9 +128,9 @@ using namespace funcexp; #include "ha_mcs_sysvars.h" #include "ha_mcs_datatype.h" -#include "statistics.h" #include "ha_mcs_logging.h" #include "ha_subquery.h" +#include "statistics_manager/statistics.h" namespace cal_impl_if { diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index ba700ff93..cc71e9425 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -14,10 +14,17 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -// One include file to deal with all the MySQL pollution of the -// global namespace -// -// Don't include ANY mysql headers anywhere except here! + +/* One include file to deal with all the MySQL pollution of the + global namespace + + Don't include ANY mysql headers anywhere except here! + + WARN: if any cmake build target uses this include file, + GenError from server must be added to the target dependencies + to generate mysqld_error.h used below +*/ + #pragma once #ifdef TEST_MCSCONFIG_H diff --git a/ddlproc/CMakeLists.txt b/ddlproc/CMakeLists.txt index 64924fb79..0da0d740b 100644 --- a/ddlproc/CMakeLists.txt +++ b/ddlproc/CMakeLists.txt @@ -6,4 +6,4 @@ set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp) columnstore_executable(DDLProc ${DDLProc_SRCS}) -columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp) +columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool loggingcpp) diff --git a/dmlproc/CMakeLists.txt b/dmlproc/CMakeLists.txt index af9ed6492..036d7b0ea 100644 --- a/dmlproc/CMakeLists.txt +++ b/dmlproc/CMakeLists.txt @@ -12,7 +12,6 @@ columnstore_link( DMLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} - ${NETSNMP_LIBRARIES} threadpool ddlcleanuputil batchloader diff --git a/oam/oamcpp/CMakeLists.txt b/oam/oamcpp/CMakeLists.txt index d5b4ac307..cdce7a55b 100644 --- a/oam/oamcpp/CMakeLists.txt +++ b/oam/oamcpp/CMakeLists.txt @@ -5,6 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp) columnstore_library(oamcpp ${oamcpp_LIB_SRCS}) -columnstore_link(oamcpp loggingcpp) +columnstore_link(oamcpp loggingcpp idbboot) target_compile_options(oamcpp PRIVATE -Wno-unused-result) diff --git a/oamapps/columnstoreSupport/CMakeLists.txt b/oamapps/columnstoreSupport/CMakeLists.txt deleted file mode 100644 index feaf17aa2..000000000 --- a/oamapps/columnstoreSupport/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -include_directories(${ENGINE_COMMON_INCLUDES}) - -# ########## next target ############### - -set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp) - -columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS}) -target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result) -columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS}) - -columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR}) -columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR}) -columnstore_install_program(configReport.sh, ${ENGINE_BINDIR}) -columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR}) -columnstore_install_program(logReport.sh, ${ENGINE_BINDIR}) -columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR}) diff --git a/oamapps/columnstoreSupport/alarmReport.sh b/oamapps/columnstoreSupport/alarmReport.sh deleted file mode 100755 index ec81e2228..000000000 --- a/oamapps/columnstoreSupport/alarmReport.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /bin/sh -# -# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - SERVER=$1 -else - SERVER="localhost" -fi - -if [ $2 ] ; then - DATE=$2 -else - DATE=" " -fi - -#get temp directory -tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir` - -rm -f ${tmpDir}/logReport.log - -{ -echo " " -echo "******************** Alarm Report for $SERVER ********************" -echo " " - -echo "-- Today's Alarms --" -echo " " -cat /var/log/mariadb/columnstore/alarm.log 2>/dev/null - -if test -f /var/log/mariadb/columnstore/archive/alarm.log-$DATE ; then - echo "-- Archived Alarms --" - echo " " - cat /var/log/mariadb/columnstore/archive/alarm.log-$DATE 2>/dev/null -fi - -} > ${tmpDir}/logReport.log - -exit 0 diff --git a/oamapps/columnstoreSupport/approximateRowCount.sh b/oamapps/columnstoreSupport/approximateRowCount.sh deleted file mode 100755 index 26f3e41aa..000000000 --- a/oamapps/columnstoreSupport/approximateRowCount.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# -# Estimates the row count for a given table. Uses number of extents * 8M for the estimate. -# - -# -# Initialize variables. -# - -if [ -z "$MYSQLCMD" ]; then - MYSQLCMD="mysql -u root" -fi - -# -# Validate that there are two parameters - schema and table. -# -if [ $# -ne 2 ]; then - echo "" - echo "Reports the approximate row count for the given table." - echo "" - echo "Parameters:" - echo " Schema" - echo " Table" -fi -db=$1 -table=$2 - -# -# Validate that the table exists. -# -sql="select count(*) from systable where \`schema\`='$db' and tablename='$table';" -count=`$MYSQLCMD calpontsys --skip-column-names -e "$sql;"` -if [ $count -le 0 ]; then - echo "" - echo "$db.$table does not exist in Columnstore." - echo "" - exit 1 -fi - -# -# Grab the objectid and column width for a column in the table. -# -sql="select objectid from syscolumn where \`schema\`='$db' and tablename='$table' limit 1;" -objectid=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"` -sql="select columnlength from syscolumn where objectid=$objectid;" -colWidth=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"` - -# -# Use editem to count the extents. -# -extentCount=`editem -o $objectid | wc -l` -let extentCount-=2 # Take out the 2 extra rows for header and blank line at end. -let approximateRowCount=$extentCount*8192*1024; - -echo "" -echo "Approximate row count for $db.$table is $approximateRowCount." -echo "" - -exit 0 diff --git a/oamapps/columnstoreSupport/bulklogReport.sh b/oamapps/columnstoreSupport/bulklogReport.sh deleted file mode 100755 index 83076f4eb..000000000 --- a/oamapps/columnstoreSupport/bulklogReport.sh +++ /dev/null @@ -1,30 +0,0 @@ -#! /bin/sh -# -# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - OUT_FILE=$2 -else - OUT_FILE=${MODULE}_logReport.txt -fi - -{ - -if test -d /var/lib/columnstore/data/bulk ; then - echo " " - echo "-- Check for Errors in Bulk Logs --" - echo " " - echo "################# egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err #################" - echo " " - egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err 2>/dev/null -fi - -} >> $OUT_FILE - -exit 0 diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp deleted file mode 100644 index 51bd670b9..000000000 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ /dev/null @@ -1,897 +0,0 @@ -/* Copyright (C) 2013 Calpont Corp. */ -/* Copyright (C) 2016 MariaDB Corporation */ - -/****************************************************************************************** - * $Id: columnstoreSupport.cpp 64 2006-10-12 22:21:51Z dhill $ - * - * - * - ******************************************************************************************/ -/** - * @file - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "stdio.h" -#include "ctype.h" -#include -#include -#include - -#include "mcsconfig.h" -#include "liboamcpp.h" -#include "configcpp.h" -#include "installdir.h" -#include "mcsSupportUtil.h" -#include "columnstoreversion.h" - -using namespace std; -using namespace oam; -using namespace config; - -typedef struct Child_Module_struct -{ - std::string moduleName; - std::string moduleIP; - std::string hostName; -} ChildModule; - -typedef std::vector ChildModuleList; - -string currentDate; -string systemName; -string localModule; -string localModuleHostName; -ChildModuleList childmodulelist; -ChildModuleList parentmodulelist; -ChildModule childmodule; - -string rootPassword = ""; -string debug_flag = "0"; -string mysqlpw = " "; -string tmpDir; - -int runningThreads = 0; -pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; - -typedef boost::tuple threadInfo_t; - -bool LOCAL = false; - -void* childReportThread(threadInfo_t* st) -{ - assert(st); - ChildModuleList::iterator& list = boost::get<0>(*st); - string reportType = boost::get<1>(*st); - - string remoteModuleName = (*list).moduleName; - string remoteModuleIP = (*list).moduleIP; - string remoteHostName = (*list).hostName; - - pthread_mutex_lock(&mutex1); - runningThreads++; - // cout << "++ " << runningThreads << endl; - pthread_mutex_unlock(&mutex1); - - string outputFile; - - if (reportType == "log") - { - outputFile = remoteModuleName + "_" + reportType + "Report.tar.gz"; - } - else - { - outputFile = remoteModuleName + "_" + reportType + "Report.txt"; - - FILE* pOutputFile = fopen(outputFile.c_str(), "a"); - if (pOutputFile == NULL) - { - printf("Could not open file: %s", outputFile.c_str()); - exit(1); - } - - fprintf(pOutputFile, - "********************************************************************************\n" - "\n" - " System %s\n" - " columnstoreSupportReport script ran from Module %s on %s\n" - " SoftwareVersion = %s-%s" - "\n" - "********************************************************************************\n" - "\n" - " %s report\n" - "\n" - "********************************************************************************\n", - systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), - columnstore_release.c_str(), reportType.c_str()); - } - - cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl; - - string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + ";" + reportType + "Report.sh " + - remoteModuleName + "' " + debug_flag + " - forcetty"; - int rtnCode = system(cmd.c_str()); - - if (WEXITSTATUS(rtnCode) != 0) - { - cout << "Error with running remote_command.sh, exiting..." << endl; - } - - cmd = "remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile + - " > /dev/null 2>&1"; - rtnCode = system(cmd.c_str()); - - if (WEXITSTATUS(rtnCode) != 0) - cout << "ERROR: failed to retrieve " << tmpDir << "/" << outputFile << " from " + remoteHostName << endl; - - pthread_mutex_lock(&mutex1); - runningThreads--; - // cout << "-- " << runningThreads << endl; - pthread_mutex_unlock(&mutex1); - - // exit thread - pthread_exit(0); -} - -void* reportThread(string* reporttype) -{ - assert(reporttype); - string reportType = *reporttype; - - Oam oam; - - pthread_mutex_lock(&mutex1); - runningThreads++; - // cout << "++ " << runningThreads << endl; - pthread_mutex_unlock(&mutex1); - - string outputFile = localModule + "_" + reportType + "Report.txt"; - - FILE* pOutputFile = fopen(outputFile.c_str(), "a"); - if (pOutputFile == NULL) - { - printf("Could not open file: %s", outputFile.c_str()); - exit(1); - } - // get local report - fprintf(pOutputFile, - "********************************************************************************\n" - "\n" - " System %s\n" - " columnstoreSupportReport script ran from Module %s on %s\n" - " SoftwareVersion = %s-%s" - "\n" - "********************************************************************************\n" - "\n" - " %s report\n" - "\n" - "********************************************************************************\n", - systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), - columnstore_release.c_str(), reportType.c_str()); - - fclose(pOutputFile); - // run on child servers and get report - if (!LOCAL) - { - ChildModuleList::iterator list1 = childmodulelist.begin(); - - for (; list1 != childmodulelist.end(); list1++) - { - threadInfo_t* st = new threadInfo_t; - *st = boost::make_tuple(list1, reportType); - - pthread_t childreportthread; - int status = pthread_create(&childreportthread, NULL, (void* (*)(void*)) & childReportThread, st); - - if (status != 0) - { - cout << "ERROR: childreportthread: pthread_create failed, return status = " + oam.itoa(status) - << endl; - } - - sleep(1); - } - } - - if (reportType == "log") - { - // run log config on local server - cout << "Get log config data for " + localModule << endl; - - string cmd = "logReport.sh " + localModule + " " + outputFile; - system(cmd.c_str()); - } - else - { - string cmd = reportType + "Report.sh " + localModule + " " + outputFile; - system(cmd.c_str()); - - if (reportType == "config") - { - pOutputFile = fopen(outputFile.c_str(), "a"); - if (pOutputFile == NULL) - { - printf("Could not open file: %s", outputFile.c_str()); - exit(1); - } - - fprintf(pOutputFile, - "\n******************** System Network Configuration ******************************\n\n"); - getSystemNetworkConfig(pOutputFile); - - fprintf(pOutputFile, - "\n******************** System Module Configure **********************************\n\n"); - getModuleTypeConfig(pOutputFile); - - fprintf(pOutputFile, - "\n******************** System Storage Configuration *****************************\n\n"); - getStorageConfig(pOutputFile); - - fprintf(pOutputFile, - "\n******************** System Storage Status ************************************\n\n"); - getStorageStatus(pOutputFile); - - // BT: most of this is tedious to collect and can be manually looked up in the debug.log file - // fprintf(pOutputFile,"\n******************** System Status - // ********************************************\n\n"); printSystemStatus(pOutputFile); - // printProcessStatus(pOutputFile); - // printAlarmSummary(pOutputFile); - // - // fprintf(pOutputFile,"\n******************** System Directories - // ***************************************\n\n"); getSystemDirectories(pOutputFile); - - boost::filesystem::path configFile = - std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml"); - boost::filesystem::copy_file(configFile, "./Columnstore.xml", - boost::filesystem::copy_options::overwrite_existing); - boost::filesystem::path SMconfigFile = - std::string(MCSSYSCONFDIR) + std::string("/columnstore/storagemanager.cnf"); - boost::filesystem::copy_file(SMconfigFile, "./storagemanager.cnf", - boost::filesystem::copy_options::overwrite_existing); - system("sed -i 's/.*aws_access_key_id.*/aws_access_key_id={PRIVATE}/' ./storagemanager.cnf"); - system("sed -i 's/.*aws_secret_access_key.*/aws_secret_access_key={PRIVATE}/' ./storagemanager.cnf"); - fclose(pOutputFile); - } - - /* - // TODO: This can be ported from mcsadmin if needed most info included does not seem useful at this time - if (reportType == "resource" ) - { - if (LOCAL) - { - fprintf(pOutputFile,"\n******************** mcsadmin getModuleResourceUsage - **************************\n\n"); string cmd = "mcsadmin getModuleResourceUsage " + localModule + " >> " + - outputFile; system(cmd.c_str()); - } - else - { - fprintf(pOutputFile,"\n******************** mcsadmin getSystemResourceUsage - **************************\n\n"); string cmd = "mcsadmin getSystemResourceUsage >> " + outputFile; - system(cmd.c_str()); - } - }*/ - } - - // exit thread - pthread_mutex_lock(&mutex1); - runningThreads--; - // cout << "-- " << runningThreads << endl; - pthread_mutex_unlock(&mutex1); - - pthread_exit(0); -} - -int main(int argc, char* argv[]) -{ - Oam oam; - - Config* sysConfig = Config::makeConfig(); - string SystemSection = "SystemConfig"; - string InstallSection = "Installation"; - - bool HARDWARE = false; - bool CONFIG = false; - bool DBMS = false; - bool RESOURCE = false; - bool LOG = false; - bool BULKLOG = false; - bool HADOOP = false; - - // get current time and date - time_t now; - now = time(NULL); - struct tm tm; - localtime_r(&now, &tm); - char timestamp[200]; - strftime(timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm); - currentDate = timestamp; - - char helpArg[3] = "-h"; - - // Get System Name - try - { - oam.getSystemConfig("SystemName", systemName); - } - catch (...) - { - systemName = "unassigned"; - } - - // get Local Module Name and Server Install Indicator - string singleServerInstall = "n"; - - oamModuleInfo_t st; - - try - { - st = oam.getModuleInfo(); - localModule = boost::get<0>(st); - } - catch (...) - { - cout << endl << "**** Failed : Failed to read Local Module Name" << endl; - exit(-1); - } - - if (argc == 1) - { - argv[1] = &helpArg[0]; - argc = 2; - } - - string DataFilePlugin; - - try - { - DataFilePlugin = sysConfig->getConfig(SystemSection, "DataFilePlugin"); - } - catch (...) - { - cout << "ERROR: Problem accessing Columnstore configuration file" << endl; - exit(-1); - } - - tmpDir = startup::StartUp::tmpDir(); - - for (int i = 1; i < argc; i++) - { - if (string("-h") == argv[i]) - { - cout << endl; - cout << "'columnstoreSupport' generates a Set of System Support Report Files in a tar file" << endl; - cout << "called columnstoreSupportReport.'system-name'.tar.gz in the local directory." << endl; - cout << "It should be run on the server with the DBRM front-end." << endl; - cout << "Check the Admin Guide for additional information." << endl; - cout << endl; - cout << "Usage: columnstoreSupport [-h][-a][-hw][-s][-c][-db][-r][-l][-bl][-lc][-p " - "'root-password'][-de]"; - - cout << endl; - cout << " -h help" << endl; - cout << " -a Output all Reports (excluding Bulk Logs Reports)" << endl; - cout << " -hw Output Hardware Reports only" << endl; - cout << " -c Output Configuration/Status Reports only" << endl; - cout << " -db Output DBMS Reports only" << endl; - cout << " -r Output Resource Reports only" << endl; - cout << " -l Output Columnstore Log/Alarms Reports only" << endl; - cout << " -bl Output Columnstore Bulk Log Reports only" << endl; - cout << " -lc Output Reports for Local Server only" << endl; - cout << " -p password (multi-server systems), root-password or 'ssh' to use 'ssh keys'" - << endl; - cout << " -de Debug Flag" << endl; - - exit(0); - } - else - { - if (string("-a") == argv[i]) - { - HARDWARE = true; - CONFIG = true; - DBMS = true; - RESOURCE = true; - LOG = true; - HADOOP = (DataFilePlugin.empty() ? false : true); - } - else if (string("-hw") == argv[i]) - HARDWARE = true; - else if (string("-c") == argv[i]) - CONFIG = true; - else if (string("-db") == argv[i]) - DBMS = true; - else if (string("-r") == argv[i]) - RESOURCE = true; - else if (string("-l") == argv[i]) - LOG = true; - else if (string("-bl") == argv[i]) - BULKLOG = true; - else if (string("-lc") == argv[i]) - LOCAL = true; - else if (string("-p") == argv[i]) - { - i++; - - if (argc == i) - { - cout << "ERROR: missing root password argument" << endl; - exit(-1); - } - - rootPassword = argv[i]; - - // add single quote for special characters - if (rootPassword != "ssh") - { - rootPassword = "'" + rootPassword + "'"; - } - } - else if (string("-mp") == argv[i]) - { - i++; - - if (argc == i) - { - cout << "ERROR: missing MariaDB Columnstore root user password argument" << endl; - exit(-1); - } - - mysqlpw = argv[i]; - mysqlpw = "'" + mysqlpw + "'"; - } - else if (string("-de") == argv[i]) - debug_flag = "1"; - else if (string("-hd") == argv[i]) - { - HADOOP = (DataFilePlugin.empty() ? false : true); - } - else - { - cout << "Invalid Option of '" << argv[i] << "', run with '-h' for help" << endl; - exit(1); - } - } - } - - // default to -a if nothing is set - if (!HARDWARE && !CONFIG && !DBMS && !RESOURCE && !LOG && !BULKLOG && !HADOOP) - { - HARDWARE = true; - CONFIG = true; - DBMS = true; - RESOURCE = true; - LOG = true; - HADOOP = (DataFilePlugin.empty() ? false : true); - } - - // get Parent OAM Module Name and setup of it's Custom OS files - string PrimaryUMModuleName; - - try - { - PrimaryUMModuleName = sysConfig->getConfig(SystemSection, "PrimaryUMModuleName"); - } - catch (...) - { - cout << "ERROR: Problem getting Parent OAM Module Name" << endl; - exit(-1); - } - - if (PrimaryUMModuleName == "unassigned") - PrimaryUMModuleName = localModule; - - if ((localModule != PrimaryUMModuleName) && DBMS) - { - char* pcommand = 0; - char* p; - string argument = "n"; - - while (true) - { - cout << endl << "You selected to get the DBMS data." << endl; - cout << "You need to run the columnstoreSupport command on module '" << PrimaryUMModuleName - << "' to get that information." << endl; - cout << "Or you can proceed on to get all data except the DBMS." << endl; - - pcommand = readline(" Do you want to proceed: (y or n) [n]: "); - - if (pcommand && *pcommand) - { - p = strtok(pcommand, " "); - argument = p; - free(pcommand); - pcommand = 0; - } - - if (pcommand) - { - free(pcommand); - pcommand = 0; - } - - if (argument == "y") - { - cout << endl; - break; - } - else if (argument == "n") - exit(1); - } - } - - // get number of worker-nodes, will tell us if a single server system - // get Parent OAM Module Name and setup of it's Custom OS files - try - { - string NumWorkers = sysConfig->getConfig("DBRM_Controller", "NumWorkers"); - - if (NumWorkers == "1") - singleServerInstall = "y"; - } - catch (...) - { - } - - if (singleServerInstall == "n" && !LOCAL) - if (HARDWARE || CONFIG || RESOURCE || LOG || HADOOP) - if (rootPassword.empty()) - { - cout << "ERROR: Multi-Module System, Password Argument required or use '-lc' option, check help for " - "more information" - << endl; - exit(-1); - } - - // get Parent OAM Module Name and setup of it's Custom OS files - // string parentOAMModuleName; - ChildModule parentOAMModule; - - try - { - parentOAMModule.moduleName = sysConfig->getConfig(SystemSection, "ParentOAMModuleName"); - } - catch (...) - { - cout << "ERROR: Problem getting Parent OAM Module Name" << endl; - exit(-1); - } - - // Get list of configured system modules - SystemModuleTypeConfig sysModuleTypeConfig; - - try - { - oam.getSystemConfig(sysModuleTypeConfig); - } - catch (...) - { - cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl; - exit(-1); - } - - string ModuleSection = "SystemModuleConfig"; - - for (unsigned int i = 0; i < sysModuleTypeConfig.moduletypeconfig.size(); i++) - { - string moduleType = sysModuleTypeConfig.moduletypeconfig[i].ModuleType; - int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount; - - if (moduleCount == 0) - // no modules equipped for this Module Type, skip - continue; - - // get IP addresses and Host Names - DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for (; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end(); listPT++) - { - string moduleName = (*listPT).DeviceName; - HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin(); - string moduleIPAddr = (*pt1).IPAddr; - string moduleHostName = (*pt1).HostName; - - if (moduleName == localModule) - { - localModuleHostName = moduleHostName; - } - - // save Child modules - if (moduleName != localModule && moduleType != "xm") - { - childmodule.moduleName = moduleName; - childmodule.moduleIP = moduleIPAddr; - childmodule.hostName = moduleHostName; - childmodulelist.push_back(childmodule); - } - - if (moduleName == parentOAMModule.moduleName) - { - parentOAMModule.moduleIP = moduleIPAddr; - parentOAMModule.hostName = moduleHostName; - parentOAMModule.moduleName = moduleName; - } - } - } // end of i for loop - - // create a clean Columnstore Support Report - system("rm -f *_configReport.txt"); - system("rm -f *_dbmsReport.txt"); - system("rm -f *_hardwareReport.txt"); - system("rm -f *_logReport.txt"); - system("rm -f *_bulklogReport.txt"); - system("rm -f *_resourceReport.txt"); - - // - // Configuration - // - if (CONFIG) - { - string reportType = "config"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); - if (status != 0) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // Alarms and Columnstore Logs - // - if (LOG) - { - string reportType = "log"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); - if (status != 0) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // Bulk Logs - // - if (BULKLOG) - { - string reportType = "bulklog"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); - if (status != 0) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // Hardware - // - if (HARDWARE) - { - string reportType = "hardware"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); - if (status != 0) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // Resources - // - if (RESOURCE) - { - string reportType = "resource"; - cout << "Get " + reportType + " report data for " + localModule << endl; - pthread_t reportthread; - int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType); - if (status != 0) - { - cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status); - } - sleep(1); - } - - // - // DBMS - // - if (DBMS) - { - cout << "Get dbms report data for " << localModule << endl; - - string outputFile = localModule + "_dbmsReport.txt"; - - FILE* pOutputFile = fopen(outputFile.c_str(), "w"); - if (pOutputFile == NULL) - { - cout << "Could not open file: " + outputFile << endl; - exit(1); - } - - fprintf(pOutputFile, - "********************************************************************************\n" - "\n" - " System %s\n" - " columnstoreSupportReport script ran from Module %s on %s\n" - " SoftwareVersion = %s-%s" - "\n" - "********************************************************************************\n" - "\n" - " DBMS report\n" - "\n" - "********************************************************************************\n", - systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(), - columnstore_release.c_str()); - - fclose(pOutputFile); - - // run DBMS report on local server - bool FAILED = false; - - if (localModule != PrimaryUMModuleName) - { - cout << " FAILED: run columnstoreSupport on '" << PrimaryUMModuleName << "' to get the dbrm report" - << endl; - FAILED = true; - } - else - { - // check if mysql is supported and get info - string logFile = tmpDir + "/idbmysql.log"; - string columnstoreMysql = "mysql -u root "; - string cmd = columnstoreMysql + " -e 'status' > " + logFile + " 2>&1"; - system(cmd.c_str()); - - // check for mysql password set - string pwprompt = " "; - - if (checkLogStatus(logFile, "ERROR 1045")) - { - cout << "NOTE: MariaDB Columnstore root user password is set" << endl; - - // needs a password, was password entered on command line - if (mysqlpw == " ") - { - // go check columnstore.cnf - string file = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; - ifstream oldFile(file.c_str()); - - vector lines; - char line[200]; - string buf; - - while (oldFile.getline(line, 200)) - { - buf = line; - string::size_type pos = buf.find("password", 0); - - if (pos != string::npos) - { - string::size_type pos1 = buf.find("=", 0); - - if (pos1 != string::npos) - { - pos = buf.find("#", 0); - - if (pos == string::npos) - { - // password arg in columnstore.cnf, go get password - cout << "NOTE: Using password from columnstore.cnf" << endl; - mysqlpw = buf.substr(pos1 + 1, 80); - cout << mysqlpw << endl; - break; - } - } - } - } - - oldFile.close(); - - if (mysqlpw == " ") - { - cout << "NOTE: No password provide on command line or found uncommented in columnstore.cnf" - << endl; - cout << endl; - string prompt = " *** Enter MariaDB Columnstore password > "; - mysqlpw = getpass(prompt.c_str()); - } - } - - // check for mysql password set - pwprompt = "--password=" + mysqlpw; - - string cmd = columnstoreMysql + pwprompt + " -e 'status' > " + logFile + " 2>&1"; - system(cmd.c_str()); - - if (checkLogStatus(logFile, "ERROR 1045")) - { - cout << "FAILED: Failed login using MariaDB Columnstore root user password '" << mysqlpw << "'" - << endl; - FAILED = true; - } - } - - if (!FAILED) - { - string cmd = "dbmsReport.sh " + localModule + " " + outputFile + " " + std::string(MCSSUPPORTDIR) + - " " + pwprompt; - system(cmd.c_str()); - } - } - - /* - BT: This doesn't appear to do anything - fprintf(pOutputFile,"\n******************** Database Size Report - *************************************\n\n"); getStorageStatus(pOutputFile); - - string file = "databaseSizeReport"; - ifstream File (file.c_str()); - - if (File) - { - string cmd = "databaseSizeReport >> " + outputFile; - system(cmd.c_str()); - } - */ - - boost::filesystem::path configFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf"; - boost::filesystem::copy_file(configFile, "./columnstore.cnf", - boost::filesystem::copy_options::overwrite_existing); - } - - int wait = 0; - - while (true) - { - // cout << "check " << runningThreads << endl; - if (runningThreads < 1) - break; - - sleep(2); - wait++; - - // give it 60 minutes to complete - if (wait >= 3600 * 5) - { - cout << "Timed out (60 minutes) waiting for Requests to complete" << endl; - } - } - - system("unix2dos *Report.txt > /dev/null 2>&1"); - system( - "rm -rf columnstoreSupportReport;" - "mkdir columnstoreSupportReport;" - "mv *Report.txt columnstoreSupportReport/. > /dev/null 2>&1;" - "mv Columnstore.xml columnstoreSupportReport/. > /dev/null 2>&1;" - "mv columnstore.cnf columnstoreSupportReport/. > /dev/null 2>&1;" - "mv storagemanager.cnf columnstoreSupportReport/. > /dev/null 2>&1;" - "mv *Report.tar.gz columnstoreSupportReport/. > /dev/null 2>&1"); - string cmd = "tar -zcf columnstoreSupportReport." + systemName + ".tar.gz columnstoreSupportReport/*"; - system(cmd.c_str()); - - cout << endl - << "Columnstore Support Script Successfully completed, files located in columnstoreSupportReport." + - systemName + ".tar.gz" - << endl; -} diff --git a/oamapps/columnstoreSupport/configReport.sh b/oamapps/columnstoreSupport/configReport.sh deleted file mode 100755 index cb642f137..000000000 --- a/oamapps/columnstoreSupport/configReport.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# -# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - OUT_FILE=$2 -else - OUT_FILE=${MODULE}_logReport.txt -fi - -{ -echo " " -echo "******************** Configuration/Status Report for ${MODULE} ********************" -echo " " - -chkconfig=`which chkconfig 2>/dev/null` -if [ -n "$chkconfig" ]; then - echo "-- chkconfig configuration --" - echo " " - echo "################# chkconfig --list | grep columnstore #################" - echo " " - chkconfig --list | grep columnstore 2>/dev/null -fi - -systemctl=`which systemctl 2>/dev/null` -if [ -n "$systemctl" ]; then - echo "-- systemctl configuration --" - echo " " - echo "################# systemctl list-unit-files --type=service | grep columnstore #################" - echo " " - systemctl list-unit-files --type=service | grep columnstore 2>/dev/null - echo "################# systemctl list-unit-files --type=service | grep mariadb #################" - echo " " - systemctl list-unit-files --type=service | grep mariadb 2>/dev/null -fi - -updaterc=`which update-rc.d 2>/dev/null` -if [ -n "$updaterc" ]; then - echo "-- services configuration --" - echo " " - echo "################# service --status-all | grep columnstore #################" - echo " " - service --status-all | grep columnstore 2>/dev/null -fi - - -echo " " -echo "-- fstab Configuration --" -echo " " -echo "################# cat /etc/fstab #################" -echo " " -cat /etc/fstab 2>/dev/null - -echo " " -echo "-- Server Processes --" -echo " " -echo "################# ps axu #################" -echo " " -ps axu - -echo " " -echo "-- Server Processes with resource usage --" -echo " " -echo "################# top -b -n 1 #################" -echo " " -top -b -n 1 - -} >> $OUT_FILE - -exit 0 diff --git a/oamapps/columnstoreSupport/dbmsReport.sh b/oamapps/columnstoreSupport/dbmsReport.sh deleted file mode 100755 index caac0cfd9..000000000 --- a/oamapps/columnstoreSupport/dbmsReport.sh +++ /dev/null @@ -1,64 +0,0 @@ -#! /bin/sh -# -# $Id: dbmsReport.sh -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - OUT_FILE=$2 -else - OUT_FILE=${MODULE}_logReport.txt -fi - -if [ $3 ] ; then - MCSSUPPORTDIR=$3 -else - MCSSUPPORTDIR="/usr/share/columnstore" -fi - -if [ $4 ] ; then - PW_PROMPT=$4 -else - PW_PROMPT="" -fi - -{ - -columnstoreMysql="mysql -u root ${PW_PROMPT} " - -if ${columnstoreMysql} -V > /dev/null 2>&1; then - echo " " - echo "******************** DBMS Columnstore Version *********************************" - echo " " - ${columnstoreMysql} -e 'status' - echo " " - echo "******************** DBMS Columnstore System Column ***************************" - echo " " - ${columnstoreMysql} -e 'desc calpontsys.syscolumn;' - echo " " - echo "******************** DBMS Columnstore System Table ****************************" - echo " " - ${columnstoreMysql} -e 'desc calpontsys.systable;' - echo " " - echo "******************** DBMS Columnstore System Catalog Data *********************" - echo " " - ${columnstoreMysql} calpontsys < $MCSSUPPORTDIR/dumpcat_mysql.sql - echo " " - echo "******************** DBMS Columnstore System Table Data ***********************" - echo "******************** DBMS Columnstore Databases *******************************" - echo " " - ${columnstoreMysql} -e 'show databases;' - echo " " - echo "******************** DBMS Columnstore variables *******************************" - echo " " - ${columnstoreMysql} -e 'show variables;' - echo " " -fi -} >> $OUT_FILE - -exit 0 - diff --git a/oamapps/columnstoreSupport/getMinMax.sh b/oamapps/columnstoreSupport/getMinMax.sh deleted file mode 100755 index 0e4a0aa2f..000000000 --- a/oamapps/columnstoreSupport/getMinMax.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash - -# -# Reports the max value from the extent map for the given column. -# - -# -# Initialize variables. -# - -if [ -z "$MYSQLCMD" ]; then - MYSQLCMD="mysql -u root" -fi - -# -# Validate that there are three parameters - schema and table and columnname. -# -if [ $# -ne 3 ]; then - echo "" - echo "Reports the max value for the given column." - echo "" - echo "Parameters:" - echo " Schema" - echo " Table" - echo " Column" - exit 1 -fi -db=$1 -table=$2 -column=$3 - -# -# Validate that the column exists. -# -sql="select count(*) from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';" -count=`$MYSQLCMD calpontsys --skip-column-names -e "$sql;"` -if [ $count -le 0 ]; then - echo "" - echo "$db.$table.$column does not exist in Columnstore." - echo "" - exit 1 -fi - -# -# Validate that the column type is one that this script supports. -# Supported Types: -# 6 int -# 8 date -# 9 bigint -# 11 datetime -sql="select datatype from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';" -dataType=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"` -if [ $dataType -ne 6 ] && [ $dataType -ne 8 ] && [ $dataType -ne 9 ] && [ $dataType -ne 11 ]; then - echo "" - echo "The column data type must be an int, bigint, date, or datetime." - echo "" - exit 1 -fi - -# -# Grab the objectid for the column. -# -sql="select objectid from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';" -objectid=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"` - -# -# Set the editem specific parameter if the column is a date or datetime. -# -if [ $dataType -eq 8 ]; then - parm="-t" -elif [ $dataType -eq 11 ]; then - parm="-s" -fi - -# -# Use the editem utility to get the min and max value. -# -editem -o $objectid $parm | grep max | awk -v dataType=$dataType ' - BEGIN { - allValid=1; - foundValidExtent=0; - } - { - if(dataType == 11) { - state=substr($14, 1, length($14)-1); # Datetime has date and time as two fields. - thisMin=$6 " " substr($7, 1, length($7)-1); - thisMax=$9 " " substr($10, 1, length($10)-1); - } - else { - state=substr($12, 1, length($12)-1); - thisMin=substr($6, 1, length($6)-1); - thisMax=substr($8, 1, length($8)-1); - } - if(state == "valid") { - if(!foundValidExtent) { - min=thisMin; - max=thisMax; - foundValidExtent=1; - } - else { - if(thisMin < min) { - min=thisMin; - } - if(thisMax > max) { - max=thisMax; - } - } - } - else { - allValid=0; - } - } - END { - if(foundValidExtent == 1) { - print ""; - print "Min=" min; - print "Max=" max; - print ""; - if(allValid == 0) { - print "Not all extents had min and max values set. Answer is incomplete." - } - } - else { - print ""; - print "There were not any extents with valid min/max values. Unable to provide answer."; - print ""; - } - }' - -exit 0 diff --git a/oamapps/columnstoreSupport/hardwareReport.sh b/oamapps/columnstoreSupport/hardwareReport.sh deleted file mode 100755 index d0bc92d5c..000000000 --- a/oamapps/columnstoreSupport/hardwareReport.sh +++ /dev/null @@ -1,79 +0,0 @@ -#! /bin/sh -# -# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - OUT_FILE=$2 -else - OUT_FILE=${MODULE}_logReport.txt -fi - -{ -echo " " -echo "******************** Hardware Report for ${MODULE} ********************" -echo " " - -echo "-- Server OS Version --" -echo " " -echo "################# cat /proc/version #################" -echo " " -cat /proc/version 2>/dev/null -echo " " -echo "################# uname -a #################" -echo " " -uname -a -echo " " -echo "################# cat /etc/issue #################" -echo " " -cat /etc/issue 2>/dev/null -echo " " -echo "run columnstore_os_check.sh" -echo " " -echo "################# /bin/columnstore_os_check.sh #################" -echo " " -columnstore_os_check.sh 2>/dev/null - -echo " " -echo "-- Server Uptime --" -echo " " -echo "################# uptime #################" -echo " " -uptime - -echo " " -echo "-- Server cpu-info --" -echo " " -echo "################# cat /proc/cpuinfo #################" -echo " " -cat /proc/cpuinfo 2>/dev/null - -echo " " -echo "-- Server memory-info --" -echo " " -echo "################# cat /proc/meminfo #################" -echo " " -$cat /proc/meminfo 2>/dev/null - -echo " " -echo "-- Server mounts --" -echo " " -echo "################# cat /proc/mounts #################" -echo " " -cat /proc/mounts 2>/dev/null - -echo " " -echo "-- Server Ethernet Configuration --" -echo " " -echo "################# ifconfig -a #################" -echo " " -ifconfig -a 2>/dev/null - -} >> $OUT_FILE - -exit 0 diff --git a/oamapps/columnstoreSupport/logReport.sh b/oamapps/columnstoreSupport/logReport.sh deleted file mode 100755 index 35b3e9a5f..000000000 --- a/oamapps/columnstoreSupport/logReport.sh +++ /dev/null @@ -1,52 +0,0 @@ -#! /bin/sh -# -# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - OUT_FILE=$2 -else - OUT_FILE=${MODULE}_logReport.txt -fi - -#get temp directory -tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir` - -rm -f ${tmpDir}/${MODULE}_logReport.tar.gz -tar -zcf ${tmpDir}/${MODULE}_logReport.tar.gz /var/log/mariadb/columnstore > /dev/null 2>&1 -cp ${tmpDir}/${MODULE}_logReport.tar.gz . -tar -zcf ${MODULE}_mysqllogReport.tar.gz /var/log/mysql/*.err 2>/dev/null - -echo '******************** Log Configuration ********************' >> $OUT_FILE -echo '' >> $OUT_FILE -echo 'MariaDB ColumnStore System Log Configuration Data' >> $OUT_FILE -echo '' >> $OUT_FILE -configFileName=`mcsGetConfig Installation SystemLogConfigFile` -echo 'System Logging Configuration File being used: '${configFileName} >> $OUT_FILE -echo '' >> $OUT_FILE -echo -e 'Module\tConfigured Log Levels' >> $OUT_FILE -echo -e '------\t---------------------------------------' >> $OUT_FILE -moduleConfig='' -if grep -q '/var/log/mariadb/columnstore/crit.log' ${configFileName}; then - moduleConfig=${moduleConfig}' CRITICAL' -fi -if grep -q '/var/log/mariadb/columnstore/err.log' ${configFileName}; then - moduleConfig=${moduleConfig}' ERROR' -fi -if grep -q '/var/log/mariadb/columnstore/warning.log' ${configFileName}; then - moduleConfig=${moduleConfig}' WARNING' -fi -if grep -q '/var/log/mariadb/columnstore/info.log' ${configFileName}; then - moduleConfig=${moduleConfig}' INFO' -fi -if grep -q '/var/log/mariadb/columnstore/debug.log' ${configFileName}; then - moduleConfig=${moduleConfig}' DEBUG' -fi -echo -e ${MODULE}'\t'${moduleConfig} >> $OUT_FILE -exit 0 - diff --git a/oamapps/columnstoreSupport/mcsSupportUtil.cpp b/oamapps/columnstoreSupport/mcsSupportUtil.cpp deleted file mode 100644 index 586a342e5..000000000 --- a/oamapps/columnstoreSupport/mcsSupportUtil.cpp +++ /dev/null @@ -1,621 +0,0 @@ -/* Copyright (C) 2019 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#include "mcsSupportUtil.h" - -using namespace std; -using namespace oam; -using namespace config; - -void getSystemNetworkConfig(FILE* pOutputFile) -{ - Oam oam; - // get and display Module Network Config - SystemModuleTypeConfig systemmoduletypeconfig; - systemmoduletypeconfig.moduletypeconfig.clear(); - - // get max length of a host name for header formatting - - int maxSize = 9; - - try - { - oam.getSystemConfig(systemmoduletypeconfig); - - for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) - { - if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) - // end of list - break; - - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc; - - if (moduleCount > 0) - { - DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) - { - HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - - for (; pt1 != (*pt).hostConfigList.end(); pt1++) - { - if (maxSize < (int)(*pt1).HostName.size()) - maxSize = (*pt1).HostName.size(); - } - } - } - } - } - catch (exception& e) - { - fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what()); - } - - fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "Module Name", "Module Description", "NIC ID", - "Host Name", "IP Address"); - fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "-----------", "-------------------------", "------", - "---------", "---------------"); - - try - { - oam.getSystemConfig(systemmoduletypeconfig); - - for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) - { - if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) - // end of list - break; - - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc; - - if (moduleCount > 0) - { - DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) - { - string modulename = (*pt).DeviceName; - string moduleID = modulename.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE); - string modulenamedesc = moduletypedesc + " #" + moduleID; - - fprintf(pOutputFile, "%-15s%-30s", modulename.c_str(), modulenamedesc.c_str()); - - HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - - for (; pt1 != (*pt).hostConfigList.end(); pt1++) - { - /* MCOL-1607. IPAddr may be a host name here b/c it is read straight - from the config file. */ - string tmphost = getIPAddress(pt1->IPAddr); - string ipAddr; - if (tmphost.empty()) - ipAddr = pt1->IPAddr; - else - ipAddr = tmphost; - string hostname = (*pt1).HostName; - string nicID = oam.itoa((*pt1).NicID); - - if (nicID != "1") - { - fprintf(pOutputFile, "%-45s", ""); - } - fprintf(pOutputFile, "%-13s%-14s%-20s\n", nicID.c_str(), hostname.c_str(), ipAddr.c_str()); - } - } - } - } - } - catch (exception& e) - { - fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what()); - } -} - -void getModuleTypeConfig(FILE* pOutputFile) -{ - Oam oam; - SystemModuleTypeConfig systemmoduletypeconfig; - ModuleTypeConfig moduletypeconfig; - ModuleConfig moduleconfig; - systemmoduletypeconfig.moduletypeconfig.clear(); - - try - { - oam.getSystemConfig(systemmoduletypeconfig); - - fprintf(pOutputFile, "Module Type Configuration\n\n"); - - for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) - { - if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty()) - // end of list - break; - - int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; - - if (moduleCount < 1) - continue; - - string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType; - - fprintf(pOutputFile, "ModuleType '%s' Configuration information\n", moduletype.c_str()); - fprintf(pOutputFile, "ModuleDesc = %s\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc.c_str()); - fprintf(pOutputFile, "ModuleCount = %i\n", moduleCount); - - if (moduleCount > 0) - { - DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); - - for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) - { - string modulename = (*pt).DeviceName; - HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); - - for (; pt1 != (*pt).hostConfigList.end(); pt1++) - { - string ipAddr = (*pt1).IPAddr; - string servername = (*pt1).HostName; - fprintf(pOutputFile, "ModuleHostName and ModuleIPAddr for NIC ID %u on module '%s' = %s , %s\n", - (*pt1).NicID, modulename.c_str(), servername.c_str(), ipAddr.c_str()); - } - } - } - - DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin(); - - for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end(); pt++) - { - if ((*pt).dbrootConfigList.size() > 0) - { - fprintf(pOutputFile, "DBRootIDs assigned to module 'pm%u' = ", (*pt).DeviceID); - DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - - for (; pt1 != (*pt).dbrootConfigList.end();) - { - fprintf(pOutputFile, "%u", *pt1); - pt1++; - - if (pt1 != (*pt).dbrootConfigList.end()) - fprintf(pOutputFile, ", "); - } - } - fprintf(pOutputFile, "\n"); - } - - fprintf(pOutputFile, "ModuleCPUCriticalThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUCriticalThreshold); - fprintf(pOutputFile, "ModuleCPUMajorThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMajorThreshold); - fprintf(pOutputFile, "ModuleCPUMinorThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorThreshold); - fprintf(pOutputFile, "ModuleCPUMinorClearThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorClearThreshold); - fprintf(pOutputFile, "ModuleDiskCriticalThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskCriticalThreshold); - fprintf(pOutputFile, "ModuleDiskMajorThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMajorThreshold); - fprintf(pOutputFile, "ModuleDiskMinorThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMinorThreshold); - fprintf(pOutputFile, "ModuleMemCriticalThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleMemCriticalThreshold); - fprintf(pOutputFile, "ModuleMemMajorThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMajorThreshold); - fprintf(pOutputFile, "ModuleMemMinorThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMinorThreshold); - fprintf(pOutputFile, "ModuleSwapCriticalThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapCriticalThreshold); - fprintf(pOutputFile, "ModuleSwapMajorThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMajorThreshold); - fprintf(pOutputFile, "ModuleSwapMinorThreshold = %u%%\n", - systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMinorThreshold); - - DiskMonitorFileSystems::iterator pt2 = systemmoduletypeconfig.moduletypeconfig[i].FileSystems.begin(); - int id = 1; - - for (; pt2 != systemmoduletypeconfig.moduletypeconfig[i].FileSystems.end(); pt2++) - { - string fs = *pt2; - fprintf(pOutputFile, "ModuleDiskMonitorFileSystem#%i = %s\n", id, fs.c_str()); - ++id; - } - fprintf(pOutputFile, "\n"); - } - } - catch (exception& e) - { - cout << endl << "**** getModuleTypeConfig Failed = " << e.what() << endl; - } -} - -void getStorageConfig(FILE* pOutputFile) -{ - Oam oam; - try - { - systemStorageInfo_t t; - t = oam.getStorageConfig(); - - string cloud; - - try - { - oam.getSystemConfig("Cloud", cloud); - } - catch (...) - { - } - - string::size_type pos = cloud.find("amazon", 0); - - if (pos != string::npos) - cloud = "amazon"; - - fprintf(pOutputFile, "System Storage Configuration\n"); - - fprintf(pOutputFile, "Performance Module (DBRoot) Storage Type = %s\n", boost::get<0>(t).c_str()); - - if (cloud == "amazon") - fprintf(pOutputFile, "User Module Storage Type = %s\n", boost::get<3>(t).c_str()); - - fprintf(pOutputFile, "System Assigned DBRoot Count = %i\n", boost::get<1>(t)); - - DeviceDBRootList moduledbrootlist = boost::get<2>(t); - - typedef std::vector dbrootList; - dbrootList dbrootlist; - - DeviceDBRootList::iterator pt = moduledbrootlist.begin(); - - for (; pt != moduledbrootlist.end(); pt++) - { - fprintf(pOutputFile, "DBRoot IDs assigned to 'pm%u' = ", (*pt).DeviceID); - DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin(); - - for (; pt1 != (*pt).dbrootConfigList.end();) - { - fprintf(pOutputFile, "%u", *pt1); - dbrootlist.push_back(*pt1); - pt1++; - - if (pt1 != (*pt).dbrootConfigList.end()) - fprintf(pOutputFile, ", "); - } - - fprintf(pOutputFile, "\n"); - } - - // get any unassigned DBRoots - /*DBRootConfigList undbrootlist; - - try - { - oam.getUnassignedDbroot(undbrootlist); - } - catch (...) {} - - if ( !undbrootlist.empty() ) - { - fprintf(pOutputFile,"DBRoot IDs unassigned = "); - DBRootConfigList::iterator pt1 = undbrootlist.begin(); - - for ( ; pt1 != undbrootlist.end() ;) - { - fprintf(pOutputFile,"%u",*pt1); - pt1++; - - if (pt1 != undbrootlist.end()) - fprintf(pOutputFile,", "); - } - - fprintf(pOutputFile,"\n"); - }*/ - - fprintf(pOutputFile, "\n"); - - // um volumes - if (cloud == "amazon" && boost::get<3>(t) == "external") - { - ModuleTypeConfig moduletypeconfig; - oam.getSystemConfig("um", moduletypeconfig); - - for (int id = 1; id < moduletypeconfig.ModuleCount + 1; id++) - { - string volumeNameID = "UMVolumeName" + oam.itoa(id); - string volumeName = oam::UnassignedName; - string deviceNameID = "UMVolumeDeviceName" + oam.itoa(id); - string deviceName = oam::UnassignedName; - - try - { - oam.getSystemConfig(volumeNameID, volumeName); - oam.getSystemConfig(deviceNameID, deviceName); - } - catch (...) - { - } - - fprintf(pOutputFile, "Amazon EC2 Volume Name/Device Name for 'um%i': %s, %s", id, volumeName.c_str(), - deviceName.c_str()); - } - } - - // pm volumes - if (cloud == "amazon" && boost::get<0>(t) == "external") - { - fprintf(pOutputFile, "\n"); - - DBRootConfigList dbrootConfigList; - - try - { - oam.getSystemDbrootConfig(dbrootConfigList); - - DBRootConfigList::iterator pt = dbrootConfigList.begin(); - - for (; pt != dbrootConfigList.end(); pt++) - { - string volumeNameID = "PMVolumeName" + oam.itoa(*pt); - string volumeName = oam::UnassignedName; - string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt); - string deviceName = oam::UnassignedName; - - try - { - oam.getSystemConfig(volumeNameID, volumeName); - oam.getSystemConfig(deviceNameID, deviceName); - } - catch (...) - { - continue; - } - } - } - catch (exception& e) - { - cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; - } - - // print un-assigned dbroots - /*DBRootConfigList::iterator pt1 = undbrootlist.begin(); - - for ( ; pt1 != undbrootlist.end() ; pt1++) - { - string volumeNameID = "PMVolumeName" + oam.itoa(*pt1); - string volumeName = oam::UnassignedName; - string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt1); - string deviceName = oam::UnassignedName; - - try - { - oam.getSystemConfig( volumeNameID, volumeName); - oam.getSystemConfig( deviceNameID, deviceName); - } - catch (...) - { - continue; - } - }*/ - } - - string DataRedundancyConfig; - int DataRedundancyCopies; - string DataRedundancyStorageType; - - try - { - oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); - oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); - oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); - } - catch (...) - { - } - - if (DataRedundancyConfig == "y") - { - fprintf(pOutputFile, "\nData Redundant Configuration\n\n"); - fprintf(pOutputFile, "Copies Per DBroot = %i", DataRedundancyCopies); - - oamModuleInfo_t st; - string moduleType; - - try - { - st = oam.getModuleInfo(); - moduleType = boost::get<1>(st); - } - catch (...) - { - } - - if (moduleType != "pm") - return; - - try - { - DBRootConfigList dbrootConfigList; - oam.getSystemDbrootConfig(dbrootConfigList); - - DBRootConfigList::iterator pt = dbrootConfigList.begin(); - - for (; pt != dbrootConfigList.end(); pt++) - { - fprintf(pOutputFile, "DBRoot #%u has copies on PMs = ", *pt); - - string pmList = ""; - - try - { - string errmsg; - // oam.glusterctl(oam::GLUSTER_WHOHAS, oam.itoa(*pt), pmList, errmsg); - } - catch (...) - { - } - - boost::char_separator sep(" "); - boost::tokenizer > tokens(pmList, sep); - - for (boost::tokenizer >::iterator it = tokens.begin(); - it != tokens.end(); ++it) - { - fprintf(pOutputFile, "%s ", (*it).c_str()); - } - - fprintf(pOutputFile, "\n"); - } - - fprintf(pOutputFile, "\n"); - } - catch (exception& e) - { - cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl; - } - } - } - catch (exception& e) - { - cout << endl << "**** getStorageConfig Failed : " << e.what() << endl; - } -} - -void getStorageStatus(FILE* pOutputFile) -{ - Oam oam; - - fprintf(pOutputFile, "System External DBRoot Storage Statuses\n\n"); - fprintf(pOutputFile, "Component Status Last Status Change\n"); - fprintf(pOutputFile, "------------ -------------------------- ------------------------\n"); - - /*try - { - oam.getSystemStatus(systemstatus, false); - - if ( systemstatus.systemdbrootstatus.dbrootstatus.size() == 0 ) - { - fprintf(pOutputFile," No External DBRoot Storage Configured\n\n"); - return; - } - - for ( unsigned int i = 0 ; i < systemstatus.systemdbrootstatus.dbrootstatus.size(); i++) - { - if ( systemstatus.systemdbrootstatus.dbrootstatus[i].Name.empty() ) - // end of list - break; - - int state = systemstatus.systemdbrootstatus.dbrootstatus[i].OpState; - string stime = systemstatus.systemdbrootstatus.dbrootstatus[i].StateChangeDate ; - stime = stime.substr (0, 24); - fprintf(pOutputFile,"DBRoot%s%-29s%-24s\n", - systemstatus.systemdbrootstatus.dbrootstatus[i].Name.c_str(), - oamState[state].c_str(), - stime.c_str()); - } - fprintf(pOutputFile,"\n"); - } - catch (exception& e) - { - cout << endl << "**** getSystemStatus Failed = " << e.what() << endl; - }*/ - - string DataRedundancyConfig; - int DataRedundancyCopies; - string DataRedundancyStorageType; - - try - { - oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig); - oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); - oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); - } - catch (...) - { - } -} - -/******************************************************************** - * - * checkLogStatus - Check for a phrase in a log file and return status - * - ********************************************************************/ -bool checkLogStatus(std::string fileName, std::string phrase) -{ - ifstream file(fileName.c_str()); - - if (!file.is_open()) - { - return false; - } - - string buf; - - while (getline(file, buf)) - { - string::size_type pos = buf.find(phrase, 0); - - if (pos != string::npos) - // found phrase - return true; - } - - if (file.bad()) - { - return false; - } - - file.close(); - return false; -} - -/****************************************************************************************** - * @brief Get Network IP Address for Host Name - * - * purpose: Get Network IP Address for Host Name - * - ******************************************************************************************/ -string getIPAddress(string hostName) -{ - static uint32_t my_bind_addr; - struct hostent* ent; - string IPAddr = ""; - Oam oam; - - ent = gethostbyname(hostName.c_str()); - - if (ent != 0) - { - my_bind_addr = (uint32_t)((in_addr*)ent->h_addr_list[0])->s_addr; - - uint8_t split[4]; - uint32_t ip = my_bind_addr; - split[0] = (ip & 0xff000000) >> 24; - split[1] = (ip & 0x00ff0000) >> 16; - split[2] = (ip & 0x0000ff00) >> 8; - split[3] = (ip & 0x000000ff); - - IPAddr = - oam.itoa(split[3]) + "." + oam.itoa(split[2]) + "." + oam.itoa(split[1]) + "." + oam.itoa(split[0]); - } - - return IPAddr; -} diff --git a/oamapps/columnstoreSupport/mcsSupportUtil.h b/oamapps/columnstoreSupport/mcsSupportUtil.h deleted file mode 100644 index dda881255..000000000 --- a/oamapps/columnstoreSupport/mcsSupportUtil.h +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright (C) 2020 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include "configcpp.h" -#include "liboamcpp.h" - -void getSystemNetworkConfig(FILE* pOutputFile); -void getModuleTypeConfig(FILE* pOutputFile); -void getStorageConfig(FILE* pOutputFile); -void getStorageStatus(FILE* pOutputFile); -bool checkLogStatus(std::string filename, std::string phase); -std::string getIPAddress(std::string hostName); diff --git a/oamapps/columnstoreSupport/resourceReport.sh b/oamapps/columnstoreSupport/resourceReport.sh deleted file mode 100755 index b0a092563..000000000 --- a/oamapps/columnstoreSupport/resourceReport.sh +++ /dev/null @@ -1,66 +0,0 @@ -#! /bin/sh -# -# $Id: resourceReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - OUT_FILE=$2 -else - OUT_FILE=${MODULE}_logReport.txt -fi - -{ -echo " " -echo "******************** Resource Usage Report for ${MODULE} ********************" -echo " " - -echo " " -echo "-- Shared Memory --" -echo " " -echo "################# ipcs -l #################" -echo " " -ipcs -l - -echo "################# clearShm -n #################" -echo " " -clearShm -n - -echo " " -echo "-- Disk Usage --" -echo " " -echo "################# df -k #################" -echo " " -df -k - -echo " " -echo "-- Disk BRM Data files --" -echo " " -ls -l /var/lib/columnstore/data1/systemFiles/dbrm 2> /dev/null -ls -l /var/lib/columnstore/dbrm 2> /dev/null - -echo "################# cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current #################" -echo " " -cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current 2> /dev/null - -echo " " -echo "-- View Table Locks --" -echo " " -echo "################# cat bin/viewtablelock #################" -echo " " -viewtablelock 2> /dev/null - -echo " " -echo "-- BRM Extent Map --" -echo " " -echo "################# bin/editem -i #################" -echo " " -editem -i 2>/dev/null - -} >> $OUT_FILE - -exit 0 diff --git a/oamapps/columnstoreSupport/softwareReport.sh b/oamapps/columnstoreSupport/softwareReport.sh deleted file mode 100755 index 24a93274c..000000000 --- a/oamapps/columnstoreSupport/softwareReport.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $ -# -if [ $1 ] ; then - MODULE=$1 -else - MODULE="pm1" -fi - -if [ $2 ] ; then - OUT_FILE=$2 -else - OUT_FILE=${MODULE}_logReport.txt -fi - -{ -echo " " -echo "******************** Software Report for ${MODULE} ********************" -echo " " - -echo " " -echo "-- Columnstore Package Details --" -echo " " -rpm -qi MariaDB-columnstore-engine -echo " " - -} >> $OUT_FILE - -exit 0 diff --git a/oamapps/replayTransactionLog/CMakeLists.txt b/oamapps/replayTransactionLog/CMakeLists.txt deleted file mode 100644 index 678ebba7a..000000000 --- a/oamapps/replayTransactionLog/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -# -# Not used -# - -# original Makefile.am contents follow: - -# Copyright (C) 2014 InfiniDB, Inc. -# -# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free -# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in -# -# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) -# bin_PROGRAMS = ReplayTransactionLog ReplayTransactionLog_SOURCES = replaytransactionlog.cpp -# ReplayTransactionLog_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) ReplayTransactionLog_LDFLAGS = -# @idb_common_ldflags@ @idb_exec_libs@ -lreplaytxnlog $(AM_LDFLAGS) -# -# test: -# -# coverage: -# -# leakcheck: -# -# docs: -# -# bootstrap: install-data-am -# diff --git a/oamapps/replayTransactionLog/replaytransactionlog.cpp b/oamapps/replayTransactionLog/replaytransactionlog.cpp deleted file mode 100644 index 0a662510b..000000000 --- a/oamapps/replayTransactionLog/replaytransactionlog.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// WWW - Add header comment. -#include -#include -#include -#include "liboamcpp.h" - -using namespace std; -using namespace oam; - -#include "replaytxnlog.h" -namespace -{ -void usage(char* prog) -{ - cout << endl; - cout << "Usage: " << prog << " [options]" << endl; - - cout << endl; - cout << "This utility can be used after a backup is restored to report transactions that " << endl; - cout << "occurred after the backup. It begins with the first transaction that was committed " << endl; - cout << "after the backup and reports DDL and DML statements as well as imports." << endl; - cout << endl; - - cout << "Options:" << endl; - /* - cout << "-u Database user id." << endl << endl; - - cout << "-p Password." << endl << endl; - */ - cout << "-d Stop date and time as mm/dd/yy@hh:mm:ss or 'Now'." << endl; - cout << " Only transactions committed before this date and time will be reported." << endl; - cout << " The current date and time will be used if 'Now'." << endl << endl; - - /* - cout << "-i Ignore bulk load log entries." << endl; - cout << " The program will pause and prompt at bulk load entries by default." << endl << - endl; - - cout << "-e Report mode. The sql statements will be displayed to the console only. No" << - endl; cout << " transactions will be processed. The user and password will be ignored." << - endl << endl; - */ - - cout << "-h Display this help." << endl << endl; -} - -bool isRunningOnPm() -{ - Oam oam; - oamModuleInfo_t t; - string moduleType; - int installType = -1; - - char* csc_ident = getenv("CALPONT_CSC_IDENT"); - - if (csc_ident == 0 || *csc_ident == 0) - { - // get local module info valdiate running on a pm - try - { - t = oam.getModuleInfo(); - moduleType = boost::get<1>(t); - installType = boost::get<5>(t); - } - catch (exception& e) - { - moduleType = "pm"; - } - } - else - moduleType = csc_ident; - - if (installType != oam::INSTALL_COMBINE_DM_UM_PM) - { - if (moduleType != "pm") - { - cerr << "Exiting, ReplayTransactionLog can only be run on a performance module (pm)" << endl; - return false; - } - } - - return true; -} -} // namespace - -int main(int argc, char** argv) -{ - string user; - string password; - string stopDate; - bool ignoreBulk = false; - bool reportMode = false; - char c; - - // Invokes member function `int operator ()(void);' - while ((c = getopt(argc, argv, "u:p:d:ihe")) != -1) - { - switch (c) - { - /* - case 'u': - user = optarg; - break; - case 'p': - password = optarg; - break; - */ - case 'd': stopDate = optarg; break; - - /* - case 'i': - ignoreBulk = true; - break; - case 'e': - reportMode = true; - break; - */ - case 'h': - usage(argv[0]); - return 0; - break; - - default: - usage(argv[0]); - return 1; - break; - } - } - - if (!isRunningOnPm()) - { - return 0; - } - - ReplayTxnLog replayTxnLog(user, password, stopDate, ignoreBulk, reportMode); - replayTxnLog.process(); - - return 0; -} diff --git a/oamapps/replayTransactionLog/tdriver.cpp b/oamapps/replayTransactionLog/tdriver.cpp deleted file mode 100644 index 809786d7d..000000000 --- a/oamapps/replayTransactionLog/tdriver.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * wweeks@calpont.com * - * * - ***************************************************************************/ - -using namespace std; -#include -#include -#include -#include -#include "sessionmanager.h" - -#include - -using namespace execplan; - -int maxNewTxns = 1000; -int maxTxns = 1000; - -class ExecPlanTest : public CppUnit::TestFixture -{ - CPPUNIT_TEST_SUITE(ExecPlanTest); - - CPPUNIT_TEST_SUITE_END(); - - private: - public: - void setUp() - { - } - - void tearDown() - { - } -}; // test suite - -CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); - -#include -#include - -int main(int argc, char* argv[]) -{ - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest(registry.makeTest()); - bool wasSuccessful = runner.run("", false); - return (wasSuccessful ? 0 : 1); -} diff --git a/oamapps/sessionWalker/CMakeLists.txt b/oamapps/sessionWalker/CMakeLists.txt deleted file mode 100644 index fe6fe4fea..000000000 --- a/oamapps/sessionWalker/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -# -# Not used -# - -# original Makefile.am contents follow: - -# Copyright (C) 2014 InfiniDB, Inc. -# -# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free -# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in -# -# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) -# bin_PROGRAMS = sessionWalker sessionWalker_SOURCES = sessionwalker.cpp sessionWalker_CPPFLAGS = @idb_common_includes@ -# $(AM_CPPFLAGS) sessionWalker_LDFLAGS = @idb_common_ldflags@ @idb_common_libs@ @idb_write_libs@ @netsnmp_libs@ -# $(AM_LDFLAGS) -# -# test: -# -# coverage: -# -# leakcheck: -# -# docs: -# -# bootstrap: install-data-am -# diff --git a/oamapps/sessionWalker/sessionwalker.cpp b/oamapps/sessionWalker/sessionwalker.cpp deleted file mode 100644 index 06cf08bd0..000000000 --- a/oamapps/sessionWalker/sessionwalker.cpp +++ /dev/null @@ -1,135 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * $Id: sessionwalker.cpp 3072 2013-04-04 19:04:45Z rdempsey $ - * - * jrodriguez@calpont.com - * * - ***************************************************************************/ - -#include -using namespace std; - -#include "sessionmonitor.h" -using namespace execplan; - -#include "vendordmlstatement.h" -#include "calpontdmlpackage.h" -#include "calpontdmlfactory.h" -using namespace dmlpackage; - -#include "bytestream.h" -#include "messagequeue.h" -using namespace messageqcpp; - -namespace -{ -void usage() -{ - cout << "sessionwalker [-d|-h]" << endl - << " -r rollback all transactions found" << endl - << " -h display this help" << endl; -} - -void rollback(const SessionMonitor::MonSIDTIDEntry& txn) -{ - VendorDMLStatement dmlStmt("ROLLBACK;", txn.sessionid); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - - if (pDMLPackage == 0) - { - return; - } - - ByteStream bytestream; - pDMLPackage->write(bytestream); - delete pDMLPackage; - MessageQueueClient mq("DMLProc"); - - try - { - cout << "sending ROLLBACK for sessionID " << txn.sessionid << endl; - mq.write(bytestream); - bytestream = mq.read(); - } - catch (...) - { - } -} - -} // namespace - -int main(int argc, char** argv) -{ - bool rflg = false; - opterr = 0; - int c; - - while ((c = getopt(argc, argv, "rh")) != EOF) - switch (c) - { - case 'r': rflg = true; break; - - case 'h': - usage(); - return 0; - break; - - default: - usage(); - return 1; - break; - } - - vector toTxns; - SessionMonitor* monitor = new SessionMonitor(); - - toTxns.clear(); - toTxns = monitor->timedOutTxns(); // get timed out txns - - vector::iterator iter = toTxns.begin(); - vector::iterator end = toTxns.end(); - - vector tmp; - - while (iter != end) - { - if ((*iter)->sessionid > 0) - tmp.push_back(*iter); - - ++iter; - } - - toTxns.swap(tmp); - - cout << toTxns.size() << " timed out transactions." << endl; - - for (unsigned idx = 0; idx < toTxns.size(); idx++) - { - monitor->printTxns(*toTxns[idx]); - - if (rflg) - { - rollback(*toTxns[idx]); - } - } - - delete monitor; - - return 0; -} diff --git a/oamapps/sessionWalker/tdriver.cpp b/oamapps/sessionWalker/tdriver.cpp deleted file mode 100644 index 29e1771fd..000000000 --- a/oamapps/sessionWalker/tdriver.cpp +++ /dev/null @@ -1,189 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -using namespace std; -#include -#include -#include -#include -#include "sessionmonitor.h" -#include "sessionmanager.h" - -#include - -using namespace execplan; - -int maxNewTxns = 1000; -int maxTxns = 1000; - -class ExecPlanTest : public CppUnit::TestFixture -{ - CPPUNIT_TEST_SUITE(ExecPlanTest); - - CPPUNIT_TEST(MonitorTestPlan_1); - CPPUNIT_TEST_SUITE_END(); - - private: - public: - void setUp() - { - } - - void tearDown() - { - } - - int verifyLen; - SessionManager* manager; - SessionManager::TxnID managerTxns[1000]; - - int createTxns(const int& start, const int& end) - { - int first = start; - int last = end; - int newTxns = 0; - - verifyLen = manager->verifySize(); - - for (int idx = first; idx < last && verifyLen < maxNewTxns; idx++) - { - managerTxns[idx] = manager->newTxnID((uint32_t)idx + 1000); - CPPUNIT_ASSERT(managerTxns[idx].id > 0); - CPPUNIT_ASSERT(managerTxns[idx].valid == true); - verifyLen = manager->verifySize(); - CPPUNIT_ASSERT(verifyLen > 0); - newTxns++; - } - - CPPUNIT_ASSERT(newTxns == last - first); - return newTxns; - } - - int closeTxns(const int& start, const int& end) - { - int first = start; - int last = end; - int totalClosed = 0; - - for (int idx = first; idx < last; idx++) - { - try - { - SessionManager::TxnID tmp = manager->getTxnID(idx + 1000); - - if (tmp.valid == true) - { - manager->committed(tmp); - CPPUNIT_ASSERT(tmp.valid == false); - totalClosed++; - } - } - catch (exception& e) - { - cerr << e.what() << endl; - continue; - } - } - - return totalClosed; - - } // closeTxns - - void MonitorTestPlan_1() - { - int currStartTxn = 0; - int currEndTxn = 5; - int txnCntIncr = 5; - const int sleepTime = 1; - const int iterMax = 1; - vector toTxns; - - manager = new SessionManager(); - // CPPUNIT_ASSERT(manager->verifySize()==0); - - SessionMonitor* monitor = NULL; - - for (int jdx = 0; jdx < iterMax; jdx++) - { - // store the current state of the SessionManager - monitor = new SessionMonitor(); - monitor->AgeLimit(sleepTime); - delete monitor; - int idx = 0; - int grpStart = currStartTxn; - - for (idx = 0; idx < 3; idx++) - { - createTxns(currStartTxn, currEndTxn); - // CPPUNIT_ASSERT(manager->verifySize()==(idx+1)*txnCntIncr); - - currStartTxn += txnCntIncr; - currEndTxn += txnCntIncr; - sleep(sleepTime + 1); // make sessions time out - - monitor = new SessionMonitor(); // read Monitor data - monitor->AgeLimit(sleepTime); - toTxns.clear(); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == (uint32_t)txnCntIncr * idx); - - delete monitor; - } - - int grpEnd = currEndTxn; - monitor = new SessionMonitor(); - monitor->AgeLimit(sleepTime); - closeTxns(grpStart, grpEnd); // close this iteration of txns - // CPPUNIT_ASSERT(manager->verifySize()==0); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == 0); - - delete monitor; - } - - monitor = new SessionMonitor(); // readload Monitor data - monitor->AgeLimit(sleepTime - 1); - - toTxns.clear(); - toTxns = monitor->timedOutTxns(); // get timed out txns - CPPUNIT_ASSERT(toTxns.size() == 0); - delete monitor; - - // CPPUNIT_ASSERT(manager->verifySize()==0); - delete manager; - } - -}; // test suite - -CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest); - -#include -#include - -int main(int argc, char* argv[]) -{ - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest(registry.makeTest()); - bool wasSuccessful = runner.run("", false); - return (wasSuccessful ? 0 : 1); -} diff --git a/primitives/blockcache/CMakeLists.txt b/primitives/blockcache/CMakeLists.txt index 9ca0c950f..60deb6c52 100644 --- a/primitives/blockcache/CMakeLists.txt +++ b/primitives/blockcache/CMakeLists.txt @@ -14,4 +14,4 @@ set(dbbc_STAT_SRCS fsutils.cpp ) columnstore_static_library(dbbc ${dbbc_STAT_SRCS}) -columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp) +columnstore_link(dbbc loggingcpp) diff --git a/primitives/linux-port/CMakeLists.txt b/primitives/linux-port/CMakeLists.txt index 1a0032707..254a2d786 100644 --- a/primitives/linux-port/CMakeLists.txt +++ b/primitives/linux-port/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc) set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp) columnstore_static_library(processor ${processor_STAT_SRCS}) -columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp) +columnstore_link(processor loggingcpp) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index dbb82f54e..b8f1df2fb 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -40,7 +40,6 @@ using namespace boost; #include "simd_sse.h" #include "simd_arm.h" #include "utils/common/columnwidth.h" -#include "utils/common/bit_cast.h" #include "exceptclasses.h" diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index cd1232812..017f8bea3 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -3,25 +3,26 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../linux-port) # ########## next target ############### set(PrimProc_SRCS - primproc.cpp + activestatementcounter.cpp batchprimitiveprocessor.cpp bppseeder.cpp bppsendthread.cpp columncommand.cpp command.cpp dictstep.cpp + femsghandler.cpp filtercommand.cpp logger.cpp passthrucommand.cpp primitiveserver.cpp + primproc.cpp pseudocc.cpp + rssmonfcn.cpp rtscommand.cpp - umsocketselector.cpp + samenodepseudosocket.cpp serviceexemgr.cpp sqlfrontsessionthread.cpp - rssmonfcn.cpp - activestatementcounter.cpp - femsghandler.cpp + umsocketselector.cpp ../../utils/common/crashtrace.cpp ) @@ -31,11 +32,11 @@ target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS}) columnstore_link( PrimProc ${ENGINE_LDFLAGS} - ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} threadpool cacheutils dbbc processor loggingcpp + statistics_manager ) diff --git a/utils/messageqcpp/samenodepseudosocket.cpp b/primitives/primproc/samenodepseudosocket.cpp similarity index 100% rename from utils/messageqcpp/samenodepseudosocket.cpp rename to primitives/primproc/samenodepseudosocket.cpp diff --git a/utils/messageqcpp/samenodepseudosocket.h b/primitives/primproc/samenodepseudosocket.h similarity index 100% rename from utils/messageqcpp/samenodepseudosocket.h rename to primitives/primproc/samenodepseudosocket.h diff --git a/primitives/primproc/serviceexemgr.cpp b/primitives/primproc/serviceexemgr.cpp index 2de371f21..6e759cb78 100644 --- a/primitives/primproc/serviceexemgr.cpp +++ b/primitives/primproc/serviceexemgr.cpp @@ -78,7 +78,7 @@ #include "dbrm.h" #include "mariadb_my_sys.h" -#include "statistics.h" +#include "statistics_manager/statistics.h" #include "serviceexemgr.h" #include "sqlfrontsessionthread.h" diff --git a/primitives/primproc/serviceexemgr.h b/primitives/primproc/serviceexemgr.h index a94f88c89..9da16ce0a 100644 --- a/primitives/primproc/serviceexemgr.h +++ b/primitives/primproc/serviceexemgr.h @@ -58,7 +58,7 @@ #include "dbrm.h" #include "mariadb_my_sys.h" -#include "statistics.h" +#include "statistics_manager/statistics.h" namespace exemgr { @@ -69,7 +69,7 @@ class Opt int m_debug; bool m_e; bool m_fg; - Opt() : m_debug(0), m_e(false), m_fg(false){}; + Opt() : m_debug(0), m_e(false), m_fg(false) {}; Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) { int c; diff --git a/primitives/primproc/sqlfrontsessionthread.h b/primitives/primproc/sqlfrontsessionthread.h index 92464a733..4e07ac3fb 100644 --- a/primitives/primproc/sqlfrontsessionthread.h +++ b/primitives/primproc/sqlfrontsessionthread.h @@ -56,76 +56,77 @@ #include "dbrm.h" #include "mariadb_my_sys.h" -#include "statistics.h" +#include "statistics_manager/statistics.h" #include "serviceexemgr.h" namespace exemgr { - class SQLFrontSessionThread +class SQLFrontSessionThread +{ + public: + SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, + joblist::ResourceManager* rm) + : fIos(ios) + , fEc(ec) + , fRm(rm) + , fStatsRetrieved(false) + , fTeleClient(globServiceExeMgr->getTeleServerParms()) + , fOamCachePtr(oam::OamCache::makeOamCache()) { - public: - SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec, - joblist::ResourceManager* rm) - : fIos(ios) - , fEc(ec) - , fRm(rm) - , fStatsRetrieved(false) - , fTeleClient(globServiceExeMgr->getTeleServerParms()) - , fOamCachePtr(oam::OamCache::makeOamCache()) - { - } + } - private: - messageqcpp::IOSocket fIos; - joblist::DistributedEngineComm* fEc; - joblist::ResourceManager* fRm; - querystats::QueryStats fStats; + private: + messageqcpp::IOSocket fIos; + joblist::DistributedEngineComm* fEc; + joblist::ResourceManager* fRm; + querystats::QueryStats fStats; - // Variables used to store return stats - bool fStatsRetrieved; + // Variables used to store return stats + bool fStatsRetrieved; - querytele::QueryTeleClient fTeleClient; + querytele::QueryTeleClient fTeleClient; - oam::OamCache* fOamCachePtr; // this ptr is copyable... + oam::OamCache* fOamCachePtr; // this ptr is copyable... - //...Reinitialize stats for start of a new query - void initStats(uint32_t sessionId, std::string& sqlText) - { - initMaxMemPct(sessionId); + //...Reinitialize stats for start of a new query + void initStats(uint32_t sessionId, std::string& sqlText) + { + initMaxMemPct(sessionId); - fStats.reset(); - fStats.setStartTime(); - fStats.fSessionID = sessionId; - fStats.fQuery = sqlText; - fStatsRetrieved = false; - } - //...Get % memory usage during latest query for sesssionId. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static uint64_t getMaxMemPct(uint32_t sessionId); - //...Delete sessionMemMap entry for the specified session's memory % use. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void deleteMaxMemPct(uint32_t sessionId); - //...Get and log query stats to specified output stream - const std::string formatQueryStats( - joblist::SJLP& jl, // joblist associated with query - const std::string& label, // header label to print in front of log output - bool includeNewLine, // include line breaks in query stats std::string - bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); - static void incThreadCntPerSession(uint32_t sessionId); - static void decThreadCntPerSession(uint32_t sessionId); - //...Init sessionMemMap entry for specified session to 0 memory %. - //...SessionId >= 0x80000000 is system catalog query we can ignore. - static void initMaxMemPct(uint32_t sessionId); - //... Round off to human readable format (KB, MB, or GB). - const std::string roundBytes(uint64_t value) const; - void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); - void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, - boost::shared_ptr csc); - void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); - void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); - void analyzeTableHandleStats(messageqcpp::ByteStream& bs); - uint64_t roundMB(uint64_t value) const; - public: - void operator()(); - }; -} + fStats.reset(); + fStats.setStartTime(); + fStats.fSessionID = sessionId; + fStats.fQuery = sqlText; + fStatsRetrieved = false; + } + //...Get % memory usage during latest query for sesssionId. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static uint64_t getMaxMemPct(uint32_t sessionId); + //...Delete sessionMemMap entry for the specified session's memory % use. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void deleteMaxMemPct(uint32_t sessionId); + //...Get and log query stats to specified output stream + const std::string formatQueryStats( + joblist::SJLP& jl, // joblist associated with query + const std::string& label, // header label to print in front of log output + bool includeNewLine, // include line breaks in query stats std::string + bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); + static void incThreadCntPerSession(uint32_t sessionId); + static void decThreadCntPerSession(uint32_t sessionId); + //...Init sessionMemMap entry for specified session to 0 memory %. + //...SessionId >= 0x80000000 is system catalog query we can ignore. + static void initMaxMemPct(uint32_t sessionId); + //... Round off to human readable format (KB, MB, or GB). + const std::string roundBytes(uint64_t value) const; + void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); + void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, + boost::shared_ptr csc); + void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); + void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); + void analyzeTableHandleStats(messageqcpp::ByteStream& bs); + uint64_t roundMB(uint64_t value) const; + + public: + void operator()(); +}; +} // namespace exemgr diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index c72171cbd..1b01b403f 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -64,7 +64,7 @@ configure_file( link_directories(${CMAKE_BINARY_DIR}/lib) set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) -columnstore_library(storagemanager SHARED ${storagemanager_SRCS}) +columnstore_library(storagemanager ${storagemanager_SRCS}) add_dependencies(storagemanager marias3 external_boost) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 600b3607e..a0d6971c2 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,14 +1,15 @@ -add_subdirectory(dbbuilder) -add_subdirectory(editem) -add_subdirectory(dbloadxml) -add_subdirectory(getConfig) -add_subdirectory(cplogger) add_subdirectory(clearShm) +add_subdirectory(cleartablelock) +add_subdirectory(configMgt) +add_subdirectory(cplogger) +add_subdirectory(dbbuilder) +add_subdirectory(dbloadxml) +add_subdirectory(ddlcleanup) +add_subdirectory(editem) +add_subdirectory(getConfig) +add_subdirectory(idbmeminfo) +add_subdirectory(passwd) +add_subdirectory(rebuildEM) +add_subdirectory(rgprint) add_subdirectory(setConfig) add_subdirectory(viewtablelock) -add_subdirectory(cleartablelock) -add_subdirectory(ddlcleanup) -add_subdirectory(idbmeminfo) -add_subdirectory(rebuildEM) -add_subdirectory(passwd) -add_subdirectory(configMgt) diff --git a/tools/cleartablelock/CMakeLists.txt b/tools/cleartablelock/CMakeLists.txt index 30dc54edd..01dcc211e 100644 --- a/tools/cleartablelock/CMakeLists.txt +++ b/tools/cleartablelock/CMakeLists.txt @@ -6,4 +6,4 @@ set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp) columnstore_executable(cleartablelock ${cleartablelock_SRCS}) -columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) +columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS}) diff --git a/tools/configMgt/CMakeLists.txt b/tools/configMgt/CMakeLists.txt index 9e6b3f3e9..15d6d7693 100644 --- a/tools/configMgt/CMakeLists.txt +++ b/tools/configMgt/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(autoConfigure_SRCS autoConfigure.cpp) add_executable(autoConfigure ${autoConfigure_SRCS}) -columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) +columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) diff --git a/tools/dbbuilder/CMakeLists.txt b/tools/dbbuilder/CMakeLists.txt index 9d687f912..d6bceb767 100644 --- a/tools/dbbuilder/CMakeLists.txt +++ b/tools/dbbuilder/CMakeLists.txt @@ -6,4 +6,4 @@ set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp) columnstore_executable(dbbuilder ${dbbuilder_SRCS}) -columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) +columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS}) diff --git a/tools/ddlcleanup/CMakeLists.txt b/tools/ddlcleanup/CMakeLists.txt index 9ba55097e..4e6016c04 100644 --- a/tools/ddlcleanup/CMakeLists.txt +++ b/tools/ddlcleanup/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(ddlcleanup_SRCS ddlcleanup.cpp) columnstore_executable(ddlcleanup ${ddlcleanup_SRCS}) -columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil) +columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ddlcleanuputil) diff --git a/tools/editem/CMakeLists.txt b/tools/editem/CMakeLists.txt index b0dc993c6..ba6dde613 100644 --- a/tools/editem/CMakeLists.txt +++ b/tools/editem/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(editem_SRCS editem.cpp) columnstore_executable(editem ${editem_SRCS}) -columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) +columnstore_link(editem ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) diff --git a/tools/rgprint/CMakeLists.txt b/tools/rgprint/CMakeLists.txt index 788079df9..699cef3e9 100644 --- a/tools/rgprint/CMakeLists.txt +++ b/tools/rgprint/CMakeLists.txt @@ -6,4 +6,4 @@ set(rgprint_SRCS rgprint.cpp) columnstore_executable(rgprint ${rgprint_SRCS}) -columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) +columnstore_link(rgprint ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS}) diff --git a/tools/setConfig/CMakeLists.txt b/tools/setConfig/CMakeLists.txt index b43566721..85c16b0a8 100644 --- a/tools/setConfig/CMakeLists.txt +++ b/tools/setConfig/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(setConfig_SRCS main.cpp) columnstore_executable(mcsSetConfig ${setConfig_SRCS}) -columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) +columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 9e7b74f4c..cae710dea 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,26 +1,28 @@ # add_subdirectory(boost_idb) -add_subdirectory(startup) -add_subdirectory(common) -add_subdirectory(configcpp) -add_subdirectory(loggingcpp) -add_subdirectory(messageqcpp) -add_subdirectory(threadpool) -add_subdirectory(rwlock) -add_subdirectory(dataconvert) -add_subdirectory(joiner) -add_subdirectory(rowgroup) -add_subdirectory(cacheutils) -add_subdirectory(funcexp) -add_subdirectory(udfsdk) -add_subdirectory(compress) add_subdirectory(batchloader) -add_subdirectory(ddlcleanup) -add_subdirectory(querystats) -add_subdirectory(windowfunction) -add_subdirectory(idbdatafile) -add_subdirectory(querytele) -add_subdirectory(libmysql_client) -add_subdirectory(regr) +add_subdirectory(cacheutils) add_subdirectory(cloudio) +add_subdirectory(common) +add_subdirectory(compress) +add_subdirectory(configcpp) +add_subdirectory(dataconvert) +add_subdirectory(ddlcleanup) +add_subdirectory(funcexp) +add_subdirectory(idbdatafile) +add_subdirectory(joiner) add_subdirectory(libmarias3) +add_subdirectory(libmysql_client) +add_subdirectory(loggingcpp) +add_subdirectory(mariadb_charset) +add_subdirectory(messageqcpp) add_subdirectory(pron) +add_subdirectory(querystats) +add_subdirectory(querytele) +add_subdirectory(regr) +add_subdirectory(rowgroup) +add_subdirectory(rwlock) +add_subdirectory(startup) +add_subdirectory(statistics_manager) +add_subdirectory(threadpool) +add_subdirectory(udfsdk) +add_subdirectory(windowfunction) diff --git a/utils/common/conststring.h b/utils/basic/conststring.h similarity index 100% rename from utils/common/conststring.h rename to utils/basic/conststring.h diff --git a/utils/batchloader/CMakeLists.txt b/utils/batchloader/CMakeLists.txt index 25f58197e..694e88ef0 100644 --- a/utils/batchloader/CMakeLists.txt +++ b/utils/batchloader/CMakeLists.txt @@ -6,4 +6,4 @@ set(batchloader_LIB_SRCS batchloader.cpp) columnstore_library(batchloader ${batchloader_LIB_SRCS}) -columnstore_link(batchloader ${NETSNMP_LIBRARIES} loggingcpp) +columnstore_link(batchloader oamcpp loggingcpp) diff --git a/utils/cacheutils/CMakeLists.txt b/utils/cacheutils/CMakeLists.txt index 0c9c5d63c..666834835 100644 --- a/utils/cacheutils/CMakeLists.txt +++ b/utils/cacheutils/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(cacheutils_LIB_SRCS cacheutils.cpp) columnstore_library(cacheutils ${cacheutils_LIB_SRCS}) -columnstore_link(cacheutils messageqcpp) +columnstore_link(cacheutils messageqcpp boost_thread) diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt index c5bb954d0..131030ace 100755 --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -14,7 +14,7 @@ columnstore_library(cloudio ${cloudio_LIB_SRCS}) # IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for # idbdatafile, we should be able to reverse the dependency like so: -columnstore_link(cloudio idbdatafile messageqcpp) +columnstore_link(cloudio idbdatafile messageqcpp boost_thread) add_executable(cloudio_component_test component_test.cpp) add_executable(cloudio_end_to_end_test end_to_end_test.cpp) diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index dc19cd77d..d99ee7590 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -9,10 +9,17 @@ set(common_LIB_SRCS MonitorProcMem.cpp nullvaluemanip.cpp threadnaming.cpp - utils_utf8.cpp - statistics.cpp string_prefixes.cpp ) columnstore_library(common ${common_LIB_SRCS}) -columnstore_link(common boost_filesystem configcpp loggingcpp messageqcpp) +columnstore_link( + common + PRIVATE + boost_filesystem + configcpp + loggingcpp + messageqcpp + idbdatafile + mariadb_charset +) diff --git a/utils/common/bit_cast.h b/utils/common/bit_cast.h deleted file mode 100644 index e2a99e21a..000000000 --- a/utils/common/bit_cast.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (C) 2020 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include -#include - -namespace utils -{ -template -std::enable_if_t< - sizeof(To) == sizeof(From) && std::is_trivially_copyable_v && std::is_trivially_copyable_v, To> -// constexpr support needs compiler magic -bitCast(const From& src) noexcept -{ - static_assert(std::is_trivially_constructible_v, - "This implementation additionally requires " - "destination type to be trivially constructible"); - - To dst; - std::memcpy(&dst, &src, sizeof(To)); - return dst; -} -} // namespace utils diff --git a/utils/common/branchpred.h b/utils/common/branchpred.h index b0c4feb08..2553c1390 100644 --- a/utils/common/branchpred.h +++ b/utils/common/branchpred.h @@ -19,12 +19,6 @@ #pragma once -#if !defined(__GNUC__) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) -#ifndef __builtin_expect -#define __builtin_expect(x, expected_value) (x) -#endif -#endif - #ifndef LIKELY #define LIKELY(x) __builtin_expect((x), 1) #define UNLIKELY(x) __builtin_expect((x), 0) diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 0c060ae89..1a9ca6635 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -17,6 +17,9 @@ #pragma once +#include +#include + #include "branchpred.h" namespace utils diff --git a/utils/common/genericparser.h b/utils/common/genericparser.h index d2bce2d26..8f4c67101 100644 --- a/utils/common/genericparser.h +++ b/utils/common/genericparser.h @@ -17,7 +17,7 @@ #pragma once -#include "conststring.h" +#include "basic/conststring.h" namespace genericparser { diff --git a/utils/common/hashfamily.h b/utils/common/hashfamily.h index 4b09e01a8..954779e40 100644 --- a/utils/common/hashfamily.h +++ b/utils/common/hashfamily.h @@ -18,7 +18,7 @@ #pragma once #include "hasher.h" -#include "collation.h" +#include "mariadb_charset/collation.h" namespace utils { diff --git a/utils/common/nullstring.h b/utils/common/nullstring.h index 313a5bcb9..0b4bd62a9 100644 --- a/utils/common/nullstring.h +++ b/utils/common/nullstring.h @@ -25,7 +25,7 @@ #include #include #include "exceptclasses.h" -#include "conststring.h" +#include "basic/conststring.h" #include "mcs_datatype_basic.h" namespace utils diff --git a/utils/common/string_prefixes.cpp b/utils/common/string_prefixes.cpp index 9f9c5c2e7..a8c45c84e 100644 --- a/utils/common/string_prefixes.cpp +++ b/utils/common/string_prefixes.cpp @@ -18,7 +18,7 @@ /* handling of the conversion of string prefixes to int64_t for quick range checking */ -#include "collation.h" +#include "mariadb_charset/collation.h" #include "joblisttypes.h" #include "string_prefixes.h" diff --git a/utils/common/syncstream.h b/utils/common/syncstream.h deleted file mode 100644 index d00b41150..000000000 --- a/utils/common/syncstream.h +++ /dev/null @@ -1,157 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/** @file */ - -/* - * classes isyncstream and osyncstream provide a C++ iostream interface - * for C stdio FILE* streams. The current implementation does not provide - * the necessary methods to support seeking. The I/O buffering of the - * input FILE* is used. The C++ iostream library calls syncbuf::sync() - * for every line, so output buffering is line-by-line. - * */ - -/* -#include "syncstream.h" - -void copyStream(istream& iss, ostream& oss) -{ - string line; - getline(iss, line); - while (iss.good()) - { - oss << line << endl; - getline(iss, line); - } -} - -main() -{ - FILE* ifp; - FILE* ofp; - - ... - - isyncstream iss(ifp); - osyncstream oss(ofp); - - copyStream(iss, oss); - - ... -} -*/ - -#pragma once - -#include -#include - -namespace syncstream -{ -/** A streambuf implementation for C stdio FILE* streams. - * - * Adapted from http://www.drdobbs.com/184401305 - */ -class syncbuf : public std::streambuf -{ - public: - /** ctor */ - syncbuf(FILE* f) : std::streambuf(), fptr(f) - { - } - - protected: - /** Write character in the case of overflow */ - virtual int overflow(int c = EOF) - { - return (c != EOF ? fputc(c, fptr) : EOF); - } - /** Get character in the case of overflow */ - virtual int underflow() - { - int c = getc(fptr); - - if (c != EOF) - ungetc(c, fptr); - - return c; - } - /** Get character in the case of overflow and advance get pointer */ - virtual int uflow() - { - return getc(fptr); - } - /** put character back in the case of backup underflow */ - virtual int pbackfail(int c = EOF) - { - return (c != EOF ? ungetc(c, fptr) : EOF); - } - /** Synchronize stream buffer */ - virtual int sync() - { - return fflush(fptr); - } - - private: - FILE* fptr; -}; - -/** An istream adaptor for input FILE* streams */ -class isyncstream : public std::istream -{ - public: - /** ctor */ - isyncstream() : istream(&buf), buf(0) - { - } - /** ctor */ - isyncstream(FILE* fptr) : istream(&buf), buf(fptr) - { - } - /** const streambuf accessor */ - const syncbuf* rdbuf() const - { - return &buf; - } - - private: - syncbuf buf; -}; - -/** An ostream adaptor for output FILE* streams */ -class osyncstream : public std::ostream -{ - public: - /** ctor */ - osyncstream() : ostream(&buf), buf(0) - { - } - /** ctor */ - osyncstream(FILE* fptr) : ostream(&buf), buf(fptr) - { - } - /** const streambuf accessor */ - const syncbuf* rdbuf() const - { - return &buf; - } - - private: - syncbuf buf; -}; - -} // namespace syncstream diff --git a/utils/common/utils_utf8.h b/utils/common/utils_utf8.h index 352bbb4f7..49f856bb5 100644 --- a/utils/common/utils_utf8.h +++ b/utils/common/utils_utf8.h @@ -20,16 +20,10 @@ #pragma once +#include #include -#if defined(__FreeBSD__) -//#include -#else -#include -#endif #include - #include -#include "liboamcpp.h" // Change the name from utf8. Even change the file name to something resembling char helper namespace utf8 diff --git a/utils/ddlcleanup/CMakeLists.txt b/utils/ddlcleanup/CMakeLists.txt index c813fe428..600e73e84 100644 --- a/utils/ddlcleanup/CMakeLists.txt +++ b/utils/ddlcleanup/CMakeLists.txt @@ -2,4 +2,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(ddlcleanuputil_LIB_SRCS ddlcleanuputil.cpp) columnstore_library(ddlcleanuputil ${ddlcleanuputil_LIB_SRCS}) -columnstore_link(ddlcleanuputil PRIVATE loggingcpp ${NETSNMP_LIBRARIES}) +columnstore_link(ddlcleanuputil PRIVATE loggingcpp) diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index 6f1c758a7..752221fed 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -150,6 +150,7 @@ columnstore_link( pron loggingcpp dataconvert - ${MARIADB_STRING_LIBS} - ${NETSNMP_LIBRARIES} + mariadb_charset ) + +columnstore_link(funcexp PRIVATE ${MARIADB_STRING_LIBS}) diff --git a/utils/funcexp/functor_json.h b/utils/funcexp/functor_json.h index 55c4d1b5f..f6939a4c5 100644 --- a/utils/funcexp/functor_json.h +++ b/utils/funcexp/functor_json.h @@ -7,7 +7,7 @@ #include #include -#include "collation.h" +#include "mariadb_charset/collation.h" #include "functor_bool.h" #include "functor_int.h" #include "functor_str.h" diff --git a/utils/funcexp/jsonhelpers.h b/utils/funcexp/jsonhelpers.h index 2756925b8..8f5fde560 100644 --- a/utils/funcexp/jsonhelpers.h +++ b/utils/funcexp/jsonhelpers.h @@ -10,10 +10,10 @@ #include // #include -#include "collation.h" +#include "mariadb_charset/collation.h" #include "functor_json.h" #include "functor_str.h" -#include "collation.h" +#include "mariadb_charset/collation.h" #include "rowgroup.h" #include "treenode.h" #include "functioncolumn.h" diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index 06a1ebf44..8e8d9ab4d 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -14,4 +14,4 @@ set(idbdatafile_LIB_SRCS ) columnstore_library(idbdatafile ${idbdatafile_LIB_SRCS}) -columnstore_link(idbdatafile PRIVATE ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} boost_filesystem boost_system) +columnstore_link(idbdatafile PRIVATE ${ENGINE_OAM_LIBS} boost_filesystem boost_system compress) diff --git a/utils/joiner/CMakeLists.txt b/utils/joiner/CMakeLists.txt index 0df5c60f7..1d636162c 100644 --- a/utils/joiner/CMakeLists.txt +++ b/utils/joiner/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(joiner_LIB_SRCS tuplejoiner.cpp joinpartition.cpp) columnstore_library(joiner ${joiner_LIB_SRCS}) -columnstore_link(joiner PRIVATE loggingcpp) +columnstore_link(joiner PRIVATE loggingcpp rowgroup datatypes compress) diff --git a/utils/mariadb_charset/CMakeLists.txt b/utils/mariadb_charset/CMakeLists.txt new file mode 100644 index 000000000..9734bf1c5 --- /dev/null +++ b/utils/mariadb_charset/CMakeLists.txt @@ -0,0 +1,7 @@ +include_directories(${ENGINE_COMMON_INCLUDES}) + +# ########## next target ############### + +columnstore_static_library(mariadb_charset charset.cpp) +columnstore_link(mariadb_charset PRIVATE ${MARIADB_STRING_LIBS}) +add_dependencies(mariadb_charset loggingcpp) diff --git a/utils/common/utils_utf8.cpp b/utils/mariadb_charset/charset.cpp similarity index 72% rename from utils/common/utils_utf8.cpp rename to utils/mariadb_charset/charset.cpp index f0471f7d3..9a101592b 100644 --- a/utils/common/utils_utf8.cpp +++ b/utils/mariadb_charset/charset.cpp @@ -1,8 +1,9 @@ -/* Copyright (C) 2020 MariaDB Corporation. +/* Copyright (C) 2025 MariaDB Corporation - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -11,11 +12,12 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ - -#include "utils_utf8.h" + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#include "mariadb_charset/collation.h" #include "mariadb_my_sys.h" +#include "mcs_datatype.h" namespace datatypes { @@ -35,4 +37,3 @@ void Charset::setCharset(uint32_t charsetNumber) } } // namespace datatypes - diff --git a/utils/common/collation.h b/utils/mariadb_charset/collation.h similarity index 99% rename from utils/common/collation.h rename to utils/mariadb_charset/collation.h index e5781b8d9..411eca6aa 100644 --- a/utils/common/collation.h +++ b/utils/mariadb_charset/collation.h @@ -35,7 +35,7 @@ #include "mcsconfig.h" #include "exceptclasses.h" -#include "conststring.h" +#include "basic/conststring.h" /* Redefine definitions used by MariaDB m_ctype.h. diff --git a/utils/messageqcpp/CMakeLists.txt b/utils/messageqcpp/CMakeLists.txt index 527e9395a..c41a88168 100644 --- a/utils/messageqcpp/CMakeLists.txt +++ b/utils/messageqcpp/CMakeLists.txt @@ -8,7 +8,6 @@ set(messageqcpp_LIB_SRCS iosocket.cpp messagequeue.cpp messagequeuepool.cpp - samenodepseudosocket.cpp socketparms.cpp ) diff --git a/utils/messageqcpp/messagequeuepool.cpp b/utils/messageqcpp/messagequeuepool.cpp index 25af41a5c..2debf46cd 100644 --- a/utils/messageqcpp/messagequeuepool.cpp +++ b/utils/messageqcpp/messagequeuepool.cpp @@ -25,40 +25,57 @@ #include #include - namespace messageqcpp { using ClientMapType = std::multimap>; -struct LockedClientMap +template +struct Immortal { - LockedClientMap() + template + Immortal(Args&&... args) { + ::new (space) T(std::forward(args)...); } - ~LockedClientMap() + + operator T&() & noexcept { + return reinterpret_cast(space); } - ClientMapType clientMap; - std::mutex queueMutex; + + private: + alignas(T) unsigned char space[sizeof(T)]; }; -static int clientMapNiftyCounter; - -static typename std::aligned_storage::type clientMapBuf; - -auto& lockedMap = reinterpret_cast(clientMapBuf); - - -LockedClientMapInitilizer::LockedClientMapInitilizer () +class LockedClientMap { - if (clientMapNiftyCounter++ == 0) new (&lockedMap) LockedClientMap (); // placement new -} -LockedClientMapInitilizer::~LockedClientMapInitilizer () -{ - if (--clientMapNiftyCounter == 0) (&lockedMap)->~LockedClientMap(); -} + struct KeyToUsePrivateCtor + { + explicit KeyToUsePrivateCtor() = default; + }; + public: + LockedClientMap(const LockedClientMap&) = delete; + LockedClientMap& operator=(const LockedClientMap&) = delete; + ~LockedClientMap() = delete; + + static LockedClientMap& getInstance() + { + static Immortal instance(KeyToUsePrivateCtor{}); + return instance; + } + + LockedClientMap(KeyToUsePrivateCtor) + { + } + + private: + ClientMapType clientMap; + std::mutex queueMutex; + + friend class MessageQueueClientPool; +}; // 300 seconds idle until cleanup #define MAX_IDLE_TIME 300 @@ -70,7 +87,7 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts) MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrIp, uint64_t port) { - auto lock = std::scoped_lock(lockedMap.queueMutex); + auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex); std::ostringstream oss; oss << dnOrIp << "_" << port; @@ -93,14 +110,13 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrI newClientObject->client.reset(new MessageQueueClient(dnOrIp, port)); newClientObject->inUse = true; newClientObject->lastUsed = nowSeconds; - lockedMap.clientMap.emplace(std::move(searchString), std::move(newClientObject)); + LockedClientMap::getInstance().clientMap.emplace(std::move(searchString), std::move(newClientObject)); return newClientObject->client.get(); } MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& module) { - auto lock = std::scoped_lock(lockedMap.queueMutex); - + auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex); MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module); @@ -116,13 +132,11 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& modul clock_gettime(CLOCK_MONOTONIC, &now); uint64_t nowSeconds = TimeSpecToSeconds(&now); - - newClientObject->client.reset(new MessageQueueClient(module)); newClientObject->inUse = true; newClientObject->lastUsed = nowSeconds; auto result = newClientObject->client.get(); - lockedMap.clientMap.emplace(std::move(module), std::move(newClientObject)); + LockedClientMap::getInstance().clientMap.emplace(std::move(module), std::move(newClientObject)); return result; } @@ -133,11 +147,10 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search uint64_t nowSeconds = TimeSpecToSeconds(&now); MessageQueueClient* returnClient = NULL; - auto it = lockedMap.clientMap.begin(); - + auto it = LockedClientMap::getInstance().clientMap.begin(); // Scan pool - while (it != lockedMap.clientMap.end()) + while (it != LockedClientMap::getInstance().clientMap.end()) { ClientObject* clientObject = it->second.get(); uint64_t elapsedTime = nowSeconds - clientObject->lastUsed; @@ -149,7 +162,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search // Do this so we don't invalidate current interator auto toDelete = it; it++; - lockedMap.clientMap.erase(toDelete); + LockedClientMap::getInstance().clientMap.erase(toDelete); continue; } @@ -163,7 +176,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search // Do this so we don't invalidate current interator auto toDelete = it; it++; - lockedMap.clientMap.erase(toDelete); + LockedClientMap::getInstance().clientMap.erase(toDelete); continue; } } @@ -193,10 +206,10 @@ void MessageQueueClientPool::releaseInstance(MessageQueueClient* client) if (client == NULL) return; - auto lock = std::scoped_lock(lockedMap.queueMutex); - auto it = lockedMap.clientMap.begin(); + auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex); + auto it = LockedClientMap::getInstance().clientMap.begin(); - while (it != lockedMap.clientMap.end()) + while (it != LockedClientMap::getInstance().clientMap.end()) { if (it->second->client.get() == client) { @@ -221,15 +234,14 @@ void MessageQueueClientPool::deleteInstance(MessageQueueClient* client) if (client == NULL) return; + auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex); + auto it = LockedClientMap::getInstance().clientMap.begin(); - auto lock = std::scoped_lock(lockedMap.queueMutex); - auto it = lockedMap.clientMap.begin(); - - while (it != lockedMap.clientMap.end()) + while (it != LockedClientMap::getInstance().clientMap.end()) { if (it->second->client.get() == client) { - lockedMap.clientMap.erase(it); + LockedClientMap::getInstance().clientMap.erase(it); return; } diff --git a/utils/messageqcpp/messagequeuepool.h b/utils/messageqcpp/messagequeuepool.h index 17899b76a..6af5e1cc6 100644 --- a/utils/messageqcpp/messagequeuepool.h +++ b/utils/messageqcpp/messagequeuepool.h @@ -26,12 +26,6 @@ namespace messageqcpp { - -static struct LockedClientMapInitilizer { - LockedClientMapInitilizer (); - ~LockedClientMapInitilizer (); -} clientMapInitilizer; // static initializer for every translation unit - struct ClientObject { std::unique_ptr client; @@ -49,8 +43,8 @@ class MessageQueueClientPool static MessageQueueClient* findInPool(const std::string& search); private: - MessageQueueClientPool(){}; - ~MessageQueueClientPool(){}; + MessageQueueClientPool() {}; + ~MessageQueueClientPool() {}; }; } // namespace messageqcpp diff --git a/utils/querystats/CMakeLists.txt b/utils/querystats/CMakeLists.txt index 0d91ead9e..ff6b13f91 100644 --- a/utils/querystats/CMakeLists.txt +++ b/utils/querystats/CMakeLists.txt @@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(querystats_LIB_SRCS querystats.cpp) columnstore_library(querystats ${querystats_LIB_SRCS}) -columnstore_link(querystats PRIVATE loggingcpp) +columnstore_link(querystats PRIVATE loggingcpp messageqcpp) diff --git a/utils/querytele/CMakeLists.txt b/utils/querytele/CMakeLists.txt index f5e58ae60..9fd9af6a9 100644 --- a/utils/querytele/CMakeLists.txt +++ b/utils/querytele/CMakeLists.txt @@ -5,7 +5,7 @@ set(querytele_LIB_SRCS querytele.cpp queryteleclient.cpp querytele_constants.cpp ) columnstore_library(querytele ${querytele_LIB_SRCS}) -columnstore_link(querytele ${THRIFT_LIBRARY}) +columnstore_link(querytele ${THRIFT_LIBRARY} boost_thread) target_include_directories(querytele PRIVATE ${THRIFT_INCLUDE_DIRS}) -add_dependencies(querytele external_boost external_thrift) +add_dependencies(querytele external_thrift) diff --git a/utils/regr/CMakeLists.txt b/utils/regr/CMakeLists.txt index f2f378f34..e4ce77d95 100755 --- a/utils/regr/CMakeLists.txt +++ b/utils/regr/CMakeLists.txt @@ -21,11 +21,12 @@ set(regr_LIB_SRCS add_definitions(-DMYSQL_DYNAMIC_PLUGIN) columnstore_library(regr ${regr_LIB_SRCS}) -columnstore_link(regr PRIVATE loggingcpp) +columnstore_link(regr PRIVATE loggingcpp messageqcpp) set(regr_mysql_LIB_SRCS regrmysql.cpp modamysql.cpp) +# Do anyone use it? columnstore_mysql_plugin_library(regr_mysql SHARED ${regr_mysql_LIB_SRCS}) -add_dependencies(regr_mysql external_boost) +add_dependencies(regr_mysql external_boost GenError) # for "idb_mysql.h" that uses generated mysqld_error.h set_target_properties(regr_mysql PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../) diff --git a/utils/regr/moda.h b/utils/regr/moda.h index aaa7fc771..5d3bb244e 100644 --- a/utils/regr/moda.h +++ b/utils/regr/moda.h @@ -45,7 +45,7 @@ #include "calpontsystemcatalog.h" #include "windowfunctioncolumn.h" #include "hasher.h" -#include "collation.h" +#include "mariadb_charset/collation.h" #define EXPORT @@ -277,7 +277,7 @@ class Moda_impl_T : public mcsv1_UDAF { public: // Defaults OK - Moda_impl_T() : cs(8){}; + Moda_impl_T() : cs(8) {}; ~Moda_impl_T() override = default; mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; @@ -305,7 +305,7 @@ class moda : public mcsv1_UDAF { public: // Defaults OK - moda() : mcsv1_UDAF(){}; + moda() : mcsv1_UDAF() {}; ~moda() override = default; mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index 2ba4c2d46..b22e93b43 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -5,5 +5,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rowgroup_LIB_SRCS rowaggregation.cpp rowgroup.cpp rowstorage.cpp) columnstore_library(rowgroup ${rowgroup_LIB_SRCS}) -columnstore_link(rowgroup PRIVATE ${NETSNMP_LIBRARIES} funcexp loggingcpp) +columnstore_link(rowgroup PRIVATE funcexp loggingcpp compress) add_dependencies(rowgroup external_boost) diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 50baf681c..5917885b3 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -53,7 +53,7 @@ #include "branchpred.h" #include "datatypes/mcs_int128.h" -#include "collation.h" +#include "mariadb_charset/collation.h" #include "common/hashfamily.h" #include "buffertypes.h" diff --git a/utils/statistics_manager/CMakeLists.txt b/utils/statistics_manager/CMakeLists.txt new file mode 100644 index 000000000..8bcdfd8ca --- /dev/null +++ b/utils/statistics_manager/CMakeLists.txt @@ -0,0 +1,6 @@ +include_directories(${ENGINE_COMMON_INCLUDES}) + +# ########## next target ############### + +columnstore_static_library(statistics_manager statistics.cpp) +columnstore_link(statistics_manager PRIVATE rowgroup) diff --git a/utils/common/statistics.cpp b/utils/statistics_manager/statistics.cpp similarity index 99% rename from utils/common/statistics.cpp rename to utils/statistics_manager/statistics.cpp index ea0dd29ff..ff68eb381 100644 --- a/utils/common/statistics.cpp +++ b/utils/statistics_manager/statistics.cpp @@ -116,9 +116,8 @@ void StatisticsManager::analyzeSample(bool traceOn) // MCV statistics. std::vector> mcvList(columnMCV.begin(), columnMCV.end()); std::sort(mcvList.begin(), mcvList.end(), - [](const std::pair& a, const std::pair& b) { - return a.second > b.second; - }); + [](const std::pair& a, const std::pair& b) + { return a.second > b.second; }); // 200 buckets as Microsoft does. const auto mcvSize = std::min(columnMCV.size(), static_cast(200)); diff --git a/utils/common/statistics.h b/utils/statistics_manager/statistics.h similarity index 100% rename from utils/common/statistics.h rename to utils/statistics_manager/statistics.h diff --git a/utils/threadpool/CMakeLists.txt b/utils/threadpool/CMakeLists.txt index 1502bb4b3..496fba57a 100644 --- a/utils/threadpool/CMakeLists.txt +++ b/utils/threadpool/CMakeLists.txt @@ -2,6 +2,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp fair_threadpool.cpp) columnstore_library(threadpool ${threadpool_LIB_SRCS}) -columnstore_link(threadpool PRIVATE boost_chrono loggingcpp) - -add_dependencies(threadpool external_boost) +columnstore_link(threadpool PRIVATE boost_chrono boost_thread loggingcpp messageqcpp) diff --git a/utils/udfsdk/CMakeLists.txt b/utils/udfsdk/CMakeLists.txt index e13f07ab2..25e5020ff 100755 --- a/utils/udfsdk/CMakeLists.txt +++ b/utils/udfsdk/CMakeLists.txt @@ -11,9 +11,11 @@ set(udfsdk_LIB_SRCS ) columnstore_library(udfsdk ${udfsdk_LIB_SRCS}) -columnstore_link(udfsdk PRIVATE loggingcpp) +columnstore_link(udfsdk PRIVATE loggingcpp messageqcpp) # Do anyone use it? add_definitions(-DMYSQL_DYNAMIC_PLUGIN) set(udf_mysql_LIB_SRCS udfmysql.cpp) columnstore_mysql_plugin_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS}) + +add_dependencies(udf_mysql GenError) # for "idb_mysql.h" that uses generated mysqld_error.h diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 4468c1c05..8b62dc2b2 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -34,7 +34,7 @@ set(brm_LIB_SRCS ) columnstore_library(brm ${brm_LIB_SRCS}) -columnstore_link(brm loggingcpp datatypes) +columnstore_link(brm loggingcpp datatypes oamcpp boost_thread messageqcpp) # ########## next target ############### @@ -62,53 +62,41 @@ columnstore_link(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS set(reset_locks_SRCS reset_locks.cpp) columnstore_executable(reset_locks ${reset_locks_SRCS}) -columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) # ########## next target ############### set(rollback_SRCS rollback.cpp) columnstore_executable(rollback ${rollback_SRCS}) -columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) # ########## next target ############### set(save_brm_SRCS save_brm.cpp) columnstore_executable(save_brm ${save_brm_SRCS}) -columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) # ########## next target ############### set(load_brm_SRCS load_brm.cpp) columnstore_executable(load_brm ${load_brm_SRCS}) -columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) columnstore_executable(mcs-load-em load_em.cpp) -columnstore_link( - mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} -) +columnstore_link(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) columnstore_executable(mcs-load-brm-from-file load_brm_from_file.cpp) columnstore_link( - mcs-load-brm-from-file - ${ENGINE_LDFLAGS} - ${MARIADB_CLIENT_LIBS} - ${ENGINE_OAM_LIBS} - ${ENGINE_EXEC_LIBS} - ${NETSNMP_LIBRARIES} + mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} boost_program_options ) columnstore_executable(mcs-shmem-locks shmem_locks.cpp) columnstore_link( - mcs-shmem-locks - ${ENGINE_LDFLAGS} - ${MARIADB_CLIENT_LIBS} - ${ENGINE_OAM_LIBS} - ${ENGINE_EXEC_LIBS} - ${NETSNMP_LIBRARIES} + mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} boost_program_options ) diff --git a/writeengine/bulk/CMakeLists.txt b/writeengine/bulk/CMakeLists.txt index 0bb90a586..229fa19c2 100644 --- a/writeengine/bulk/CMakeLists.txt +++ b/writeengine/bulk/CMakeLists.txt @@ -29,7 +29,7 @@ set(we_bulk_STAT_SRCS add_definitions(-D_FILE_OFFSET_BITS=64) columnstore_static_library(we_bulk ${we_bulk_STAT_SRCS}) -columnstore_link(we_bulk ${NETSNMP_LIBRARIES} loggingcpp boost_program_options) +columnstore_link(we_bulk loggingcpp boost_program_options) remove_definitions(-D_FILE_OFFSET_BITS=64) @@ -38,13 +38,13 @@ remove_definitions(-D_FILE_OFFSET_BITS=64) set(cpimport.bin_SRCS cpimport.cpp) columnstore_executable(cpimport.bin ${cpimport.bin_SRCS}) -add_dependencies(cpimport.bin marias3) + columnstore_link( cpimport.bin ${ENGINE_LDFLAGS} - ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ${S3API_DEPS} we_bulk we_xml + marias3 ) diff --git a/writeengine/client/CMakeLists.txt b/writeengine/client/CMakeLists.txt index 52633ceca..910680f2c 100644 --- a/writeengine/client/CMakeLists.txt +++ b/writeengine/client/CMakeLists.txt @@ -5,7 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(writeengineclient_LIB_SRCS we_clients.cpp we_ddlcommandclient.cpp we_dmlcommandclient.cpp) columnstore_library(writeengineclient ${writeengineclient_LIB_SRCS}) - -add_dependencies(writeengineclient loggingcpp) - -columnstore_link(writeengineclient ${NETSNMP_LIBRARIES}) +columnstore_link(writeengineclient boost_thread oamcpp messageqcpp loggingcpp) diff --git a/writeengine/redistribute/CMakeLists.txt b/writeengine/redistribute/CMakeLists.txt index 29b8b94ee..5ca49b172 100644 --- a/writeengine/redistribute/CMakeLists.txt +++ b/writeengine/redistribute/CMakeLists.txt @@ -7,9 +7,6 @@ set(writeengineredistribute_LIB_SRCS we_redistribute.cpp we_redistributecontrol. ) columnstore_library(writeengineredistribute ${writeengineredistribute_LIB_SRCS}) - -add_dependencies(writeengineredistribute loggingcpp) - -columnstore_link(writeengineredistribute ${NETSNMP_LIBRARIES}) +columnstore_link(writeengineredistribute loggingcpp oamcpp boost_thread messageqcpp) target_compile_definitions(writeengineredistribute PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index 9ebc5cd72..0843f246f 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -18,9 +18,4 @@ set(WriteEngineServer_SRCS ) columnstore_executable(WriteEngineServer ${WriteEngineServer_SRCS}) - -add_dependencies(WriteEngineServer loggingcpp) - -columnstore_link( - WriteEngineServer ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} threadpool writeengineredistribute -) +columnstore_link(WriteEngineServer ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool writeengineredistribute loggingcpp) diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 7772351ab..158011e6e 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -40,7 +40,7 @@ #include "IDBDataFile.h" #include "IDBPolicy.h" #include "nullstring.h" -#include "collation.h" // For CHARSET_INFO struct +#include "mariadb_charset/collation.h" // For CHARSET_INFO struct #undef EXPORT #undef DELETE @@ -408,8 +408,8 @@ struct JobColumn /** @brief Job Column Structure */ int compressionType; /** @brief compression type */ bool autoIncFlag; /** @brief auto increment flag */ DctnryStruct dctnry; /** @brief dictionary structure */ - int128_t fMinIntSat; /** @brief For integer type, the min saturation value */ - uint128_t fMaxIntSat; /** @brief For integer type, the max saturation value */ + int128_t fMinIntSat; /** @brief For integer type, the min saturation value */ + uint128_t fMaxIntSat; /** @brief For integer type, the max saturation value */ double fMinDblSat; /** @brief for float/double, the min saturation value */ double fMaxDblSat; /** @brief for float/double, the max saturation value */ bool fWithDefault; /** @brief With default */ @@ -447,10 +447,9 @@ struct JobColumn /** @brief Job Column Structure */ , cs(nullptr) { } - JobColumn(const std::string& colName_, OID mapOid_, const std::string& typeName_, - int width_, int definedWidth_, int compressionType_, int dctnryCompressionType_, - int64_t minIntSat_, uint64_t maxIntSat_, bool withDefault_, - unsigned long long defaultUInt_) + JobColumn(const std::string& colName_, OID mapOid_, const std::string& typeName_, int width_, + int definedWidth_, int compressionType_, int dctnryCompressionType_, int64_t minIntSat_, + uint64_t maxIntSat_, bool withDefault_, unsigned long long defaultUInt_) : colName(colName_) , mapOid(mapOid_) , dataType(execplan::CalpontSystemCatalog::INT) diff --git a/writeengine/splitter/CMakeLists.txt b/writeengine/splitter/CMakeLists.txt index 7b05928f6..8d2d12251 100644 --- a/writeengine/splitter/CMakeLists.txt +++ b/writeengine/splitter/CMakeLists.txt @@ -16,15 +16,13 @@ set(cpimport_SRCS columnstore_executable(cpimport ${cpimport_SRCS}) -add_dependencies(cpimport loggingcpp) - columnstore_link( cpimport ${ENGINE_LDFLAGS} - ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} batchloader threadpool marias3 boost_program_options + loggingcpp ) diff --git a/writeengine/wrapper/CMakeLists.txt b/writeengine/wrapper/CMakeLists.txt index 28a7b50f9..2a1e71325 100644 --- a/writeengine/wrapper/CMakeLists.txt +++ b/writeengine/wrapper/CMakeLists.txt @@ -37,7 +37,4 @@ set(writeengine_LIB_SRCS add_definitions(-D_FILE_OFFSET_BITS=64) columnstore_library(writeengine ${writeengine_LIB_SRCS}) - -add_dependencies(writeengine loggingcpp) - -columnstore_link(writeengine ${NETSNMP_LIBRARIES}) +columnstore_link(writeengine loggingcpp oamcpp boost_thread compress messageqcpp) diff --git a/writeengine/xml/CMakeLists.txt b/writeengine/xml/CMakeLists.txt index 98ab2fbb3..2ef5b0e51 100644 --- a/writeengine/xml/CMakeLists.txt +++ b/writeengine/xml/CMakeLists.txt @@ -6,6 +6,5 @@ set(we_xml_STAT_SRCS we_xmlop.cpp we_xmljob.cpp we_xmlgendata.cpp we_xmlgenproc. add_definitions(-D_FILE_OFFSET_BITS=64) -add_library(we_xml STATIC ${we_xml_STAT_SRCS}) - -add_dependencies(we_xml loggingcpp) +columnstore_static_library(we_xml ${we_xml_STAT_SRCS}) +columnstore_link(we_xml loggingcpp)