database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Deep build refactoring phase 2 (#3564)

Browse Source 
* configcpp refactored

* chore(build): massive removals, auto add files to debian install file

* chore(build): configure before autobake

* chore(build): use custom cmake commands for components, mariadb-plugin-columnstore.install generated

* chore(build): install deps as separate step for build-packages

* more deps

* chore(codemanagement, build): build refactoring stage2

* chore(safety): Locked Map for MessageqCpp with a simpler way

 Please enter the commit message for your changes. Lines starting

* chore(codemanagement, ci): better coredumps handling, deps fixed

* Delete build/bootstrap_mcs.py

* Update charset.cpp (add license)
This commit is contained in:
Leonid Fedorov
2025-07-17 16:14:10 +04:00
committed by GitHub
parent d0ee5dae32
commit 449029a827
107 changed files with 354 additions and 3327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -88,8 +88,3 @@ set(COMPONENTS
foreach(component ${COMPONENTS}) foreach(component ${COMPONENTS})
add_subdirectory(${component}) add_subdirectory(${component})
endforeach() endforeach()
add_dependencies(udf_mysql GenError)
add_dependencies(funcexp GenError)
add_dependencies(oamcpp GenError)
add_dependencies(regr_mysql GenError)

19
cmake/ColumnstoreLibrary.cmake
View File

@ -82,6 +82,16 @@ macro(columnstore_shared_library libname)
columnstore_install_target(${libname} ${ENGINE_LIBDIR}) columnstore_install_target(${libname} ${ENGINE_LIBDIR})
endmacro() endmacro()
macro(columnstore_static_library libname)
add_definitions(-fPIC -DPIC)
add_library(${libname} STATIC ${ARGN})
endmacro()
macro(columnstore_shared_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${ENGINE_LIBDIR})
endmacro()
macro(columnstore_library libname) macro(columnstore_library libname)
if(COLUMNSTORE_STATIC_LIBRARIES) if(COLUMNSTORE_STATIC_LIBRARIES)
columnstore_static_library(${libname} ${ARGN}) columnstore_static_library(${libname} ${ARGN})
@ -99,6 +109,15 @@ macro(columnstore_link libname)
target_link_libraries(${libname} ${ARGN}) target_link_libraries(${libname} ${ARGN})
endmacro() endmacro()
macro(columnstore_mysql_plugin_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${MARIADB_PLUGINDIR})
endmacro()
macro(columnstore_link libname)
target_link_libraries(${libname} ${ARGN})
endmacro()
macro(columnstore_executable executable_name) macro(columnstore_executable executable_name)
add_executable(${executable_name} ${ARGN}) add_executable(${executable_name} ${ARGN})
columnstore_install_target(${executable_name} ${ENGINE_BINDIR}) columnstore_install_target(${executable_name} ${ENGINE_BINDIR})

21
datatypes/mcs_datatype.h
View File

@ -20,7 +20,7 @@
#include <sstream> #include <sstream>
#include <boost/any.hpp> #include <boost/any.hpp>
#include "exceptclasses.h" #include "exceptclasses.h"
#include "conststring.h" #include "basic/conststring.h"
#include "mcs_datatype_basic.h" #include "mcs_datatype_basic.h"
#include "mcs_numeric_limits.h" #include "mcs_numeric_limits.h"
#include "mcs_data_condition.h" #include "mcs_data_condition.h"
@ -796,7 +796,7 @@ class MinMaxPartitionInfo : public MinMaxInfo
uint64_t m_status; uint64_t m_status;
public: public:
MinMaxPartitionInfo() : m_status(0){}; MinMaxPartitionInfo() : m_status(0) {};
explicit MinMaxPartitionInfo(const BRM::EMEntry& entry); explicit MinMaxPartitionInfo(const BRM::EMEntry& entry);
void set_invalid() void set_invalid()
{ {
@ -1102,7 +1102,8 @@ class TypeHandlerBit : public TypeHandler
idbassert(0); // QQ idbassert(0); // QQ
return 1; return 1;
} }
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{ {
return "0"; // QQ return "0"; // QQ
} }
@ -1135,7 +1136,7 @@ class TypeHandlerBit : public TypeHandler
const ConvertFromStringParam& /*prm*/, const std::string& /*str*/, const ConvertFromStringParam& /*prm*/, const std::string& /*str*/,
bool& /*pushWarning*/) const override; bool& /*pushWarning*/) const override;
const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& /*attr*/) const override const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{ {
idbassert(0); idbassert(0);
return nullptr; return nullptr;
@ -2033,7 +2034,8 @@ class TypeHandlerReal : public TypeHandler
{ {
return {}; // QQ: real types were not handled in IDB_format() return {}; // QQ: real types were not handled in IDB_format()
} }
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{ {
return "0"; // QQ return "0"; // QQ
} }
@ -2329,7 +2331,8 @@ class TypeHandlerBlob : public TypeHandlerStr
{ {
return storeValueToFieldBlobText(row, pos, f); return storeValueToFieldBlobText(row, pos, f);
} }
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{ {
return "0"; // QQ return "0"; // QQ
} }
@ -2355,7 +2358,8 @@ class TypeHandlerText : public TypeHandlerStr
{ {
return storeValueToFieldBlobText(row, pos, f); return storeValueToFieldBlobText(row, pos, f);
} }
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{ {
return "0"; // QQ return "0"; // QQ
} }
@ -2386,7 +2390,8 @@ class TypeHandlerClob : public TypeHandlerStr
idbassert(0); // QQ idbassert(0); // QQ
return 1; return 1;
} }
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{ {
return "0"; // QQ return "0"; // QQ
} }

4
datatypes/mcs_string.h
View File

@ -17,8 +17,8 @@
#pragma once #pragma once
#include "conststring.h" #include "basic/conststring.h"
#include "collation.h" // class Charset #include "mariadb_charset/collation.h" // class Charset
namespace datatypes namespace datatypes
{ {

2
dbcon/ddlpackage/CMakeLists.txt
View File

@ -42,4 +42,4 @@ columnstore_library(
${FLEX_ddl_scan_OUTPUTS} ${FLEX_ddl_scan_OUTPUTS}
) )
columnstore_link(ddlpackage loggingcpp) columnstore_link(ddlpackage loggingcpp messageqcpp)

4
dbcon/ddlpackage/sqlparser.h
View File

@ -27,7 +27,7 @@
*/ */
#include <stdexcept> #include <stdexcept>
#include "collation.h" // CHARSET_INFO #include "mariadb_charset/collation.h" // CHARSET_INFO
#include "ddlpkg.h" #include "ddlpkg.h"
#include "mariadb_my_sys.h" // myf, MYF() #include "mariadb_my_sys.h" // myf, MYF()
@ -87,7 +87,7 @@ struct pass_to_bison
myf utf8_flag; myf utf8_flag;
pass_to_bison(ParseTree* pt) pass_to_bison(ParseTree* pt)
: fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)){}; : fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)) {};
}; };
class SqlParser class SqlParser

2
dbcon/ddlpackageproc/CMakeLists.txt
View File

@ -13,4 +13,4 @@ set(ddlpackageproc_LIB_SRCS
) )
columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS}) columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS})
columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp) columnstore_link(ddlpackageproc loggingcpp oamcpp messageqcpp)

2
dbcon/dmlpackage/CMakeLists.txt
View File

@ -41,4 +41,4 @@ columnstore_library(
${FLEX_dml_scan_OUTPUTS} ${FLEX_dml_scan_OUTPUTS}
) )
columnstore_link(dmlpackage loggingcpp) columnstore_link(dmlpackage loggingcpp messageqcpp)

2
dbcon/dmlpackageproc/CMakeLists.txt
View File

@ -13,4 +13,4 @@ set(dmlpackageproc_LIB_SRCS
) )
columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS}) columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS})
columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp) columnstore_link(dmlpackageproc loggingcpp oamcpp messageqcpp)

10
dbcon/execplan/CMakeLists.txt
View File

@ -46,4 +46,12 @@ set(execplan_LIB_SRCS
) )
columnstore_library(execplan ${execplan_LIB_SRCS}) columnstore_library(execplan ${execplan_LIB_SRCS})
columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp) columnstore_link(
execplan
messageqcpp
${ENGINE_DT_LIB}
pron
oamcpp
loggingcpp
querytele
)

22
dbcon/execplan/calpontsystemcatalog.h
View File

@ -49,7 +49,7 @@
#undef max #undef max
#include "mcs_datatype.h" #include "mcs_datatype.h"
#include "collation.h" // CHARSET_INFO, class Charset #include "mariadb_charset/collation.h" // CHARSET_INFO, class Charset
#include "nullstring.h" #include "nullstring.h"
class ExecPlanTest; class ExecPlanTest;
@ -75,7 +75,8 @@ const int32_t IDB_VTABLE_ID = CNX_VTABLE_ID;
/** /**
* A struct to hold a list of table partitions. * A struct to hold a list of table partitions.
*/ */
struct Partitions { struct Partitions
{
std::vector<std::string> fPartNames; std::vector<std::string> fPartNames;
void serialize(messageqcpp::ByteStream& b) const void serialize(messageqcpp::ByteStream& b) const
{ {
@ -96,12 +97,11 @@ struct Partitions {
b >> t; b >> t;
fPartNames.push_back(t); fPartNames.push_back(t);
} }
} }
}; };
bool operator <(const Partitions& a, const Partitions& b); bool operator<(const Partitions& a, const Partitions& b);
bool operator ==(const Partitions& a, const Partitions& b); bool operator==(const Partitions& a, const Partitions& b);
bool operator !=(const Partitions& a, const Partitions& b); // for GCC9 bool operator!=(const Partitions& a, const Partitions& b); // for GCC9
/** The CalpontSystemCatalog class /** The CalpontSystemCatalog class
* *
@ -259,9 +259,9 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog
public: public:
ColType() = default; ColType() = default;
ColType(const ColType& rhs); ColType(const ColType& rhs);
ColType(int32_t colWidth_, int32_t scale_, int32_t precision_, ColType(int32_t colWidth_, int32_t scale_, int32_t precision_, const ConstraintType& constraintType_,
const ConstraintType& constraintType_, const DictOID& ddn_, int32_t colPosition_, const DictOID& ddn_, int32_t colPosition_, int32_t compressionType_, OID columnOID_,
int32_t compressionType_, OID columnOID_, const ColDataType& colDataType_); const ColDataType& colDataType_);
ColType& operator=(const ColType& rhs); ColType& operator=(const ColType& rhs);
CHARSET_INFO* getCharset() const; CHARSET_INFO* getCharset() const;
@ -1301,8 +1301,8 @@ const int OID_SYSCOLUMN_MINVALUE = SYSCOLUMN_BASE + 19; /** @brief min va
const int OID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_BASE + 20; /** @brief max value col */ const int OID_SYSCOLUMN_MAXVALUE = SYSCOLUMN_BASE + 20; /** @brief max value col */
const int OID_SYSCOLUMN_COMPRESSIONTYPE = SYSCOLUMN_BASE + 21; /** @brief compression type */ const int OID_SYSCOLUMN_COMPRESSIONTYPE = SYSCOLUMN_BASE + 21; /** @brief compression type */
const int OID_SYSCOLUMN_NEXTVALUE = SYSCOLUMN_BASE + 22; /** @brief next value */ const int OID_SYSCOLUMN_NEXTVALUE = SYSCOLUMN_BASE + 22; /** @brief next value */
const int OID_SYSCOLUMN_CHARSETNUM = SYSCOLUMN_BASE + 23; /** @brief character set number for the column */ const int OID_SYSCOLUMN_CHARSETNUM = SYSCOLUMN_BASE + 23; /** @brief character set number for the column */
const int SYSCOLUMN_MAX = SYSCOLUMN_BASE + 24; // be sure this is one more than the highest # const int SYSCOLUMN_MAX = SYSCOLUMN_BASE + 24; // be sure this is one more than the highest #
/***************************************************** /*****************************************************
* SYSTABLE columns dictionary OID definition * SYSTABLE columns dictionary OID definition

2
dbcon/execplan/clientrotator.cpp
View File

@ -47,8 +47,6 @@ using namespace logging;
#include "clientrotator.h" #include "clientrotator.h"
//#include "idb_mysql.h"
/** Debug macro */ /** Debug macro */
#ifdef INFINIDB_DEBUG #ifdef INFINIDB_DEBUG
#define IDEBUG(x) \ #define IDEBUG(x) \

2
dbcon/execplan/predicateoperator.h
View File

@ -39,7 +39,7 @@
#include "returnedcolumn.h" #include "returnedcolumn.h"
#include "dataconvert.h" #include "dataconvert.h"
#include "collation.h" // CHARSET_INFO #include "mariadb_charset/collation.h" // CHARSET_INFO
namespace messageqcpp namespace messageqcpp
{ {

2
dbcon/execplan/pseudocolumn.cpp
View File

@ -28,7 +28,7 @@ using namespace std;
#include "basic/string_utils.h" #include "basic/string_utils.h"
#include "bytestream.h" #include "bytestream.h"
#include "collation.h" #include "mariadb_charset/collation.h"
using namespace messageqcpp; using namespace messageqcpp;

10
dbcon/joblist/CMakeLists.txt
View File

@ -68,7 +68,15 @@ columnstore_library(joblist ${joblist_LIB_SRCS})
target_include_directories( target_include_directories(
joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR} joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR}
) )
columnstore_link(joblist loggingcpp) columnstore_link(
joblist
loggingcpp
boost_thread
oamcpp
querytele
messageqcpp
statistics_manager
)
if(WITH_ORDERBY_UT) if(WITH_ORDERBY_UT)
columnstore_executable(job_orderby_tests orderby-tests.cpp) columnstore_executable(job_orderby_tests orderby-tests.cpp)

2
dbcon/joblist/jlf_tuplejoblist.cpp
View File

@ -71,7 +71,7 @@ using namespace dataconvert;
#include "jlf_tuplejoblist.h" #include "jlf_tuplejoblist.h"
using namespace joblist; using namespace joblist;
#include "statistics.h" #include "statistics_manager/statistics.h"
#ifdef __clang__ #ifdef __clang__
#pragma clang diagnostic push #pragma clang diagnostic push

4
dbcon/mysql/CMakeLists.txt
View File

@ -63,7 +63,7 @@ if(COMMAND mysql_add_plugin)
${PLUGIN_EXEC_LIBS} ${PLUGIN_EXEC_LIBS}
${PLUGIN_WRITE_LIBS} ${PLUGIN_WRITE_LIBS}
joblist_server joblist_server
${NETSNMP_LIBRARIES} statistics_manager
${MARIADB_CLIENT_LIBS} ${MARIADB_CLIENT_LIBS}
${S3API_DEPS} ${S3API_DEPS}
threadpool threadpool
@ -89,10 +89,10 @@ else()
${S3API_DEPS} ${S3API_DEPS}
${ENGINE_LDFLAGS} ${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS} ${ENGINE_WRITE_LIBS}
${NETSNMP_LIBRARIES}
${SERVER_BUILD_DIR}/libservices/libmysqlservices.a ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a
threadpool threadpool
loggingcpp loggingcpp
statistics_manager
marias3 marias3
) )
# define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist) # define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist)

2
dbcon/mysql/ha_mcs_impl.cpp
View File

@ -128,9 +128,9 @@ using namespace funcexp;
#include "ha_mcs_sysvars.h" #include "ha_mcs_sysvars.h"
#include "ha_mcs_datatype.h" #include "ha_mcs_datatype.h"
#include "statistics.h"
#include "ha_mcs_logging.h" #include "ha_mcs_logging.h"
#include "ha_subquery.h" #include "ha_subquery.h"
#include "statistics_manager/statistics.h"
namespace cal_impl_if namespace cal_impl_if
{ {

15
dbcon/mysql/idb_mysql.h
View File

@ -14,10 +14,17 @@
along with this program; if not, write to the Free Software along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */ MA 02110-1301, USA. */
// One include file to deal with all the MySQL pollution of the
// global namespace /* One include file to deal with all the MySQL pollution of the
// global namespace
// Don't include ANY mysql headers anywhere except here!
Don't include ANY mysql headers anywhere except here!
WARN: if any cmake build target uses this include file,
GenError from server must be added to the target dependencies
to generate mysqld_error.h used below
*/
#pragma once #pragma once
#ifdef TEST_MCSCONFIG_H #ifdef TEST_MCSCONFIG_H

2
ddlproc/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp)
columnstore_executable(DDLProc ${DDLProc_SRCS}) columnstore_executable(DDLProc ${DDLProc_SRCS})
columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp) columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool loggingcpp)

1
dmlproc/CMakeLists.txt
View File

@ -12,7 +12,6 @@ columnstore_link(
DMLProc DMLProc
${ENGINE_LDFLAGS} ${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS} ${ENGINE_WRITE_LIBS}
${NETSNMP_LIBRARIES}
threadpool threadpool
ddlcleanuputil ddlcleanuputil
batchloader batchloader

2
oam/oamcpp/CMakeLists.txt
View File

@ -5,6 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp) set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp)
columnstore_library(oamcpp ${oamcpp_LIB_SRCS}) columnstore_library(oamcpp ${oamcpp_LIB_SRCS})
columnstore_link(oamcpp loggingcpp) columnstore_link(oamcpp loggingcpp idbboot)
target_compile_options(oamcpp PRIVATE -Wno-unused-result) target_compile_options(oamcpp PRIVATE -Wno-unused-result)

16
oamapps/columnstoreSupport/CMakeLists.txt
View File

@ -1,16 +0,0 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp)
columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS})
target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result)
columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(configReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(logReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR})

39
oamapps/columnstoreSupport/alarmReport.sh
View File

@ -1,39 +0,0 @@
#! /bin/sh
#
# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
SERVER=$1
else
SERVER="localhost"
fi
if [ $2 ] ; then
DATE=$2
else
DATE=" "
fi
#get temp directory
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/logReport.log
{
echo " "
echo "******************** Alarm Report for $SERVER ********************"
echo " "
echo "-- Today's Alarms --"
echo " "
cat /var/log/mariadb/columnstore/alarm.log 2>/dev/null
if test -f /var/log/mariadb/columnstore/archive/alarm.log-$DATE ; then
echo "-- Archived Alarms --"
echo " "
cat /var/log/mariadb/columnstore/archive/alarm.log-$DATE 2>/dev/null
fi
} > ${tmpDir}/logReport.log
exit 0

60
oamapps/columnstoreSupport/approximateRowCount.sh
View File

@ -1,60 +0,0 @@
#!/bin/bash
#
# Estimates the row count for a given table. Uses number of extents * 8M for the estimate.
#
#
# Initialize variables.
#
if [ -z "$MYSQLCMD" ]; then
MYSQLCMD="mysql -u root"
fi
#
# Validate that there are two parameters - schema and table.
#
if [ $# -ne 2 ]; then
echo ""
echo "Reports the approximate row count for the given table."
echo ""
echo "Parameters:"
echo " Schema"
echo " Table"
fi
db=$1
table=$2
#
# Validate that the table exists.
#
sql="select count(*) from systable where \`schema\`='$db' and tablename='$table';"
count=`$MYSQLCMD calpontsys --skip-column-names -e "$sql;"`
if [ $count -le 0 ]; then
echo ""
echo "$db.$table does not exist in Columnstore."
echo ""
exit 1
fi
#
# Grab the objectid and column width for a column in the table.
#
sql="select objectid from syscolumn where \`schema\`='$db' and tablename='$table' limit 1;"
objectid=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
sql="select columnlength from syscolumn where objectid=$objectid;"
colWidth=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
#
# Use editem to count the extents.
#
extentCount=`editem -o $objectid | wc -l`
let extentCount-=2 # Take out the 2 extra rows for header and blank line at end.
let approximateRowCount=$extentCount*8192*1024;
echo ""
echo "Approximate row count for $db.$table is $approximateRowCount."
echo ""
exit 0

30
oamapps/columnstoreSupport/bulklogReport.sh
View File

@ -1,30 +0,0 @@
#! /bin/sh
#
# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
if test -d /var/lib/columnstore/data/bulk ; then
echo " "
echo "-- Check for Errors in Bulk Logs --"
echo " "
echo "################# egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err #################"
echo " "
egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err 2>/dev/null
fi
} >> $OUT_FILE
exit 0

897
oamapps/columnstoreSupport/columnstoreSupport.cpp
View File

@ -1,897 +0,0 @@
/* Copyright (C) 2013 Calpont Corp. */
/* Copyright (C) 2016 MariaDB Corporation */
/******************************************************************************************
* $Id: columnstoreSupport.cpp 64 2006-10-12 22:21:51Z dhill $
*
*
*
******************************************************************************************/
/**
* @file
*/
#include <iterator>
#include <numeric>
#include <deque>
#include <iostream>
#include <ostream>
#include <fstream>
#include <cstdlib>
#include <string>
#include <limits.h>
#include <sstream>
#include <exception>
#include <stdexcept>
#include <vector>
#include "stdio.h"
#include "ctype.h"
#include <netdb.h>
#include <readline.h>
#include <boost/filesystem.hpp>
#include "mcsconfig.h"
#include "liboamcpp.h"
#include "configcpp.h"
#include "installdir.h"
#include "mcsSupportUtil.h"
#include "columnstoreversion.h"
using namespace std;
using namespace oam;
using namespace config;
typedef struct Child_Module_struct
{
std::string moduleName;
std::string moduleIP;
std::string hostName;
} ChildModule;
typedef std::vector<ChildModule> ChildModuleList;
string currentDate;
string systemName;
string localModule;
string localModuleHostName;
ChildModuleList childmodulelist;
ChildModuleList parentmodulelist;
ChildModule childmodule;
string rootPassword = "";
string debug_flag = "0";
string mysqlpw = " ";
string tmpDir;
int runningThreads = 0;
pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
typedef boost::tuple<ChildModuleList::iterator, string> threadInfo_t;
bool LOCAL = false;
void* childReportThread(threadInfo_t* st)
{
assert(st);
ChildModuleList::iterator& list = boost::get<0>(*st);
string reportType = boost::get<1>(*st);
string remoteModuleName = (*list).moduleName;
string remoteModuleIP = (*list).moduleIP;
string remoteHostName = (*list).hostName;
pthread_mutex_lock(&mutex1);
runningThreads++;
// cout << "++ " << runningThreads << endl;
pthread_mutex_unlock(&mutex1);
string outputFile;
if (reportType == "log")
{
outputFile = remoteModuleName + "_" + reportType + "Report.tar.gz";
}
else
{
outputFile = remoteModuleName + "_" + reportType + "Report.txt";
FILE* pOutputFile = fopen(outputFile.c_str(), "a");
if (pOutputFile == NULL)
{
printf("Could not open file: %s", outputFile.c_str());
exit(1);
}
fprintf(pOutputFile,
"********************************************************************************\n"
"\n"
" System %s\n"
" columnstoreSupportReport script ran from Module %s on %s\n"
" SoftwareVersion = %s-%s"
"\n"
"********************************************************************************\n"
"\n"
" %s report\n"
"\n"
"********************************************************************************\n",
systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(),
columnstore_release.c_str(), reportType.c_str());
}
cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl;
string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + ";" + reportType + "Report.sh " +
remoteModuleName + "' " + debug_flag + " - forcetty";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << "Error with running remote_command.sh, exiting..." << endl;
}
cmd = "remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile +
" > /dev/null 2>&1";
rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
cout << "ERROR: failed to retrieve " << tmpDir << "/" << outputFile << " from " + remoteHostName << endl;
pthread_mutex_lock(&mutex1);
runningThreads--;
// cout << "-- " << runningThreads << endl;
pthread_mutex_unlock(&mutex1);
// exit thread
pthread_exit(0);
}
void* reportThread(string* reporttype)
{
assert(reporttype);
string reportType = *reporttype;
Oam oam;
pthread_mutex_lock(&mutex1);
runningThreads++;
// cout << "++ " << runningThreads << endl;
pthread_mutex_unlock(&mutex1);
string outputFile = localModule + "_" + reportType + "Report.txt";
FILE* pOutputFile = fopen(outputFile.c_str(), "a");
if (pOutputFile == NULL)
{
printf("Could not open file: %s", outputFile.c_str());
exit(1);
}
// get local report
fprintf(pOutputFile,
"********************************************************************************\n"
"\n"
" System %s\n"
" columnstoreSupportReport script ran from Module %s on %s\n"
" SoftwareVersion = %s-%s"
"\n"
"********************************************************************************\n"
"\n"
" %s report\n"
"\n"
"********************************************************************************\n",
systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(),
columnstore_release.c_str(), reportType.c_str());
fclose(pOutputFile);
// run on child servers and get report
if (!LOCAL)
{
ChildModuleList::iterator list1 = childmodulelist.begin();
for (; list1 != childmodulelist.end(); list1++)
{
threadInfo_t* st = new threadInfo_t;
*st = boost::make_tuple(list1, reportType);
pthread_t childreportthread;
int status = pthread_create(&childreportthread, NULL, (void* (*)(void*)) & childReportThread, st);
if (status != 0)
{
cout << "ERROR: childreportthread: pthread_create failed, return status = " + oam.itoa(status)
<< endl;
}
sleep(1);
}
}
if (reportType == "log")
{
// run log config on local server
cout << "Get log config data for " + localModule << endl;
string cmd = "logReport.sh " + localModule + " " + outputFile;
system(cmd.c_str());
}
else
{
string cmd = reportType + "Report.sh " + localModule + " " + outputFile;
system(cmd.c_str());
if (reportType == "config")
{
pOutputFile = fopen(outputFile.c_str(), "a");
if (pOutputFile == NULL)
{
printf("Could not open file: %s", outputFile.c_str());
exit(1);
}
fprintf(pOutputFile,
"\n******************** System Network Configuration ******************************\n\n");
getSystemNetworkConfig(pOutputFile);
fprintf(pOutputFile,
"\n******************** System Module Configure **********************************\n\n");
getModuleTypeConfig(pOutputFile);
fprintf(pOutputFile,
"\n******************** System Storage Configuration *****************************\n\n");
getStorageConfig(pOutputFile);
fprintf(pOutputFile,
"\n******************** System Storage Status ************************************\n\n");
getStorageStatus(pOutputFile);
// BT: most of this is tedious to collect and can be manually looked up in the debug.log file
// fprintf(pOutputFile,"\n******************** System Status
// ********************************************\n\n"); printSystemStatus(pOutputFile);
// printProcessStatus(pOutputFile);
// printAlarmSummary(pOutputFile);
//
// fprintf(pOutputFile,"\n******************** System Directories
// ***************************************\n\n"); getSystemDirectories(pOutputFile);
boost::filesystem::path configFile =
std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml");
boost::filesystem::copy_file(configFile, "./Columnstore.xml",
boost::filesystem::copy_options::overwrite_existing);
boost::filesystem::path SMconfigFile =
std::string(MCSSYSCONFDIR) + std::string("/columnstore/storagemanager.cnf");
boost::filesystem::copy_file(SMconfigFile, "./storagemanager.cnf",
boost::filesystem::copy_options::overwrite_existing);
system("sed -i 's/.*aws_access_key_id.*/aws_access_key_id={PRIVATE}/' ./storagemanager.cnf");
system("sed -i 's/.*aws_secret_access_key.*/aws_secret_access_key={PRIVATE}/' ./storagemanager.cnf");
fclose(pOutputFile);
}
/*
// TODO: This can be ported from mcsadmin if needed most info included does not seem useful at this time
if (reportType == "resource" )
{
if (LOCAL)
{
fprintf(pOutputFile,"\n******************** mcsadmin getModuleResourceUsage
**************************\n\n"); string cmd = "mcsadmin getModuleResourceUsage " + localModule + " >> " +
outputFile; system(cmd.c_str());
}
else
{
fprintf(pOutputFile,"\n******************** mcsadmin getSystemResourceUsage
**************************\n\n"); string cmd = "mcsadmin getSystemResourceUsage >> " + outputFile;
system(cmd.c_str());
}
}*/
}
// exit thread
pthread_mutex_lock(&mutex1);
runningThreads--;
// cout << "-- " << runningThreads << endl;
pthread_mutex_unlock(&mutex1);
pthread_exit(0);
}
int main(int argc, char* argv[])
{
Oam oam;
Config* sysConfig = Config::makeConfig();
string SystemSection = "SystemConfig";
string InstallSection = "Installation";
bool HARDWARE = false;
bool CONFIG = false;
bool DBMS = false;
bool RESOURCE = false;
bool LOG = false;
bool BULKLOG = false;
bool HADOOP = false;
// get current time and date
time_t now;
now = time(NULL);
struct tm tm;
localtime_r(&now, &tm);
char timestamp[200];
strftime(timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm);
currentDate = timestamp;
char helpArg[3] = "-h";
// Get System Name
try
{
oam.getSystemConfig("SystemName", systemName);
}
catch (...)
{
systemName = "unassigned";
}
// get Local Module Name and Server Install Indicator
string singleServerInstall = "n";
oamModuleInfo_t st;
try
{
st = oam.getModuleInfo();
localModule = boost::get<0>(st);
}
catch (...)
{
cout << endl << "**** Failed : Failed to read Local Module Name" << endl;
exit(-1);
}
if (argc == 1)
{
argv[1] = &helpArg[0];
argc = 2;
}
string DataFilePlugin;
try
{
DataFilePlugin = sysConfig->getConfig(SystemSection, "DataFilePlugin");
}
catch (...)
{
cout << "ERROR: Problem accessing Columnstore configuration file" << endl;
exit(-1);
}
tmpDir = startup::StartUp::tmpDir();
for (int i = 1; i < argc; i++)
{
if (string("-h") == argv[i])
{
cout << endl;
cout << "'columnstoreSupport' generates a Set of System Support Report Files in a tar file" << endl;
cout << "called columnstoreSupportReport.'system-name'.tar.gz in the local directory." << endl;
cout << "It should be run on the server with the DBRM front-end." << endl;
cout << "Check the Admin Guide for additional information." << endl;
cout << endl;
cout << "Usage: columnstoreSupport [-h][-a][-hw][-s][-c][-db][-r][-l][-bl][-lc][-p "
"'root-password'][-de]";
cout << endl;
cout << " -h help" << endl;
cout << " -a Output all Reports (excluding Bulk Logs Reports)" << endl;
cout << " -hw Output Hardware Reports only" << endl;
cout << " -c Output Configuration/Status Reports only" << endl;
cout << " -db Output DBMS Reports only" << endl;
cout << " -r Output Resource Reports only" << endl;
cout << " -l Output Columnstore Log/Alarms Reports only" << endl;
cout << " -bl Output Columnstore Bulk Log Reports only" << endl;
cout << " -lc Output Reports for Local Server only" << endl;
cout << " -p password (multi-server systems), root-password or 'ssh' to use 'ssh keys'"
<< endl;
cout << " -de Debug Flag" << endl;
exit(0);
}
else
{
if (string("-a") == argv[i])
{
HARDWARE = true;
CONFIG = true;
DBMS = true;
RESOURCE = true;
LOG = true;
HADOOP = (DataFilePlugin.empty() ? false : true);
}
else if (string("-hw") == argv[i])
HARDWARE = true;
else if (string("-c") == argv[i])
CONFIG = true;
else if (string("-db") == argv[i])
DBMS = true;
else if (string("-r") == argv[i])
RESOURCE = true;
else if (string("-l") == argv[i])
LOG = true;
else if (string("-bl") == argv[i])
BULKLOG = true;
else if (string("-lc") == argv[i])
LOCAL = true;
else if (string("-p") == argv[i])
{
i++;
if (argc == i)
{
cout << "ERROR: missing root password argument" << endl;
exit(-1);
}
rootPassword = argv[i];
// add single quote for special characters
if (rootPassword != "ssh")
{
rootPassword = "'" + rootPassword + "'";
}
}
else if (string("-mp") == argv[i])
{
i++;
if (argc == i)
{
cout << "ERROR: missing MariaDB Columnstore root user password argument" << endl;
exit(-1);
}
mysqlpw = argv[i];
mysqlpw = "'" + mysqlpw + "'";
}
else if (string("-de") == argv[i])
debug_flag = "1";
else if (string("-hd") == argv[i])
{
HADOOP = (DataFilePlugin.empty() ? false : true);
}
else
{
cout << "Invalid Option of '" << argv[i] << "', run with '-h' for help" << endl;
exit(1);
}
}
}
// default to -a if nothing is set
if (!HARDWARE && !CONFIG && !DBMS && !RESOURCE && !LOG && !BULKLOG && !HADOOP)
{
HARDWARE = true;
CONFIG = true;
DBMS = true;
RESOURCE = true;
LOG = true;
HADOOP = (DataFilePlugin.empty() ? false : true);
}
// get Parent OAM Module Name and setup of it's Custom OS files
string PrimaryUMModuleName;
try
{
PrimaryUMModuleName = sysConfig->getConfig(SystemSection, "PrimaryUMModuleName");
}
catch (...)
{
cout << "ERROR: Problem getting Parent OAM Module Name" << endl;
exit(-1);
}
if (PrimaryUMModuleName == "unassigned")
PrimaryUMModuleName = localModule;
if ((localModule != PrimaryUMModuleName) && DBMS)
{
char* pcommand = 0;
char* p;
string argument = "n";
while (true)
{
cout << endl << "You selected to get the DBMS data." << endl;
cout << "You need to run the columnstoreSupport command on module '" << PrimaryUMModuleName
<< "' to get that information." << endl;
cout << "Or you can proceed on to get all data except the DBMS." << endl;
pcommand = readline(" Do you want to proceed: (y or n) [n]: ");
if (pcommand && *pcommand)
{
p = strtok(pcommand, " ");
argument = p;
free(pcommand);
pcommand = 0;
}
if (pcommand)
{
free(pcommand);
pcommand = 0;
}
if (argument == "y")
{
cout << endl;
break;
}
else if (argument == "n")
exit(1);
}
}
// get number of worker-nodes, will tell us if a single server system
// get Parent OAM Module Name and setup of it's Custom OS files
try
{
string NumWorkers = sysConfig->getConfig("DBRM_Controller", "NumWorkers");
if (NumWorkers == "1")
singleServerInstall = "y";
}
catch (...)
{
}
if (singleServerInstall == "n" && !LOCAL)
if (HARDWARE || CONFIG || RESOURCE || LOG || HADOOP)
if (rootPassword.empty())
{
cout << "ERROR: Multi-Module System, Password Argument required or use '-lc' option, check help for "
"more information"
<< endl;
exit(-1);
}
// get Parent OAM Module Name and setup of it's Custom OS files
// string parentOAMModuleName;
ChildModule parentOAMModule;
try
{
parentOAMModule.moduleName = sysConfig->getConfig(SystemSection, "ParentOAMModuleName");
}
catch (...)
{
cout << "ERROR: Problem getting Parent OAM Module Name" << endl;
exit(-1);
}
// Get list of configured system modules
SystemModuleTypeConfig sysModuleTypeConfig;
try
{
oam.getSystemConfig(sysModuleTypeConfig);
}
catch (...)
{
cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl;
exit(-1);
}
string ModuleSection = "SystemModuleConfig";
for (unsigned int i = 0; i < sysModuleTypeConfig.moduletypeconfig.size(); i++)
{
string moduleType = sysModuleTypeConfig.moduletypeconfig[i].ModuleType;
int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount;
if (moduleCount == 0)
// no modules equipped for this Module Type, skip
continue;
// get IP addresses and Host Names
DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin();
for (; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end(); listPT++)
{
string moduleName = (*listPT).DeviceName;
HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin();
string moduleIPAddr = (*pt1).IPAddr;
string moduleHostName = (*pt1).HostName;
if (moduleName == localModule)
{
localModuleHostName = moduleHostName;
}
// save Child modules
if (moduleName != localModule && moduleType != "xm")
{
childmodule.moduleName = moduleName;
childmodule.moduleIP = moduleIPAddr;
childmodule.hostName = moduleHostName;
childmodulelist.push_back(childmodule);
}
if (moduleName == parentOAMModule.moduleName)
{
parentOAMModule.moduleIP = moduleIPAddr;
parentOAMModule.hostName = moduleHostName;
parentOAMModule.moduleName = moduleName;
}
}
} // end of i for loop
// create a clean Columnstore Support Report
system("rm -f *_configReport.txt");
system("rm -f *_dbmsReport.txt");
system("rm -f *_hardwareReport.txt");
system("rm -f *_logReport.txt");
system("rm -f *_bulklogReport.txt");
system("rm -f *_resourceReport.txt");
//
// Configuration
//
if (CONFIG)
{
string reportType = "config";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// Alarms and Columnstore Logs
//
if (LOG)
{
string reportType = "log";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// Bulk Logs
//
if (BULKLOG)
{
string reportType = "bulklog";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// Hardware
//
if (HARDWARE)
{
string reportType = "hardware";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// Resources
//
if (RESOURCE)
{
string reportType = "resource";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// DBMS
//
if (DBMS)
{
cout << "Get dbms report data for " << localModule << endl;
string outputFile = localModule + "_dbmsReport.txt";
FILE* pOutputFile = fopen(outputFile.c_str(), "w");
if (pOutputFile == NULL)
{
cout << "Could not open file: " + outputFile << endl;
exit(1);
}
fprintf(pOutputFile,
"********************************************************************************\n"
"\n"
" System %s\n"
" columnstoreSupportReport script ran from Module %s on %s\n"
" SoftwareVersion = %s-%s"
"\n"
"********************************************************************************\n"
"\n"
" DBMS report\n"
"\n"
"********************************************************************************\n",
systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(),
columnstore_release.c_str());
fclose(pOutputFile);
// run DBMS report on local server
bool FAILED = false;
if (localModule != PrimaryUMModuleName)
{
cout << " FAILED: run columnstoreSupport on '" << PrimaryUMModuleName << "' to get the dbrm report"
<< endl;
FAILED = true;
}
else
{
// check if mysql is supported and get info
string logFile = tmpDir + "/idbmysql.log";
string columnstoreMysql = "mysql -u root ";
string cmd = columnstoreMysql + " -e 'status' > " + logFile + " 2>&1";
system(cmd.c_str());
// check for mysql password set
string pwprompt = " ";
if (checkLogStatus(logFile, "ERROR 1045"))
{
cout << "NOTE: MariaDB Columnstore root user password is set" << endl;
// needs a password, was password entered on command line
if (mysqlpw == " ")
{
// go check columnstore.cnf
string file = std::string(MCSMYCNFDIR) + "/columnstore.cnf";
ifstream oldFile(file.c_str());
vector<string> lines;
char line[200];
string buf;
while (oldFile.getline(line, 200))
{
buf = line;
string::size_type pos = buf.find("password", 0);
if (pos != string::npos)
{
string::size_type pos1 = buf.find("=", 0);
if (pos1 != string::npos)
{
pos = buf.find("#", 0);
if (pos == string::npos)
{
// password arg in columnstore.cnf, go get password
cout << "NOTE: Using password from columnstore.cnf" << endl;
mysqlpw = buf.substr(pos1 + 1, 80);
cout << mysqlpw << endl;
break;
}
}
}
}
oldFile.close();
if (mysqlpw == " ")
{
cout << "NOTE: No password provide on command line or found uncommented in columnstore.cnf"
<< endl;
cout << endl;
string prompt = " *** Enter MariaDB Columnstore password > ";
mysqlpw = getpass(prompt.c_str());
}
}
// check for mysql password set
pwprompt = "--password=" + mysqlpw;
string cmd = columnstoreMysql + pwprompt + " -e 'status' > " + logFile + " 2>&1";
system(cmd.c_str());
if (checkLogStatus(logFile, "ERROR 1045"))
{
cout << "FAILED: Failed login using MariaDB Columnstore root user password '" << mysqlpw << "'"
<< endl;
FAILED = true;
}
}
if (!FAILED)
{
string cmd = "dbmsReport.sh " + localModule + " " + outputFile + " " + std::string(MCSSUPPORTDIR) +
" " + pwprompt;
system(cmd.c_str());
}
}
/*
BT: This doesn't appear to do anything
fprintf(pOutputFile,"\n******************** Database Size Report
*************************************\n\n"); getStorageStatus(pOutputFile);
string file = "databaseSizeReport";
ifstream File (file.c_str());
if (File)
{
string cmd = "databaseSizeReport >> " + outputFile;
system(cmd.c_str());
}
*/
boost::filesystem::path configFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf";
boost::filesystem::copy_file(configFile, "./columnstore.cnf",
boost::filesystem::copy_options::overwrite_existing);
}
int wait = 0;
while (true)
{
// cout << "check " << runningThreads << endl;
if (runningThreads < 1)
break;
sleep(2);
wait++;
// give it 60 minutes to complete
if (wait >= 3600 * 5)
{
cout << "Timed out (60 minutes) waiting for Requests to complete" << endl;
}
}
system("unix2dos *Report.txt > /dev/null 2>&1");
system(
"rm -rf columnstoreSupportReport;"
"mkdir columnstoreSupportReport;"
"mv *Report.txt columnstoreSupportReport/. > /dev/null 2>&1;"
"mv Columnstore.xml columnstoreSupportReport/. > /dev/null 2>&1;"
"mv columnstore.cnf columnstoreSupportReport/. > /dev/null 2>&1;"
"mv storagemanager.cnf columnstoreSupportReport/. > /dev/null 2>&1;"
"mv *Report.tar.gz columnstoreSupportReport/. > /dev/null 2>&1");
string cmd = "tar -zcf columnstoreSupportReport." + systemName + ".tar.gz columnstoreSupportReport/*";
system(cmd.c_str());
cout << endl
<< "Columnstore Support Script Successfully completed, files located in columnstoreSupportReport." +
systemName + ".tar.gz"
<< endl;
}

76
oamapps/columnstoreSupport/configReport.sh
View File

@ -1,76 +0,0 @@
#!/bin/bash
#
# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
echo " "
echo "******************** Configuration/Status Report for ${MODULE} ********************"
echo " "
chkconfig=`which chkconfig 2>/dev/null`
if [ -n "$chkconfig" ]; then
echo "-- chkconfig configuration --"
echo " "
echo "################# chkconfig --list | grep columnstore #################"
echo " "
chkconfig --list | grep columnstore 2>/dev/null
fi
systemctl=`which systemctl 2>/dev/null`
if [ -n "$systemctl" ]; then
echo "-- systemctl configuration --"
echo " "
echo "################# systemctl list-unit-files --type=service | grep columnstore #################"
echo " "
systemctl list-unit-files --type=service | grep columnstore 2>/dev/null
echo "################# systemctl list-unit-files --type=service | grep mariadb #################"
echo " "
systemctl list-unit-files --type=service | grep mariadb 2>/dev/null
fi
updaterc=`which update-rc.d 2>/dev/null`
if [ -n "$updaterc" ]; then
echo "-- services configuration --"
echo " "
echo "################# service --status-all | grep columnstore #################"
echo " "
service --status-all | grep columnstore 2>/dev/null
fi
echo " "
echo "-- fstab Configuration --"
echo " "
echo "################# cat /etc/fstab #################"
echo " "
cat /etc/fstab 2>/dev/null
echo " "
echo "-- Server Processes --"
echo " "
echo "################# ps axu #################"
echo " "
ps axu
echo " "
echo "-- Server Processes with resource usage --"
echo " "
echo "################# top -b -n 1 #################"
echo " "
top -b -n 1
} >> $OUT_FILE
exit 0

64
oamapps/columnstoreSupport/dbmsReport.sh
View File

@ -1,64 +0,0 @@
#! /bin/sh
#
# $Id: dbmsReport.sh
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
if [ $3 ] ; then
MCSSUPPORTDIR=$3
else
MCSSUPPORTDIR="/usr/share/columnstore"
fi
if [ $4 ] ; then
PW_PROMPT=$4
else
PW_PROMPT=""
fi
{
columnstoreMysql="mysql -u root ${PW_PROMPT} "
if ${columnstoreMysql} -V > /dev/null 2>&1; then
echo " "
echo "******************** DBMS Columnstore Version *********************************"
echo " "
${columnstoreMysql} -e 'status'
echo " "
echo "******************** DBMS Columnstore System Column ***************************"
echo " "
${columnstoreMysql} -e 'desc calpontsys.syscolumn;'
echo " "
echo "******************** DBMS Columnstore System Table ****************************"
echo " "
${columnstoreMysql} -e 'desc calpontsys.systable;'
echo " "
echo "******************** DBMS Columnstore System Catalog Data *********************"
echo " "
${columnstoreMysql} calpontsys < $MCSSUPPORTDIR/dumpcat_mysql.sql
echo " "
echo "******************** DBMS Columnstore System Table Data ***********************"
echo "******************** DBMS Columnstore Databases *******************************"
echo " "
${columnstoreMysql} -e 'show databases;'
echo " "
echo "******************** DBMS Columnstore variables *******************************"
echo " "
${columnstoreMysql} -e 'show variables;'
echo " "
fi
} >> $OUT_FILE
exit 0

130
oamapps/columnstoreSupport/getMinMax.sh
View File

@ -1,130 +0,0 @@
#!/bin/bash
#
# Reports the max value from the extent map for the given column.
#
#
# Initialize variables.
#
if [ -z "$MYSQLCMD" ]; then
MYSQLCMD="mysql -u root"
fi
#
# Validate that there are three parameters - schema and table and columnname.
#
if [ $# -ne 3 ]; then
echo ""
echo "Reports the max value for the given column."
echo ""
echo "Parameters:"
echo " Schema"
echo " Table"
echo " Column"
exit 1
fi
db=$1
table=$2
column=$3
#
# Validate that the column exists.
#
sql="select count(*) from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';"
count=`$MYSQLCMD calpontsys --skip-column-names -e "$sql;"`
if [ $count -le 0 ]; then
echo ""
echo "$db.$table.$column does not exist in Columnstore."
echo ""
exit 1
fi
#
# Validate that the column type is one that this script supports.
# Supported Types:
# 6 int
# 8 date
# 9 bigint
# 11 datetime
sql="select datatype from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';"
dataType=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
if [ $dataType -ne 6 ] && [ $dataType -ne 8 ] && [ $dataType -ne 9 ] && [ $dataType -ne 11 ]; then
echo ""
echo "The column data type must be an int, bigint, date, or datetime."
echo ""
exit 1
fi
#
# Grab the objectid for the column.
#
sql="select objectid from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';"
objectid=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
#
# Set the editem specific parameter if the column is a date or datetime.
#
if [ $dataType -eq 8 ]; then
parm="-t"
elif [ $dataType -eq 11 ]; then
parm="-s"
fi
#
# Use the editem utility to get the min and max value.
#
editem -o $objectid $parm | grep max | awk -v dataType=$dataType '
BEGIN {
allValid=1;
foundValidExtent=0;
}
{
if(dataType == 11) {
state=substr($14, 1, length($14)-1); # Datetime has date and time as two fields.
thisMin=$6 " " substr($7, 1, length($7)-1);
thisMax=$9 " " substr($10, 1, length($10)-1);
}
else {
state=substr($12, 1, length($12)-1);
thisMin=substr($6, 1, length($6)-1);
thisMax=substr($8, 1, length($8)-1);
}
if(state == "valid") {
if(!foundValidExtent) {
min=thisMin;
max=thisMax;
foundValidExtent=1;
}
else {
if(thisMin < min) {
min=thisMin;
}
if(thisMax > max) {
max=thisMax;
}
}
}
else {
allValid=0;
}
}
END {
if(foundValidExtent == 1) {
print "";
print "Min=" min;
print "Max=" max;
print "";
if(allValid == 0) {
print "Not all extents had min and max values set. Answer is incomplete."
}
}
else {
print "";
print "There were not any extents with valid min/max values. Unable to provide answer.";
print "";
}
}'
exit 0

79
oamapps/columnstoreSupport/hardwareReport.sh
View File

@ -1,79 +0,0 @@
#! /bin/sh
#
# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
echo " "
echo "******************** Hardware Report for ${MODULE} ********************"
echo " "
echo "-- Server OS Version --"
echo " "
echo "################# cat /proc/version #################"
echo " "
cat /proc/version 2>/dev/null
echo " "
echo "################# uname -a #################"
echo " "
uname -a
echo " "
echo "################# cat /etc/issue #################"
echo " "
cat /etc/issue 2>/dev/null
echo " "
echo "run columnstore_os_check.sh"
echo " "
echo "################# /bin/columnstore_os_check.sh #################"
echo " "
columnstore_os_check.sh 2>/dev/null
echo " "
echo "-- Server Uptime --"
echo " "
echo "################# uptime #################"
echo " "
uptime
echo " "
echo "-- Server cpu-info --"
echo " "
echo "################# cat /proc/cpuinfo #################"
echo " "
cat /proc/cpuinfo 2>/dev/null
echo " "
echo "-- Server memory-info --"
echo " "
echo "################# cat /proc/meminfo #################"
echo " "
$cat /proc/meminfo 2>/dev/null
echo " "
echo "-- Server mounts --"
echo " "
echo "################# cat /proc/mounts #################"
echo " "
cat /proc/mounts 2>/dev/null
echo " "
echo "-- Server Ethernet Configuration --"
echo " "
echo "################# ifconfig -a #################"
echo " "
ifconfig -a 2>/dev/null
} >> $OUT_FILE
exit 0

52
oamapps/columnstoreSupport/logReport.sh
View File

@ -1,52 +0,0 @@
#! /bin/sh
#
# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
#get temp directory
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/${MODULE}_logReport.tar.gz
tar -zcf ${tmpDir}/${MODULE}_logReport.tar.gz /var/log/mariadb/columnstore > /dev/null 2>&1
cp ${tmpDir}/${MODULE}_logReport.tar.gz .
tar -zcf ${MODULE}_mysqllogReport.tar.gz /var/log/mysql/*.err 2>/dev/null
echo '******************** Log Configuration ********************' >> $OUT_FILE
echo '' >> $OUT_FILE
echo 'MariaDB ColumnStore System Log Configuration Data' >> $OUT_FILE
echo '' >> $OUT_FILE
configFileName=`mcsGetConfig Installation SystemLogConfigFile`
echo 'System Logging Configuration File being used: '${configFileName} >> $OUT_FILE
echo '' >> $OUT_FILE
echo -e 'Module\tConfigured Log Levels' >> $OUT_FILE
echo -e '------\t---------------------------------------' >> $OUT_FILE
moduleConfig=''
if grep -q '/var/log/mariadb/columnstore/crit.log' ${configFileName}; then
moduleConfig=${moduleConfig}' CRITICAL'
fi
if grep -q '/var/log/mariadb/columnstore/err.log' ${configFileName}; then
moduleConfig=${moduleConfig}' ERROR'
fi
if grep -q '/var/log/mariadb/columnstore/warning.log' ${configFileName}; then
moduleConfig=${moduleConfig}' WARNING'
fi
if grep -q '/var/log/mariadb/columnstore/info.log' ${configFileName}; then
moduleConfig=${moduleConfig}' INFO'
fi
if grep -q '/var/log/mariadb/columnstore/debug.log' ${configFileName}; then
moduleConfig=${moduleConfig}' DEBUG'
fi
echo -e ${MODULE}'\t'${moduleConfig} >> $OUT_FILE
exit 0

621
oamapps/columnstoreSupport/mcsSupportUtil.cpp
View File

@ -1,621 +0,0 @@
/* Copyright (C) 2019 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "mcsSupportUtil.h"
using namespace std;
using namespace oam;
using namespace config;
void getSystemNetworkConfig(FILE* pOutputFile)
{
Oam oam;
// get and display Module Network Config
SystemModuleTypeConfig systemmoduletypeconfig;
systemmoduletypeconfig.moduletypeconfig.clear();
// get max length of a host name for header formatting
int maxSize = 9;
try
{
oam.getSystemConfig(systemmoduletypeconfig);
for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++)
{
if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty())
// end of list
break;
int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount;
string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType;
string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc;
if (moduleCount > 0)
{
DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin();
for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++)
{
HostConfigList::iterator pt1 = (*pt).hostConfigList.begin();
for (; pt1 != (*pt).hostConfigList.end(); pt1++)
{
if (maxSize < (int)(*pt1).HostName.size())
maxSize = (*pt1).HostName.size();
}
}
}
}
}
catch (exception& e)
{
fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what());
}
fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "Module Name", "Module Description", "NIC ID",
"Host Name", "IP Address");
fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "-----------", "-------------------------", "------",
"---------", "---------------");
try
{
oam.getSystemConfig(systemmoduletypeconfig);
for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++)
{
if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty())
// end of list
break;
int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount;
string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType;
string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc;
if (moduleCount > 0)
{
DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin();
for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++)
{
string modulename = (*pt).DeviceName;
string moduleID = modulename.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE);
string modulenamedesc = moduletypedesc + " #" + moduleID;
fprintf(pOutputFile, "%-15s%-30s", modulename.c_str(), modulenamedesc.c_str());
HostConfigList::iterator pt1 = (*pt).hostConfigList.begin();
for (; pt1 != (*pt).hostConfigList.end(); pt1++)
{
/* MCOL-1607. IPAddr may be a host name here b/c it is read straight
from the config file. */
string tmphost = getIPAddress(pt1->IPAddr);
string ipAddr;
if (tmphost.empty())
ipAddr = pt1->IPAddr;
else
ipAddr = tmphost;
string hostname = (*pt1).HostName;
string nicID = oam.itoa((*pt1).NicID);
if (nicID != "1")
{
fprintf(pOutputFile, "%-45s", "");
}
fprintf(pOutputFile, "%-13s%-14s%-20s\n", nicID.c_str(), hostname.c_str(), ipAddr.c_str());
}
}
}
}
}
catch (exception& e)
{
fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what());
}
}
void getModuleTypeConfig(FILE* pOutputFile)
{
Oam oam;
SystemModuleTypeConfig systemmoduletypeconfig;
ModuleTypeConfig moduletypeconfig;
ModuleConfig moduleconfig;
systemmoduletypeconfig.moduletypeconfig.clear();
try
{
oam.getSystemConfig(systemmoduletypeconfig);
fprintf(pOutputFile, "Module Type Configuration\n\n");
for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++)
{
if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty())
// end of list
break;
int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount;
if (moduleCount < 1)
continue;
string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType;
fprintf(pOutputFile, "ModuleType '%s' Configuration information\n", moduletype.c_str());
fprintf(pOutputFile, "ModuleDesc = %s\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc.c_str());
fprintf(pOutputFile, "ModuleCount = %i\n", moduleCount);
if (moduleCount > 0)
{
DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin();
for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++)
{
string modulename = (*pt).DeviceName;
HostConfigList::iterator pt1 = (*pt).hostConfigList.begin();
for (; pt1 != (*pt).hostConfigList.end(); pt1++)
{
string ipAddr = (*pt1).IPAddr;
string servername = (*pt1).HostName;
fprintf(pOutputFile, "ModuleHostName and ModuleIPAddr for NIC ID %u on module '%s' = %s , %s\n",
(*pt1).NicID, modulename.c_str(), servername.c_str(), ipAddr.c_str());
}
}
}
DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin();
for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end(); pt++)
{
if ((*pt).dbrootConfigList.size() > 0)
{
fprintf(pOutputFile, "DBRootIDs assigned to module 'pm%u' = ", (*pt).DeviceID);
DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin();
for (; pt1 != (*pt).dbrootConfigList.end();)
{
fprintf(pOutputFile, "%u", *pt1);
pt1++;
if (pt1 != (*pt).dbrootConfigList.end())
fprintf(pOutputFile, ", ");
}
}
fprintf(pOutputFile, "\n");
}
fprintf(pOutputFile, "ModuleCPUCriticalThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUCriticalThreshold);
fprintf(pOutputFile, "ModuleCPUMajorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMajorThreshold);
fprintf(pOutputFile, "ModuleCPUMinorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorThreshold);
fprintf(pOutputFile, "ModuleCPUMinorClearThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorClearThreshold);
fprintf(pOutputFile, "ModuleDiskCriticalThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskCriticalThreshold);
fprintf(pOutputFile, "ModuleDiskMajorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMajorThreshold);
fprintf(pOutputFile, "ModuleDiskMinorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMinorThreshold);
fprintf(pOutputFile, "ModuleMemCriticalThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleMemCriticalThreshold);
fprintf(pOutputFile, "ModuleMemMajorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMajorThreshold);
fprintf(pOutputFile, "ModuleMemMinorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMinorThreshold);
fprintf(pOutputFile, "ModuleSwapCriticalThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapCriticalThreshold);
fprintf(pOutputFile, "ModuleSwapMajorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMajorThreshold);
fprintf(pOutputFile, "ModuleSwapMinorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMinorThreshold);
DiskMonitorFileSystems::iterator pt2 = systemmoduletypeconfig.moduletypeconfig[i].FileSystems.begin();
int id = 1;
for (; pt2 != systemmoduletypeconfig.moduletypeconfig[i].FileSystems.end(); pt2++)
{
string fs = *pt2;
fprintf(pOutputFile, "ModuleDiskMonitorFileSystem#%i = %s\n", id, fs.c_str());
++id;
}
fprintf(pOutputFile, "\n");
}
}
catch (exception& e)
{
cout << endl << "**** getModuleTypeConfig Failed = " << e.what() << endl;
}
}
void getStorageConfig(FILE* pOutputFile)
{
Oam oam;
try
{
systemStorageInfo_t t;
t = oam.getStorageConfig();
string cloud;
try
{
oam.getSystemConfig("Cloud", cloud);
}
catch (...)
{
}
string::size_type pos = cloud.find("amazon", 0);
if (pos != string::npos)
cloud = "amazon";
fprintf(pOutputFile, "System Storage Configuration\n");
fprintf(pOutputFile, "Performance Module (DBRoot) Storage Type = %s\n", boost::get<0>(t).c_str());
if (cloud == "amazon")
fprintf(pOutputFile, "User Module Storage Type = %s\n", boost::get<3>(t).c_str());
fprintf(pOutputFile, "System Assigned DBRoot Count = %i\n", boost::get<1>(t));
DeviceDBRootList moduledbrootlist = boost::get<2>(t);
typedef std::vector<int> dbrootList;
dbrootList dbrootlist;
DeviceDBRootList::iterator pt = moduledbrootlist.begin();
for (; pt != moduledbrootlist.end(); pt++)
{
fprintf(pOutputFile, "DBRoot IDs assigned to 'pm%u' = ", (*pt).DeviceID);
DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin();
for (; pt1 != (*pt).dbrootConfigList.end();)
{
fprintf(pOutputFile, "%u", *pt1);
dbrootlist.push_back(*pt1);
pt1++;
if (pt1 != (*pt).dbrootConfigList.end())
fprintf(pOutputFile, ", ");
}
fprintf(pOutputFile, "\n");
}
// get any unassigned DBRoots
/*DBRootConfigList undbrootlist;
try
{
oam.getUnassignedDbroot(undbrootlist);
}
catch (...) {}
if ( !undbrootlist.empty() )
{
fprintf(pOutputFile,"DBRoot IDs unassigned = ");
DBRootConfigList::iterator pt1 = undbrootlist.begin();
for ( ; pt1 != undbrootlist.end() ;)
{
fprintf(pOutputFile,"%u",*pt1);
pt1++;
if (pt1 != undbrootlist.end())
fprintf(pOutputFile,", ");
}
fprintf(pOutputFile,"\n");
}*/
fprintf(pOutputFile, "\n");
// um volumes
if (cloud == "amazon" && boost::get<3>(t) == "external")
{
ModuleTypeConfig moduletypeconfig;
oam.getSystemConfig("um", moduletypeconfig);
for (int id = 1; id < moduletypeconfig.ModuleCount + 1; id++)
{
string volumeNameID = "UMVolumeName" + oam.itoa(id);
string volumeName = oam::UnassignedName;
string deviceNameID = "UMVolumeDeviceName" + oam.itoa(id);
string deviceName = oam::UnassignedName;
try
{
oam.getSystemConfig(volumeNameID, volumeName);
oam.getSystemConfig(deviceNameID, deviceName);
}
catch (...)
{
}
fprintf(pOutputFile, "Amazon EC2 Volume Name/Device Name for 'um%i': %s, %s", id, volumeName.c_str(),
deviceName.c_str());
}
}
// pm volumes
if (cloud == "amazon" && boost::get<0>(t) == "external")
{
fprintf(pOutputFile, "\n");
DBRootConfigList dbrootConfigList;
try
{
oam.getSystemDbrootConfig(dbrootConfigList);
DBRootConfigList::iterator pt = dbrootConfigList.begin();
for (; pt != dbrootConfigList.end(); pt++)
{
string volumeNameID = "PMVolumeName" + oam.itoa(*pt);
string volumeName = oam::UnassignedName;
string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt);
string deviceName = oam::UnassignedName;
try
{
oam.getSystemConfig(volumeNameID, volumeName);
oam.getSystemConfig(deviceNameID, deviceName);
}
catch (...)
{
continue;
}
}
}
catch (exception& e)
{
cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl;
}
// print un-assigned dbroots
/*DBRootConfigList::iterator pt1 = undbrootlist.begin();
for ( ; pt1 != undbrootlist.end() ; pt1++)
{
string volumeNameID = "PMVolumeName" + oam.itoa(*pt1);
string volumeName = oam::UnassignedName;
string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt1);
string deviceName = oam::UnassignedName;
try
{
oam.getSystemConfig( volumeNameID, volumeName);
oam.getSystemConfig( deviceNameID, deviceName);
}
catch (...)
{
continue;
}
}*/
}
string DataRedundancyConfig;
int DataRedundancyCopies;
string DataRedundancyStorageType;
try
{
oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig);
oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies);
oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType);
}
catch (...)
{
}
if (DataRedundancyConfig == "y")
{
fprintf(pOutputFile, "\nData Redundant Configuration\n\n");
fprintf(pOutputFile, "Copies Per DBroot = %i", DataRedundancyCopies);
oamModuleInfo_t st;
string moduleType;
try
{
st = oam.getModuleInfo();
moduleType = boost::get<1>(st);
}
catch (...)
{
}
if (moduleType != "pm")
return;
try
{
DBRootConfigList dbrootConfigList;
oam.getSystemDbrootConfig(dbrootConfigList);
DBRootConfigList::iterator pt = dbrootConfigList.begin();
for (; pt != dbrootConfigList.end(); pt++)
{
fprintf(pOutputFile, "DBRoot #%u has copies on PMs = ", *pt);
string pmList = "";
try
{
string errmsg;
// oam.glusterctl(oam::GLUSTER_WHOHAS, oam.itoa(*pt), pmList, errmsg);
}
catch (...)
{
}
boost::char_separator<char> sep(" ");
boost::tokenizer<boost::char_separator<char> > tokens(pmList, sep);
for (boost::tokenizer<boost::char_separator<char> >::iterator it = tokens.begin();
it != tokens.end(); ++it)
{
fprintf(pOutputFile, "%s ", (*it).c_str());
}
fprintf(pOutputFile, "\n");
}
fprintf(pOutputFile, "\n");
}
catch (exception& e)
{
cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl;
}
}
}
catch (exception& e)
{
cout << endl << "**** getStorageConfig Failed : " << e.what() << endl;
}
}
void getStorageStatus(FILE* pOutputFile)
{
Oam oam;
fprintf(pOutputFile, "System External DBRoot Storage Statuses\n\n");
fprintf(pOutputFile, "Component Status Last Status Change\n");
fprintf(pOutputFile, "------------ -------------------------- ------------------------\n");
/*try
{
oam.getSystemStatus(systemstatus, false);
if ( systemstatus.systemdbrootstatus.dbrootstatus.size() == 0 )
{
fprintf(pOutputFile," No External DBRoot Storage Configured\n\n");
return;
}
for ( unsigned int i = 0 ; i < systemstatus.systemdbrootstatus.dbrootstatus.size(); i++)
{
if ( systemstatus.systemdbrootstatus.dbrootstatus[i].Name.empty() )
// end of list
break;
int state = systemstatus.systemdbrootstatus.dbrootstatus[i].OpState;
string stime = systemstatus.systemdbrootstatus.dbrootstatus[i].StateChangeDate ;
stime = stime.substr (0, 24);
fprintf(pOutputFile,"DBRoot%s%-29s%-24s\n",
systemstatus.systemdbrootstatus.dbrootstatus[i].Name.c_str(),
oamState[state].c_str(),
stime.c_str());
}
fprintf(pOutputFile,"\n");
}
catch (exception& e)
{
cout << endl << "**** getSystemStatus Failed = " << e.what() << endl;
}*/
string DataRedundancyConfig;
int DataRedundancyCopies;
string DataRedundancyStorageType;
try
{
oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig);
oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies);
oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType);
}
catch (...)
{
}
}
/********************************************************************
*
* checkLogStatus - Check for a phrase in a log file and return status
*
********************************************************************/
bool checkLogStatus(std::string fileName, std::string phrase)
{
ifstream file(fileName.c_str());
if (!file.is_open())
{
return false;
}
string buf;
while (getline(file, buf))
{
string::size_type pos = buf.find(phrase, 0);
if (pos != string::npos)
// found phrase
return true;
}
if (file.bad())
{
return false;
}
file.close();
return false;
}
/******************************************************************************************
* @brief Get Network IP Address for Host Name
*
* purpose: Get Network IP Address for Host Name
*
******************************************************************************************/
string getIPAddress(string hostName)
{
static uint32_t my_bind_addr;
struct hostent* ent;
string IPAddr = "";
Oam oam;
ent = gethostbyname(hostName.c_str());
if (ent != 0)
{
my_bind_addr = (uint32_t)((in_addr*)ent->h_addr_list[0])->s_addr;
uint8_t split[4];
uint32_t ip = my_bind_addr;
split[0] = (ip & 0xff000000) >> 24;
split[1] = (ip & 0x00ff0000) >> 16;
split[2] = (ip & 0x0000ff00) >> 8;
split[3] = (ip & 0x000000ff);
IPAddr =
oam.itoa(split[3]) + "." + oam.itoa(split[2]) + "." + oam.itoa(split[1]) + "." + oam.itoa(split[0]);
}
return IPAddr;
}

34
oamapps/columnstoreSupport/mcsSupportUtil.h
View File

@ -1,34 +0,0 @@
/* Copyright (C) 2020 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <iostream>
#include <fstream>
#include <cstdlib>
#include <string>
#include <limits.h>
#include <sstream>
#include "configcpp.h"
#include "liboamcpp.h"
void getSystemNetworkConfig(FILE* pOutputFile);
void getModuleTypeConfig(FILE* pOutputFile);
void getStorageConfig(FILE* pOutputFile);
void getStorageStatus(FILE* pOutputFile);
bool checkLogStatus(std::string filename, std::string phase);
std::string getIPAddress(std::string hostName);

66
oamapps/columnstoreSupport/resourceReport.sh
View File

@ -1,66 +0,0 @@
#! /bin/sh
#
# $Id: resourceReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
echo " "
echo "******************** Resource Usage Report for ${MODULE} ********************"
echo " "
echo " "
echo "-- Shared Memory --"
echo " "
echo "################# ipcs -l #################"
echo " "
ipcs -l
echo "################# clearShm -n #################"
echo " "
clearShm -n
echo " "
echo "-- Disk Usage --"
echo " "
echo "################# df -k #################"
echo " "
df -k
echo " "
echo "-- Disk BRM Data files --"
echo " "
ls -l /var/lib/columnstore/data1/systemFiles/dbrm 2> /dev/null
ls -l /var/lib/columnstore/dbrm 2> /dev/null
echo "################# cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current #################"
echo " "
cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current 2> /dev/null
echo " "
echo "-- View Table Locks --"
echo " "
echo "################# cat bin/viewtablelock #################"
echo " "
viewtablelock 2> /dev/null
echo " "
echo "-- BRM Extent Map --"
echo " "
echo "################# bin/editem -i #################"
echo " "
editem -i 2>/dev/null
} >> $OUT_FILE
exit 0

30
oamapps/columnstoreSupport/softwareReport.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#
# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
echo " "
echo "******************** Software Report for ${MODULE} ********************"
echo " "
echo " "
echo "-- Columnstore Package Details --"
echo " "
rpm -qi MariaDB-columnstore-engine
echo " "
} >> $OUT_FILE
exit 0

34
oamapps/replayTransactionLog/CMakeLists.txt
View File

@ -1,34 +0,0 @@
#
# Not used
#
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = ReplayTransactionLog ReplayTransactionLog_SOURCES = replaytransactionlog.cpp
# ReplayTransactionLog_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) ReplayTransactionLog_LDFLAGS =
# @idb_common_ldflags@ @idb_exec_libs@ -lreplaytxnlog $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

156
oamapps/replayTransactionLog/replaytransactionlog.cpp
View File

@ -1,156 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// WWW - Add header comment.
#include <iostream>
#include <string>
#include <sstream>
#include "liboamcpp.h"
using namespace std;
using namespace oam;
#include "replaytxnlog.h"
namespace
{
void usage(char* prog)
{
cout << endl;
cout << "Usage: " << prog << " [options]" << endl;
cout << endl;
cout << "This utility can be used after a backup is restored to report transactions that " << endl;
cout << "occurred after the backup. It begins with the first transaction that was committed " << endl;
cout << "after the backup and reports DDL and DML statements as well as imports." << endl;
cout << endl;
cout << "Options:" << endl;
/*
cout << "-u <user> Database user id." << endl << endl;
cout << "-p <password> Password." << endl << endl;
*/
cout << "-d <stop date> Stop date and time as mm/dd/yy@hh:mm:ss or 'Now'." << endl;
cout << " Only transactions committed before this date and time will be reported." << endl;
cout << " The current date and time will be used if 'Now'." << endl << endl;
/*
cout << "-i Ignore bulk load log entries." << endl;
cout << " The program will pause and prompt at bulk load entries by default." << endl <<
endl;
cout << "-e Report mode. The sql statements will be displayed to the console only. No" <<
endl; cout << " transactions will be processed. The user and password will be ignored." <<
endl << endl;
*/
cout << "-h Display this help." << endl << endl;
}
bool isRunningOnPm()
{
Oam oam;
oamModuleInfo_t t;
string moduleType;
int installType = -1;
char* csc_ident = getenv("CALPONT_CSC_IDENT");
if (csc_ident == 0 || *csc_ident == 0)
{
// get local module info valdiate running on a pm
try
{
t = oam.getModuleInfo();
moduleType = boost::get<1>(t);
installType = boost::get<5>(t);
}
catch (exception& e)
{
moduleType = "pm";
}
}
else
moduleType = csc_ident;
if (installType != oam::INSTALL_COMBINE_DM_UM_PM)
{
if (moduleType != "pm")
{
cerr << "Exiting, ReplayTransactionLog can only be run on a performance module (pm)" << endl;
return false;
}
}
return true;
}
} // namespace
int main(int argc, char** argv)
{
string user;
string password;
string stopDate;
bool ignoreBulk = false;
bool reportMode = false;
char c;
// Invokes member function `int operator ()(void);'
while ((c = getopt(argc, argv, "u:p:d:ihe")) != -1)
{
switch (c)
{
/*
case 'u':
user = optarg;
break;
case 'p':
password = optarg;
break;
*/
case 'd': stopDate = optarg; break;
/*
case 'i':
ignoreBulk = true;
break;
case 'e':
reportMode = true;
break;
*/
case 'h':
usage(argv[0]);
return 0;
break;
default:
usage(argv[0]);
return 1;
break;
}
}
if (!isRunningOnPm())
{
return 0;
}
ReplayTxnLog replayTxnLog(user, password, stopDate, ignoreBulk, reportMode);
replayTxnLog.process();
return 0;
}

66
oamapps/replayTransactionLog/tdriver.cpp
View File

@ -1,66 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
* wweeks@calpont.com *
* *
***************************************************************************/
using namespace std;
#include <iostream>
#include <cstdlib>
#include <sys/types.h>
#include <unistd.h>
#include "sessionmanager.h"
#include <cppunit/extensions/HelperMacros.h>
using namespace execplan;
int maxNewTxns = 1000;
int maxTxns = 1000;
class ExecPlanTest : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(ExecPlanTest);
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp()
{
}
void tearDown()
{
}
}; // test suite
CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest);
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main(int argc, char* argv[])
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest(registry.makeTest());
bool wasSuccessful = runner.run("", false);
return (wasSuccessful ? 0 : 1);
}

34
oamapps/sessionWalker/CMakeLists.txt
View File

@ -1,34 +0,0 @@
#
# Not used
#
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = sessionWalker sessionWalker_SOURCES = sessionwalker.cpp sessionWalker_CPPFLAGS = @idb_common_includes@
# $(AM_CPPFLAGS) sessionWalker_LDFLAGS = @idb_common_ldflags@ @idb_common_libs@ @idb_write_libs@ @netsnmp_libs@
# $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

135
oamapps/sessionWalker/sessionwalker.cpp
View File

@ -1,135 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
* $Id: sessionwalker.cpp 3072 2013-04-04 19:04:45Z rdempsey $
*
* jrodriguez@calpont.com
* *
***************************************************************************/
#include <iostream>
using namespace std;
#include "sessionmonitor.h"
using namespace execplan;
#include "vendordmlstatement.h"
#include "calpontdmlpackage.h"
#include "calpontdmlfactory.h"
using namespace dmlpackage;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
namespace
{
void usage()
{
cout << "sessionwalker [-d|-h]" << endl
<< " -r rollback all transactions found" << endl
<< " -h display this help" << endl;
}
void rollback(const SessionMonitor::MonSIDTIDEntry& txn)
{
VendorDMLStatement dmlStmt("ROLLBACK;", txn.sessionid);
CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
if (pDMLPackage == 0)
{
return;
}
ByteStream bytestream;
pDMLPackage->write(bytestream);
delete pDMLPackage;
MessageQueueClient mq("DMLProc");
try
{
cout << "sending ROLLBACK for sessionID " << txn.sessionid << endl;
mq.write(bytestream);
bytestream = mq.read();
}
catch (...)
{
}
}
} // namespace
int main(int argc, char** argv)
{
bool rflg = false;
opterr = 0;
int c;
while ((c = getopt(argc, argv, "rh")) != EOF)
switch (c)
{
case 'r': rflg = true; break;
case 'h':
usage();
return 0;
break;
default:
usage();
return 1;
break;
}
vector<SessionMonitor::MonSIDTIDEntry*> toTxns;
SessionMonitor* monitor = new SessionMonitor();
toTxns.clear();
toTxns = monitor->timedOutTxns(); // get timed out txns
vector<SessionMonitor::MonSIDTIDEntry*>::iterator iter = toTxns.begin();
vector<SessionMonitor::MonSIDTIDEntry*>::iterator end = toTxns.end();
vector<SessionMonitor::MonSIDTIDEntry*> tmp;
while (iter != end)
{
if ((*iter)->sessionid > 0)
tmp.push_back(*iter);
++iter;
}
toTxns.swap(tmp);
cout << toTxns.size() << " timed out transactions." << endl;
for (unsigned idx = 0; idx < toTxns.size(); idx++)
{
monitor->printTxns(*toTxns[idx]);
if (rflg)
{
rollback(*toTxns[idx]);
}
}
delete monitor;
return 0;
}

189
oamapps/sessionWalker/tdriver.cpp
View File

@ -1,189 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
* jrodriguez@calpont.com *
* *
***************************************************************************/
using namespace std;
#include <iostream>
#include <cstdlib>
#include <sys/types.h>
#include <unistd.h>
#include "sessionmonitor.h"
#include "sessionmanager.h"
#include <cppunit/extensions/HelperMacros.h>
using namespace execplan;
int maxNewTxns = 1000;
int maxTxns = 1000;
class ExecPlanTest : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(ExecPlanTest);
CPPUNIT_TEST(MonitorTestPlan_1);
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp()
{
}
void tearDown()
{
}
int verifyLen;
SessionManager* manager;
SessionManager::TxnID managerTxns[1000];
int createTxns(const int& start, const int& end)
{
int first = start;
int last = end;
int newTxns = 0;
verifyLen = manager->verifySize();
for (int idx = first; idx < last && verifyLen < maxNewTxns; idx++)
{
managerTxns[idx] = manager->newTxnID((uint32_t)idx + 1000);
CPPUNIT_ASSERT(managerTxns[idx].id > 0);
CPPUNIT_ASSERT(managerTxns[idx].valid == true);
verifyLen = manager->verifySize();
CPPUNIT_ASSERT(verifyLen > 0);
newTxns++;
}
CPPUNIT_ASSERT(newTxns == last - first);
return newTxns;
}
int closeTxns(const int& start, const int& end)
{
int first = start;
int last = end;
int totalClosed = 0;
for (int idx = first; idx < last; idx++)
{
try
{
SessionManager::TxnID tmp = manager->getTxnID(idx + 1000);
if (tmp.valid == true)
{
manager->committed(tmp);
CPPUNIT_ASSERT(tmp.valid == false);
totalClosed++;
}
}
catch (exception& e)
{
cerr << e.what() << endl;
continue;
}
}
return totalClosed;
} // closeTxns
void MonitorTestPlan_1()
{
int currStartTxn = 0;
int currEndTxn = 5;
int txnCntIncr = 5;
const int sleepTime = 1;
const int iterMax = 1;
vector<SessionMonitor::MonSIDTIDEntry*> toTxns;
manager = new SessionManager();
// CPPUNIT_ASSERT(manager->verifySize()==0);
SessionMonitor* monitor = NULL;
for (int jdx = 0; jdx < iterMax; jdx++)
{
// store the current state of the SessionManager
monitor = new SessionMonitor();
monitor->AgeLimit(sleepTime);
delete monitor;
int idx = 0;
int grpStart = currStartTxn;
for (idx = 0; idx < 3; idx++)
{
createTxns(currStartTxn, currEndTxn);
// CPPUNIT_ASSERT(manager->verifySize()==(idx+1)*txnCntIncr);
currStartTxn += txnCntIncr;
currEndTxn += txnCntIncr;
sleep(sleepTime + 1); // make sessions time out
monitor = new SessionMonitor(); // read Monitor data
monitor->AgeLimit(sleepTime);
toTxns.clear();
toTxns = monitor->timedOutTxns(); // get timed out txns
CPPUNIT_ASSERT(toTxns.size() == (uint32_t)txnCntIncr * idx);
delete monitor;
}
int grpEnd = currEndTxn;
monitor = new SessionMonitor();
monitor->AgeLimit(sleepTime);
closeTxns(grpStart, grpEnd); // close this iteration of txns
// CPPUNIT_ASSERT(manager->verifySize()==0);
toTxns = monitor->timedOutTxns(); // get timed out txns
CPPUNIT_ASSERT(toTxns.size() == 0);
delete monitor;
}
monitor = new SessionMonitor(); // readload Monitor data
monitor->AgeLimit(sleepTime - 1);
toTxns.clear();
toTxns = monitor->timedOutTxns(); // get timed out txns
CPPUNIT_ASSERT(toTxns.size() == 0);
delete monitor;
// CPPUNIT_ASSERT(manager->verifySize()==0);
delete manager;
}
}; // test suite
CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest);
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main(int argc, char* argv[])
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest(registry.makeTest());
bool wasSuccessful = runner.run("", false);
return (wasSuccessful ? 0 : 1);
}

2
primitives/blockcache/CMakeLists.txt
View File

@ -14,4 +14,4 @@ set(dbbc_STAT_SRCS
fsutils.cpp fsutils.cpp
) )
columnstore_static_library(dbbc ${dbbc_STAT_SRCS}) columnstore_static_library(dbbc ${dbbc_STAT_SRCS})
columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp) columnstore_link(dbbc loggingcpp)

2
primitives/linux-port/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc)
set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp) set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp)
columnstore_static_library(processor ${processor_STAT_SRCS}) columnstore_static_library(processor ${processor_STAT_SRCS})
columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp) columnstore_link(processor loggingcpp)

1
primitives/linux-port/column.cpp
View File

@ -40,7 +40,6 @@ using namespace boost;
#include "simd_sse.h" #include "simd_sse.h"
#include "simd_arm.h" #include "simd_arm.h"
#include "utils/common/columnwidth.h" #include "utils/common/columnwidth.h"
#include "utils/common/bit_cast.h"
#include "exceptclasses.h" #include "exceptclasses.h"

13
primitives/primproc/CMakeLists.txt
View File

@ -3,25 +3,26 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../linux-port)
# ########## next target ############### # ########## next target ###############
set(PrimProc_SRCS set(PrimProc_SRCS
primproc.cpp activestatementcounter.cpp
batchprimitiveprocessor.cpp batchprimitiveprocessor.cpp
bppseeder.cpp bppseeder.cpp
bppsendthread.cpp bppsendthread.cpp
columncommand.cpp columncommand.cpp
command.cpp command.cpp
dictstep.cpp dictstep.cpp
femsghandler.cpp
filtercommand.cpp filtercommand.cpp
logger.cpp logger.cpp
passthrucommand.cpp passthrucommand.cpp
primitiveserver.cpp primitiveserver.cpp
primproc.cpp
pseudocc.cpp pseudocc.cpp
rssmonfcn.cpp
rtscommand.cpp rtscommand.cpp
umsocketselector.cpp samenodepseudosocket.cpp
serviceexemgr.cpp serviceexemgr.cpp
sqlfrontsessionthread.cpp sqlfrontsessionthread.cpp
rssmonfcn.cpp umsocketselector.cpp
activestatementcounter.cpp
femsghandler.cpp
../../utils/common/crashtrace.cpp ../../utils/common/crashtrace.cpp
) )
@ -31,11 +32,11 @@ target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS})
columnstore_link( columnstore_link(
PrimProc PrimProc
${ENGINE_LDFLAGS} ${ENGINE_LDFLAGS}
${NETSNMP_LIBRARIES}
${ENGINE_WRITE_LIBS} ${ENGINE_WRITE_LIBS}
threadpool threadpool
cacheutils cacheutils
dbbc dbbc
processor processor
loggingcpp loggingcpp
statistics_manager
) )

0
utils/messageqcpp/samenodepseudosocket.cpp → primitives/primproc/samenodepseudosocket.cpp
View File

0
utils/messageqcpp/samenodepseudosocket.h → primitives/primproc/samenodepseudosocket.h
View File

2
primitives/primproc/serviceexemgr.cpp
View File

@ -78,7 +78,7 @@
#include "dbrm.h" #include "dbrm.h"
#include "mariadb_my_sys.h" #include "mariadb_my_sys.h"
#include "statistics.h" #include "statistics_manager/statistics.h"
#include "serviceexemgr.h" #include "serviceexemgr.h"
#include "sqlfrontsessionthread.h" #include "sqlfrontsessionthread.h"

4
primitives/primproc/serviceexemgr.h
View File

@ -58,7 +58,7 @@
#include "dbrm.h" #include "dbrm.h"
#include "mariadb_my_sys.h" #include "mariadb_my_sys.h"
#include "statistics.h" #include "statistics_manager/statistics.h"
namespace exemgr namespace exemgr
{ {
@ -69,7 +69,7 @@ class Opt
int m_debug; int m_debug;
bool m_e; bool m_e;
bool m_fg; bool m_fg;
Opt() : m_debug(0), m_e(false), m_fg(false){}; Opt() : m_debug(0), m_e(false), m_fg(false) {};
Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false) Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false)
{ {
int c; int c;

125
primitives/primproc/sqlfrontsessionthread.h
View File

@ -56,76 +56,77 @@
#include "dbrm.h" #include "dbrm.h"
#include "mariadb_my_sys.h" #include "mariadb_my_sys.h"
#include "statistics.h" #include "statistics_manager/statistics.h"
#include "serviceexemgr.h" #include "serviceexemgr.h"
namespace exemgr namespace exemgr
{ {
class SQLFrontSessionThread class SQLFrontSessionThread
{
public:
SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec,
joblist::ResourceManager* rm)
: fIos(ios)
, fEc(ec)
, fRm(rm)
, fStatsRetrieved(false)
, fTeleClient(globServiceExeMgr->getTeleServerParms())
, fOamCachePtr(oam::OamCache::makeOamCache())
{ {
public: }
SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec,
joblist::ResourceManager* rm)
: fIos(ios)
, fEc(ec)
, fRm(rm)
, fStatsRetrieved(false)
, fTeleClient(globServiceExeMgr->getTeleServerParms())
, fOamCachePtr(oam::OamCache::makeOamCache())
{
}
private: private:
messageqcpp::IOSocket fIos; messageqcpp::IOSocket fIos;
joblist::DistributedEngineComm* fEc; joblist::DistributedEngineComm* fEc;
joblist::ResourceManager* fRm; joblist::ResourceManager* fRm;
querystats::QueryStats fStats; querystats::QueryStats fStats;
// Variables used to store return stats // Variables used to store return stats
bool fStatsRetrieved; bool fStatsRetrieved;
querytele::QueryTeleClient fTeleClient; querytele::QueryTeleClient fTeleClient;
oam::OamCache* fOamCachePtr; // this ptr is copyable... oam::OamCache* fOamCachePtr; // this ptr is copyable...
//...Reinitialize stats for start of a new query //...Reinitialize stats for start of a new query
void initStats(uint32_t sessionId, std::string& sqlText) void initStats(uint32_t sessionId, std::string& sqlText)
{ {
initMaxMemPct(sessionId); initMaxMemPct(sessionId);
fStats.reset(); fStats.reset();
fStats.setStartTime(); fStats.setStartTime();
fStats.fSessionID = sessionId; fStats.fSessionID = sessionId;
fStats.fQuery = sqlText; fStats.fQuery = sqlText;
fStatsRetrieved = false; fStatsRetrieved = false;
} }
//...Get % memory usage during latest query for sesssionId. //...Get % memory usage during latest query for sesssionId.
//...SessionId >= 0x80000000 is system catalog query we can ignore. //...SessionId >= 0x80000000 is system catalog query we can ignore.
static uint64_t getMaxMemPct(uint32_t sessionId); static uint64_t getMaxMemPct(uint32_t sessionId);
//...Delete sessionMemMap entry for the specified session's memory % use. //...Delete sessionMemMap entry for the specified session's memory % use.
//...SessionId >= 0x80000000 is system catalog query we can ignore. //...SessionId >= 0x80000000 is system catalog query we can ignore.
static void deleteMaxMemPct(uint32_t sessionId); static void deleteMaxMemPct(uint32_t sessionId);
//...Get and log query stats to specified output stream //...Get and log query stats to specified output stream
const std::string formatQueryStats( const std::string formatQueryStats(
joblist::SJLP& jl, // joblist associated with query joblist::SJLP& jl, // joblist associated with query
const std::string& label, // header label to print in front of log output const std::string& label, // header label to print in front of log output
bool includeNewLine, // include line breaks in query stats std::string bool includeNewLine, // include line breaks in query stats std::string
bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned); bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned);
static void incThreadCntPerSession(uint32_t sessionId); static void incThreadCntPerSession(uint32_t sessionId);
static void decThreadCntPerSession(uint32_t sessionId); static void decThreadCntPerSession(uint32_t sessionId);
//...Init sessionMemMap entry for specified session to 0 memory %. //...Init sessionMemMap entry for specified session to 0 memory %.
//...SessionId >= 0x80000000 is system catalog query we can ignore. //...SessionId >= 0x80000000 is system catalog query we can ignore.
static void initMaxMemPct(uint32_t sessionId); static void initMaxMemPct(uint32_t sessionId);
//... Round off to human readable format (KB, MB, or GB). //... Round off to human readable format (KB, MB, or GB).
const std::string roundBytes(uint64_t value) const; const std::string roundBytes(uint64_t value) const;
void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms); void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms);
void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep, void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep,
boost::shared_ptr<execplan::CalpontSystemCatalog> csc); boost::shared_ptr<execplan::CalpontSystemCatalog> csc);
void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg); void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg);
void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted); void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted);
void analyzeTableHandleStats(messageqcpp::ByteStream& bs); void analyzeTableHandleStats(messageqcpp::ByteStream& bs);
uint64_t roundMB(uint64_t value) const; uint64_t roundMB(uint64_t value) const;
public:
void operator()(); public:
}; void operator()();
} };
} // namespace exemgr

2
storage-manager/CMakeLists.txt
View File

@ -64,7 +64,7 @@ configure_file(
link_directories(${CMAKE_BINARY_DIR}/lib) link_directories(${CMAKE_BINARY_DIR}/lib)
set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
columnstore_library(storagemanager SHARED ${storagemanager_SRCS}) columnstore_library(storagemanager ${storagemanager_SRCS})
add_dependencies(storagemanager marias3 external_boost) add_dependencies(storagemanager marias3 external_boost)
target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)

23
tools/CMakeLists.txt
View File

@ -1,14 +1,15 @@
add_subdirectory(dbbuilder)
add_subdirectory(editem)
add_subdirectory(dbloadxml)
add_subdirectory(getConfig)
add_subdirectory(cplogger)
add_subdirectory(clearShm) add_subdirectory(clearShm)
add_subdirectory(cleartablelock)
add_subdirectory(configMgt)
add_subdirectory(cplogger)
add_subdirectory(dbbuilder)
add_subdirectory(dbloadxml)
add_subdirectory(ddlcleanup)
add_subdirectory(editem)
add_subdirectory(getConfig)
add_subdirectory(idbmeminfo)
add_subdirectory(passwd)
add_subdirectory(rebuildEM)
add_subdirectory(rgprint)
add_subdirectory(setConfig) add_subdirectory(setConfig)
add_subdirectory(viewtablelock) add_subdirectory(viewtablelock)
add_subdirectory(cleartablelock)
add_subdirectory(ddlcleanup)
add_subdirectory(idbmeminfo)
add_subdirectory(rebuildEM)
add_subdirectory(passwd)
add_subdirectory(configMgt)

2
tools/cleartablelock/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp)
columnstore_executable(cleartablelock ${cleartablelock_SRCS}) columnstore_executable(cleartablelock ${cleartablelock_SRCS})
columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})

2
tools/configMgt/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(autoConfigure_SRCS autoConfigure.cpp) set(autoConfigure_SRCS autoConfigure.cpp)
add_executable(autoConfigure ${autoConfigure_SRCS}) add_executable(autoConfigure ${autoConfigure_SRCS})
columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

2
tools/dbbuilder/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp)
columnstore_executable(dbbuilder ${dbbuilder_SRCS}) columnstore_executable(dbbuilder ${dbbuilder_SRCS})
columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})

2
tools/ddlcleanup/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(ddlcleanup_SRCS ddlcleanup.cpp) set(ddlcleanup_SRCS ddlcleanup.cpp)
columnstore_executable(ddlcleanup ${ddlcleanup_SRCS}) columnstore_executable(ddlcleanup ${ddlcleanup_SRCS})
columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil) columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ddlcleanuputil)

2
tools/editem/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(editem_SRCS editem.cpp) set(editem_SRCS editem.cpp)
columnstore_executable(editem ${editem_SRCS}) columnstore_executable(editem ${editem_SRCS})
columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) columnstore_link(editem ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

2
tools/rgprint/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(rgprint_SRCS rgprint.cpp)
columnstore_executable(rgprint ${rgprint_SRCS}) columnstore_executable(rgprint ${rgprint_SRCS})
columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) columnstore_link(rgprint ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})

2
tools/setConfig/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(setConfig_SRCS main.cpp) set(setConfig_SRCS main.cpp)
columnstore_executable(mcsSetConfig ${setConfig_SRCS}) columnstore_executable(mcsSetConfig ${setConfig_SRCS})
columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

44
utils/CMakeLists.txt
View File

@ -1,26 +1,28 @@
# add_subdirectory(boost_idb) # add_subdirectory(boost_idb)
add_subdirectory(startup)
add_subdirectory(common)
add_subdirectory(configcpp)
add_subdirectory(loggingcpp)
add_subdirectory(messageqcpp)
add_subdirectory(threadpool)
add_subdirectory(rwlock)
add_subdirectory(dataconvert)
add_subdirectory(joiner)
add_subdirectory(rowgroup)
add_subdirectory(cacheutils)
add_subdirectory(funcexp)
add_subdirectory(udfsdk)
add_subdirectory(compress)
add_subdirectory(batchloader) add_subdirectory(batchloader)
add_subdirectory(ddlcleanup) add_subdirectory(cacheutils)
add_subdirectory(querystats)
add_subdirectory(windowfunction)
add_subdirectory(idbdatafile)
add_subdirectory(querytele)
add_subdirectory(libmysql_client)
add_subdirectory(regr)
add_subdirectory(cloudio) add_subdirectory(cloudio)
add_subdirectory(common)
add_subdirectory(compress)
add_subdirectory(configcpp)
add_subdirectory(dataconvert)
add_subdirectory(ddlcleanup)
add_subdirectory(funcexp)
add_subdirectory(idbdatafile)
add_subdirectory(joiner)
add_subdirectory(libmarias3) add_subdirectory(libmarias3)
add_subdirectory(libmysql_client)
add_subdirectory(loggingcpp)
add_subdirectory(mariadb_charset)
add_subdirectory(messageqcpp)
add_subdirectory(pron) add_subdirectory(pron)
add_subdirectory(querystats)
add_subdirectory(querytele)
add_subdirectory(regr)
add_subdirectory(rowgroup)
add_subdirectory(rwlock)
add_subdirectory(startup)
add_subdirectory(statistics_manager)
add_subdirectory(threadpool)
add_subdirectory(udfsdk)
add_subdirectory(windowfunction)

0
utils/common/conststring.h → utils/basic/conststring.h
View File

2
utils/batchloader/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(batchloader_LIB_SRCS batchloader.cpp)
columnstore_library(batchloader ${batchloader_LIB_SRCS}) columnstore_library(batchloader ${batchloader_LIB_SRCS})
columnstore_link(batchloader ${NETSNMP_LIBRARIES} loggingcpp) columnstore_link(batchloader oamcpp loggingcpp)

2
utils/cacheutils/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(cacheutils_LIB_SRCS cacheutils.cpp) set(cacheutils_LIB_SRCS cacheutils.cpp)
columnstore_library(cacheutils ${cacheutils_LIB_SRCS}) columnstore_library(cacheutils ${cacheutils_LIB_SRCS})
columnstore_link(cacheutils messageqcpp) columnstore_link(cacheutils messageqcpp boost_thread)

2
utils/cloudio/CMakeLists.txt
View File

@ -14,7 +14,7 @@ columnstore_library(cloudio ${cloudio_LIB_SRCS})
# IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for # IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for
# idbdatafile, we should be able to reverse the dependency like so: # idbdatafile, we should be able to reverse the dependency like so:
columnstore_link(cloudio idbdatafile messageqcpp) columnstore_link(cloudio idbdatafile messageqcpp boost_thread)
add_executable(cloudio_component_test component_test.cpp) add_executable(cloudio_component_test component_test.cpp)
add_executable(cloudio_end_to_end_test end_to_end_test.cpp) add_executable(cloudio_end_to_end_test end_to_end_test.cpp)

13
utils/common/CMakeLists.txt
View File

@ -9,10 +9,17 @@ set(common_LIB_SRCS
MonitorProcMem.cpp MonitorProcMem.cpp
nullvaluemanip.cpp nullvaluemanip.cpp
threadnaming.cpp threadnaming.cpp
utils_utf8.cpp
statistics.cpp
string_prefixes.cpp string_prefixes.cpp
) )
columnstore_library(common ${common_LIB_SRCS}) columnstore_library(common ${common_LIB_SRCS})
columnstore_link(common boost_filesystem configcpp loggingcpp messageqcpp) columnstore_link(
common
PRIVATE
boost_filesystem
configcpp
loggingcpp
messageqcpp
idbdatafile
mariadb_charset
)

39
utils/common/bit_cast.h
View File

@ -1,39 +0,0 @@
/* Copyright (C) 2020 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <type_traits>
#include <cstring>
namespace utils
{
template <class To, class From>
std::enable_if_t<
sizeof(To) == sizeof(From) && std::is_trivially_copyable_v<From> && std::is_trivially_copyable_v<To>, To>
// constexpr support needs compiler magic
bitCast(const From& src) noexcept
{
static_assert(std::is_trivially_constructible_v<To>,
"This implementation additionally requires "
"destination type to be trivially constructible");
To dst;
std::memcpy(&dst, &src, sizeof(To));
return dst;
}
} // namespace utils

6
utils/common/branchpred.h
View File

@ -19,12 +19,6 @@
#pragma once #pragma once
#if !defined(__GNUC__) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
#ifndef __builtin_expect
#define __builtin_expect(x, expected_value) (x)
#endif
#endif
#ifndef LIKELY #ifndef LIKELY
#define LIKELY(x) __builtin_expect((x), 1) #define LIKELY(x) __builtin_expect((x), 1)
#define UNLIKELY(x) __builtin_expect((x), 0) #define UNLIKELY(x) __builtin_expect((x), 0)

3
utils/common/columnwidth.h
View File

@ -17,6 +17,9 @@
#pragma once #pragma once
#include <cstdint>
#include <new>
#include "branchpred.h" #include "branchpred.h"
namespace utils namespace utils

2
utils/common/genericparser.h
View File

@ -17,7 +17,7 @@
#pragma once #pragma once
#include "conststring.h" #include "basic/conststring.h"
namespace genericparser namespace genericparser
{ {

2
utils/common/hashfamily.h
View File

@ -18,7 +18,7 @@
#pragma once #pragma once
#include "hasher.h" #include "hasher.h"
#include "collation.h" #include "mariadb_charset/collation.h"
namespace utils namespace utils
{ {

2
utils/common/nullstring.h
View File

@ -25,7 +25,7 @@
#include <iostream> #include <iostream>
#include <memory> #include <memory>
#include "exceptclasses.h" #include "exceptclasses.h"
#include "conststring.h" #include "basic/conststring.h"
#include "mcs_datatype_basic.h" #include "mcs_datatype_basic.h"
namespace utils namespace utils

2
utils/common/string_prefixes.cpp
View File

@ -18,7 +18,7 @@
/* handling of the conversion of string prefixes to int64_t for quick range checking */ /* handling of the conversion of string prefixes to int64_t for quick range checking */
#include "collation.h" #include "mariadb_charset/collation.h"
#include "joblisttypes.h" #include "joblisttypes.h"
#include "string_prefixes.h" #include "string_prefixes.h"

157
utils/common/syncstream.h
View File

@ -1,157 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/** @file */
/*
* classes isyncstream and osyncstream provide a C++ iostream interface
* for C stdio FILE* streams. The current implementation does not provide
* the necessary methods to support seeking. The I/O buffering of the
* input FILE* is used. The C++ iostream library calls syncbuf::sync()
* for every line, so output buffering is line-by-line.
* */
/*
#include "syncstream.h"
void copyStream(istream& iss, ostream& oss)
{
string line;
getline(iss, line);
while (iss.good())
{
oss << line << endl;
getline(iss, line);
}
}
main()
{
FILE* ifp;
FILE* ofp;
...
isyncstream iss(ifp);
osyncstream oss(ofp);
copyStream(iss, oss);
...
}
*/
#pragma once
#include <iostream>
#include <cstdio>
namespace syncstream
{
/** A streambuf implementation for C stdio FILE* streams.
*
* Adapted from http://www.drdobbs.com/184401305
*/
class syncbuf : public std::streambuf
{
public:
/** ctor */
syncbuf(FILE* f) : std::streambuf(), fptr(f)
{
}
protected:
/** Write character in the case of overflow */
virtual int overflow(int c = EOF)
{
return (c != EOF ? fputc(c, fptr) : EOF);
}
/** Get character in the case of overflow */
virtual int underflow()
{
int c = getc(fptr);
if (c != EOF)
ungetc(c, fptr);
return c;
}
/** Get character in the case of overflow and advance get pointer */
virtual int uflow()
{
return getc(fptr);
}
/** put character back in the case of backup underflow */
virtual int pbackfail(int c = EOF)
{
return (c != EOF ? ungetc(c, fptr) : EOF);
}
/** Synchronize stream buffer */
virtual int sync()
{
return fflush(fptr);
}
private:
FILE* fptr;
};
/** An istream adaptor for input FILE* streams */
class isyncstream : public std::istream
{
public:
/** ctor */
isyncstream() : istream(&buf), buf(0)
{
}
/** ctor */
isyncstream(FILE* fptr) : istream(&buf), buf(fptr)
{
}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
};
/** An ostream adaptor for output FILE* streams */
class osyncstream : public std::ostream
{
public:
/** ctor */
osyncstream() : ostream(&buf), buf(0)
{
}
/** ctor */
osyncstream(FILE* fptr) : ostream(&buf), buf(fptr)
{
}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
};
} // namespace syncstream

8
utils/common/utils_utf8.h
View File

@ -20,16 +20,10 @@
#pragma once #pragma once
#include <cstdint>
#include <string> #include <string>
#if defined(__FreeBSD__)
//#include <cstdlib>
#else
#include <alloca.h>
#endif
#include <cstdlib> #include <cstdlib>
#include <clocale> #include <clocale>
#include "liboamcpp.h"
// Change the name from utf8. Even change the file name to something resembling char helper // Change the name from utf8. Even change the file name to something resembling char helper
namespace utf8 namespace utf8

2
utils/ddlcleanup/CMakeLists.txt
View File

@ -2,4 +2,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(ddlcleanuputil_LIB_SRCS ddlcleanuputil.cpp) set(ddlcleanuputil_LIB_SRCS ddlcleanuputil.cpp)
columnstore_library(ddlcleanuputil ${ddlcleanuputil_LIB_SRCS}) columnstore_library(ddlcleanuputil ${ddlcleanuputil_LIB_SRCS})
columnstore_link(ddlcleanuputil PRIVATE loggingcpp ${NETSNMP_LIBRARIES}) columnstore_link(ddlcleanuputil PRIVATE loggingcpp)

5
utils/funcexp/CMakeLists.txt
View File

@ -150,6 +150,7 @@ columnstore_link(
pron pron
loggingcpp loggingcpp
dataconvert dataconvert
${MARIADB_STRING_LIBS} mariadb_charset
${NETSNMP_LIBRARIES}
) )
columnstore_link(funcexp PRIVATE ${MARIADB_STRING_LIBS})

2
utils/funcexp/functor_json.h
View File

@ -7,7 +7,7 @@
#include <my_sys.h> #include <my_sys.h>
#include <json_lib.h> #include <json_lib.h>
#include "collation.h" #include "mariadb_charset/collation.h"
#include "functor_bool.h" #include "functor_bool.h"
#include "functor_int.h" #include "functor_int.h"
#include "functor_str.h" #include "functor_str.h"

4
utils/funcexp/jsonhelpers.h
View File

@ -10,10 +10,10 @@
#include <my_sys.h> #include <my_sys.h>
// #include <json_lib.h> // #include <json_lib.h>
#include "collation.h" #include "mariadb_charset/collation.h"
#include "functor_json.h" #include "functor_json.h"
#include "functor_str.h" #include "functor_str.h"
#include "collation.h" #include "mariadb_charset/collation.h"
#include "rowgroup.h" #include "rowgroup.h"
#include "treenode.h" #include "treenode.h"
#include "functioncolumn.h" #include "functioncolumn.h"

2
utils/idbdatafile/CMakeLists.txt
View File

@ -14,4 +14,4 @@ set(idbdatafile_LIB_SRCS
) )
columnstore_library(idbdatafile ${idbdatafile_LIB_SRCS}) columnstore_library(idbdatafile ${idbdatafile_LIB_SRCS})
columnstore_link(idbdatafile PRIVATE ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} boost_filesystem boost_system) columnstore_link(idbdatafile PRIVATE ${ENGINE_OAM_LIBS} boost_filesystem boost_system compress)

2
utils/joiner/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(joiner_LIB_SRCS tuplejoiner.cpp joinpartition.cpp) set(joiner_LIB_SRCS tuplejoiner.cpp joinpartition.cpp)
columnstore_library(joiner ${joiner_LIB_SRCS}) columnstore_library(joiner ${joiner_LIB_SRCS})
columnstore_link(joiner PRIVATE loggingcpp) columnstore_link(joiner PRIVATE loggingcpp rowgroup datatypes compress)

7
utils/mariadb_charset/CMakeLists.txt Normal file
View File

@ -0,0 +1,7 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
columnstore_static_library(mariadb_charset charset.cpp)
columnstore_link(mariadb_charset PRIVATE ${MARIADB_STRING_LIBS})
add_dependencies(mariadb_charset loggingcpp)

17
utils/common/utils_utf8.cpp → utils/mariadb_charset/charset.cpp
View File

@ -1,8 +1,9 @@
/* Copyright (C) 2020 MariaDB Corporation. /* Copyright (C) 2025 MariaDB Corporation
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or
it under the terms of the GNU General Public License as published by modify it under the terms of the GNU General Public License
the Free Software Foundation; version 2 of the License. as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -11,11 +12,12 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "utils_utf8.h"
#include "mariadb_charset/collation.h"
#include "mariadb_my_sys.h" #include "mariadb_my_sys.h"
#include "mcs_datatype.h"
namespace datatypes namespace datatypes
{ {
@ -35,4 +37,3 @@ void Charset::setCharset(uint32_t charsetNumber)
} }
} // namespace datatypes } // namespace datatypes

2
utils/common/collation.h → utils/mariadb_charset/collation.h
View File

@ -35,7 +35,7 @@
#include "mcsconfig.h" #include "mcsconfig.h"
#include "exceptclasses.h" #include "exceptclasses.h"
#include "conststring.h" #include "basic/conststring.h"
/* /*
Redefine definitions used by MariaDB m_ctype.h. Redefine definitions used by MariaDB m_ctype.h.

1
utils/messageqcpp/CMakeLists.txt
View File

@ -8,7 +8,6 @@ set(messageqcpp_LIB_SRCS
iosocket.cpp iosocket.cpp
messagequeue.cpp messagequeue.cpp
messagequeuepool.cpp messagequeuepool.cpp
samenodepseudosocket.cpp
socketparms.cpp socketparms.cpp
) )

92
utils/messageqcpp/messagequeuepool.cpp
View File

@ -25,40 +25,57 @@
#include <new> #include <new>
#include <type_traits> #include <type_traits>
namespace messageqcpp namespace messageqcpp
{ {
using ClientMapType = std::multimap<std::string, std::unique_ptr<ClientObject>>; using ClientMapType = std::multimap<std::string, std::unique_ptr<ClientObject>>;
struct LockedClientMap template <class T>
struct Immortal
{ {
LockedClientMap() template <class... Args>
Immortal(Args&&... args)
{ {
::new (space) T(std::forward<Args>(args)...);
} }
~LockedClientMap()
operator T&() & noexcept
{ {
return reinterpret_cast<T&>(space);
} }
ClientMapType clientMap;
std::mutex queueMutex; private:
alignas(T) unsigned char space[sizeof(T)];
}; };
static int clientMapNiftyCounter; class LockedClientMap
static typename std::aligned_storage<sizeof(LockedClientMap), alignof(LockedClientMap)>::type clientMapBuf;
auto& lockedMap = reinterpret_cast<LockedClientMap&>(clientMapBuf);
LockedClientMapInitilizer::LockedClientMapInitilizer ()
{ {
if (clientMapNiftyCounter++ == 0) new (&lockedMap) LockedClientMap (); // placement new struct KeyToUsePrivateCtor
} {
LockedClientMapInitilizer::~LockedClientMapInitilizer () explicit KeyToUsePrivateCtor() = default;
{ };
if (--clientMapNiftyCounter == 0) (&lockedMap)->~LockedClientMap();
}
public:
LockedClientMap(const LockedClientMap&) = delete;
LockedClientMap& operator=(const LockedClientMap&) = delete;
~LockedClientMap() = delete;
static LockedClientMap& getInstance()
{
static Immortal<LockedClientMap> instance(KeyToUsePrivateCtor{});
return instance;
}
LockedClientMap(KeyToUsePrivateCtor)
{
}
private:
ClientMapType clientMap;
std::mutex queueMutex;
friend class MessageQueueClientPool;
};
// 300 seconds idle until cleanup // 300 seconds idle until cleanup
#define MAX_IDLE_TIME 300 #define MAX_IDLE_TIME 300
@ -70,7 +87,7 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts)
MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrIp, uint64_t port) MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrIp, uint64_t port)
{ {
auto lock = std::scoped_lock(lockedMap.queueMutex); auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex);
std::ostringstream oss; std::ostringstream oss;
oss << dnOrIp << "_" << port; oss << dnOrIp << "_" << port;
@ -93,14 +110,13 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrI
newClientObject->client.reset(new MessageQueueClient(dnOrIp, port)); newClientObject->client.reset(new MessageQueueClient(dnOrIp, port));
newClientObject->inUse = true; newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds; newClientObject->lastUsed = nowSeconds;
lockedMap.clientMap.emplace(std::move(searchString), std::move(newClientObject)); LockedClientMap::getInstance().clientMap.emplace(std::move(searchString), std::move(newClientObject));
return newClientObject->client.get(); return newClientObject->client.get();
} }
MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& module) MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& module)
{ {
auto lock = std::scoped_lock(lockedMap.queueMutex); auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex);
MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module); MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module);
@ -116,13 +132,11 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& modul
clock_gettime(CLOCK_MONOTONIC, &now); clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now); uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client.reset(new MessageQueueClient(module)); newClientObject->client.reset(new MessageQueueClient(module));
newClientObject->inUse = true; newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds; newClientObject->lastUsed = nowSeconds;
auto result = newClientObject->client.get(); auto result = newClientObject->client.get();
lockedMap.clientMap.emplace(std::move(module), std::move(newClientObject)); LockedClientMap::getInstance().clientMap.emplace(std::move(module), std::move(newClientObject));
return result; return result;
} }
@ -133,11 +147,10 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search
uint64_t nowSeconds = TimeSpecToSeconds(&now); uint64_t nowSeconds = TimeSpecToSeconds(&now);
MessageQueueClient* returnClient = NULL; MessageQueueClient* returnClient = NULL;
auto it = lockedMap.clientMap.begin(); auto it = LockedClientMap::getInstance().clientMap.begin();
// Scan pool // Scan pool
while (it != lockedMap.clientMap.end()) while (it != LockedClientMap::getInstance().clientMap.end())
{ {
ClientObject* clientObject = it->second.get(); ClientObject* clientObject = it->second.get();
uint64_t elapsedTime = nowSeconds - clientObject->lastUsed; uint64_t elapsedTime = nowSeconds - clientObject->lastUsed;
@ -149,7 +162,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search
// Do this so we don't invalidate current interator // Do this so we don't invalidate current interator
auto toDelete = it; auto toDelete = it;
it++; it++;
lockedMap.clientMap.erase(toDelete); LockedClientMap::getInstance().clientMap.erase(toDelete);
continue; continue;
} }
@ -163,7 +176,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search
// Do this so we don't invalidate current interator // Do this so we don't invalidate current interator
auto toDelete = it; auto toDelete = it;
it++; it++;
lockedMap.clientMap.erase(toDelete); LockedClientMap::getInstance().clientMap.erase(toDelete);
continue; continue;
} }
} }
@ -193,10 +206,10 @@ void MessageQueueClientPool::releaseInstance(MessageQueueClient* client)
if (client == NULL) if (client == NULL)
return; return;
auto lock = std::scoped_lock(lockedMap.queueMutex); auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex);
auto it = lockedMap.clientMap.begin(); auto it = LockedClientMap::getInstance().clientMap.begin();
while (it != lockedMap.clientMap.end()) while (it != LockedClientMap::getInstance().clientMap.end())
{ {
if (it->second->client.get() == client) if (it->second->client.get() == client)
{ {
@ -221,15 +234,14 @@ void MessageQueueClientPool::deleteInstance(MessageQueueClient* client)
if (client == NULL) if (client == NULL)
return; return;
auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex);
auto it = LockedClientMap::getInstance().clientMap.begin();
auto lock = std::scoped_lock(lockedMap.queueMutex); while (it != LockedClientMap::getInstance().clientMap.end())
auto it = lockedMap.clientMap.begin();
while (it != lockedMap.clientMap.end())
{ {
if (it->second->client.get() == client) if (it->second->client.get() == client)
{ {
lockedMap.clientMap.erase(it); LockedClientMap::getInstance().clientMap.erase(it);
return; return;
} }

10
utils/messageqcpp/messagequeuepool.h
View File

@ -26,12 +26,6 @@
namespace messageqcpp namespace messageqcpp
{ {
static struct LockedClientMapInitilizer {
LockedClientMapInitilizer ();
~LockedClientMapInitilizer ();
} clientMapInitilizer; // static initializer for every translation unit
struct ClientObject struct ClientObject
{ {
std::unique_ptr<MessageQueueClient> client; std::unique_ptr<MessageQueueClient> client;
@ -49,8 +43,8 @@ class MessageQueueClientPool
static MessageQueueClient* findInPool(const std::string& search); static MessageQueueClient* findInPool(const std::string& search);
private: private:
MessageQueueClientPool(){}; MessageQueueClientPool() {};
~MessageQueueClientPool(){}; ~MessageQueueClientPool() {};
}; };
} // namespace messageqcpp } // namespace messageqcpp

2
utils/querystats/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(querystats_LIB_SRCS querystats.cpp) set(querystats_LIB_SRCS querystats.cpp)
columnstore_library(querystats ${querystats_LIB_SRCS}) columnstore_library(querystats ${querystats_LIB_SRCS})
columnstore_link(querystats PRIVATE loggingcpp) columnstore_link(querystats PRIVATE loggingcpp messageqcpp)

4
utils/querytele/CMakeLists.txt
View File

@ -5,7 +5,7 @@ set(querytele_LIB_SRCS querytele.cpp queryteleclient.cpp querytele_constants.cpp
) )
columnstore_library(querytele ${querytele_LIB_SRCS}) columnstore_library(querytele ${querytele_LIB_SRCS})
columnstore_link(querytele ${THRIFT_LIBRARY}) columnstore_link(querytele ${THRIFT_LIBRARY} boost_thread)
target_include_directories(querytele PRIVATE ${THRIFT_INCLUDE_DIRS}) target_include_directories(querytele PRIVATE ${THRIFT_INCLUDE_DIRS})
add_dependencies(querytele external_boost external_thrift) add_dependencies(querytele external_thrift)

5
utils/regr/CMakeLists.txt
View File

@ -21,11 +21,12 @@ set(regr_LIB_SRCS
add_definitions(-DMYSQL_DYNAMIC_PLUGIN) add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
columnstore_library(regr ${regr_LIB_SRCS}) columnstore_library(regr ${regr_LIB_SRCS})
columnstore_link(regr PRIVATE loggingcpp) columnstore_link(regr PRIVATE loggingcpp messageqcpp)
set(regr_mysql_LIB_SRCS regrmysql.cpp modamysql.cpp) set(regr_mysql_LIB_SRCS regrmysql.cpp modamysql.cpp)
# Do anyone use it?
columnstore_mysql_plugin_library(regr_mysql SHARED ${regr_mysql_LIB_SRCS}) columnstore_mysql_plugin_library(regr_mysql SHARED ${regr_mysql_LIB_SRCS})
add_dependencies(regr_mysql external_boost) add_dependencies(regr_mysql external_boost GenError) # for "idb_mysql.h" that uses generated mysqld_error.h
set_target_properties(regr_mysql PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../) set_target_properties(regr_mysql PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../)

6
utils/regr/moda.h
View File

@ -45,7 +45,7 @@
#include "calpontsystemcatalog.h" #include "calpontsystemcatalog.h"
#include "windowfunctioncolumn.h" #include "windowfunctioncolumn.h"
#include "hasher.h" #include "hasher.h"
#include "collation.h" #include "mariadb_charset/collation.h"
#define EXPORT #define EXPORT
@ -277,7 +277,7 @@ class Moda_impl_T<string> : public mcsv1_UDAF
{ {
public: public:
// Defaults OK // Defaults OK
Moda_impl_T() : cs(8){}; Moda_impl_T() : cs(8) {};
~Moda_impl_T() override = default; ~Moda_impl_T() override = default;
mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override;
@ -305,7 +305,7 @@ class moda : public mcsv1_UDAF
{ {
public: public:
// Defaults OK // Defaults OK
moda() : mcsv1_UDAF(){}; moda() : mcsv1_UDAF() {};
~moda() override = default; ~moda() override = default;
mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override; mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override;

2
utils/rowgroup/CMakeLists.txt
View File

@ -5,5 +5,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(rowgroup_LIB_SRCS rowaggregation.cpp rowgroup.cpp rowstorage.cpp) set(rowgroup_LIB_SRCS rowaggregation.cpp rowgroup.cpp rowstorage.cpp)
columnstore_library(rowgroup ${rowgroup_LIB_SRCS}) columnstore_library(rowgroup ${rowgroup_LIB_SRCS})
columnstore_link(rowgroup PRIVATE ${NETSNMP_LIBRARIES} funcexp loggingcpp) columnstore_link(rowgroup PRIVATE funcexp loggingcpp compress)
add_dependencies(rowgroup external_boost) add_dependencies(rowgroup external_boost)

2
utils/rowgroup/rowgroup.h
View File

@ -53,7 +53,7 @@
#include "branchpred.h" #include "branchpred.h"
#include "datatypes/mcs_int128.h" #include "datatypes/mcs_int128.h"
#include "collation.h" #include "mariadb_charset/collation.h"
#include "common/hashfamily.h" #include "common/hashfamily.h"
#include "buffertypes.h" #include "buffertypes.h"

6
utils/statistics_manager/CMakeLists.txt Normal file
View File

@ -0,0 +1,6 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
columnstore_static_library(statistics_manager statistics.cpp)
columnstore_link(statistics_manager PRIVATE rowgroup)

5
utils/common/statistics.cpp → utils/statistics_manager/statistics.cpp
View File

@ -116,9 +116,8 @@ void StatisticsManager::analyzeSample(bool traceOn)
// MCV statistics. // MCV statistics.
std::vector<pair<uint64_t, uint32_t>> mcvList(columnMCV.begin(), columnMCV.end()); std::vector<pair<uint64_t, uint32_t>> mcvList(columnMCV.begin(), columnMCV.end());
std::sort(mcvList.begin(), mcvList.end(), std::sort(mcvList.begin(), mcvList.end(),
[](const std::pair<uint64_t, uint32_t>& a, const std::pair<uint64_t, uint32_t>& b) { [](const std::pair<uint64_t, uint32_t>& a, const std::pair<uint64_t, uint32_t>& b)
return a.second > b.second; { return a.second > b.second; });
});
// 200 buckets as Microsoft does. // 200 buckets as Microsoft does.
const auto mcvSize = std::min(columnMCV.size(), static_cast<uint64_t>(200)); const auto mcvSize = std::min(columnMCV.size(), static_cast<uint64_t>(200));

0
utils/common/statistics.h → utils/statistics_manager/statistics.h
View File

4
utils/threadpool/CMakeLists.txt
View File

@ -2,6 +2,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp fair_threadpool.cpp) set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp fair_threadpool.cpp)
columnstore_library(threadpool ${threadpool_LIB_SRCS}) columnstore_library(threadpool ${threadpool_LIB_SRCS})
columnstore_link(threadpool PRIVATE boost_chrono loggingcpp) columnstore_link(threadpool PRIVATE boost_chrono boost_thread loggingcpp messageqcpp)
add_dependencies(threadpool external_boost)

4
utils/udfsdk/CMakeLists.txt
View File

@ -11,9 +11,11 @@ set(udfsdk_LIB_SRCS
) )
columnstore_library(udfsdk ${udfsdk_LIB_SRCS}) columnstore_library(udfsdk ${udfsdk_LIB_SRCS})
columnstore_link(udfsdk PRIVATE loggingcpp) columnstore_link(udfsdk PRIVATE loggingcpp messageqcpp)
# Do anyone use it? # Do anyone use it?
add_definitions(-DMYSQL_DYNAMIC_PLUGIN) add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
set(udf_mysql_LIB_SRCS udfmysql.cpp) set(udf_mysql_LIB_SRCS udfmysql.cpp)
columnstore_mysql_plugin_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS}) columnstore_mysql_plugin_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS})
add_dependencies(udf_mysql GenError) # for "idb_mysql.h" that uses generated mysqld_error.h

28
versioning/BRM/CMakeLists.txt
View File

@ -34,7 +34,7 @@ set(brm_LIB_SRCS
) )
columnstore_library(brm ${brm_LIB_SRCS}) columnstore_library(brm ${brm_LIB_SRCS})
columnstore_link(brm loggingcpp datatypes) columnstore_link(brm loggingcpp datatypes oamcpp boost_thread messageqcpp)
# ########## next target ############### # ########## next target ###############
@ -62,53 +62,41 @@ columnstore_link(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS
set(reset_locks_SRCS reset_locks.cpp) set(reset_locks_SRCS reset_locks.cpp)
columnstore_executable(reset_locks ${reset_locks_SRCS}) columnstore_executable(reset_locks ${reset_locks_SRCS})
columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
# ########## next target ############### # ########## next target ###############
set(rollback_SRCS rollback.cpp) set(rollback_SRCS rollback.cpp)
columnstore_executable(rollback ${rollback_SRCS}) columnstore_executable(rollback ${rollback_SRCS})
columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
# ########## next target ############### # ########## next target ###############
set(save_brm_SRCS save_brm.cpp) set(save_brm_SRCS save_brm.cpp)
columnstore_executable(save_brm ${save_brm_SRCS}) columnstore_executable(save_brm ${save_brm_SRCS})
columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
# ########## next target ############### # ########## next target ###############
set(load_brm_SRCS load_brm.cpp) set(load_brm_SRCS load_brm.cpp)
columnstore_executable(load_brm ${load_brm_SRCS}) columnstore_executable(load_brm ${load_brm_SRCS})
columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
columnstore_executable(mcs-load-em load_em.cpp) columnstore_executable(mcs-load-em load_em.cpp)
columnstore_link( columnstore_link(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}
)
columnstore_executable(mcs-load-brm-from-file load_brm_from_file.cpp) columnstore_executable(mcs-load-brm-from-file load_brm_from_file.cpp)
columnstore_link( columnstore_link(
mcs-load-brm-from-file mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}
${ENGINE_LDFLAGS}
${MARIADB_CLIENT_LIBS}
${ENGINE_OAM_LIBS}
${ENGINE_EXEC_LIBS}
${NETSNMP_LIBRARIES}
boost_program_options boost_program_options
) )
columnstore_executable(mcs-shmem-locks shmem_locks.cpp) columnstore_executable(mcs-shmem-locks shmem_locks.cpp)
columnstore_link( columnstore_link(
mcs-shmem-locks mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}
${ENGINE_LDFLAGS}
${MARIADB_CLIENT_LIBS}
${ENGINE_OAM_LIBS}
${ENGINE_EXEC_LIBS}
${NETSNMP_LIBRARIES}
boost_program_options boost_program_options
) )

6
writeengine/bulk/CMakeLists.txt
View File

@ -29,7 +29,7 @@ set(we_bulk_STAT_SRCS
add_definitions(-D_FILE_OFFSET_BITS=64) add_definitions(-D_FILE_OFFSET_BITS=64)
columnstore_static_library(we_bulk ${we_bulk_STAT_SRCS}) columnstore_static_library(we_bulk ${we_bulk_STAT_SRCS})
columnstore_link(we_bulk ${NETSNMP_LIBRARIES} loggingcpp boost_program_options) columnstore_link(we_bulk loggingcpp boost_program_options)
remove_definitions(-D_FILE_OFFSET_BITS=64) remove_definitions(-D_FILE_OFFSET_BITS=64)
@ -38,13 +38,13 @@ remove_definitions(-D_FILE_OFFSET_BITS=64)
set(cpimport.bin_SRCS cpimport.cpp) set(cpimport.bin_SRCS cpimport.cpp)
columnstore_executable(cpimport.bin ${cpimport.bin_SRCS}) columnstore_executable(cpimport.bin ${cpimport.bin_SRCS})
add_dependencies(cpimport.bin marias3)
columnstore_link( columnstore_link(
cpimport.bin cpimport.bin
${ENGINE_LDFLAGS} ${ENGINE_LDFLAGS}
${NETSNMP_LIBRARIES}
${ENGINE_WRITE_LIBS} ${ENGINE_WRITE_LIBS}
${S3API_DEPS} ${S3API_DEPS}
we_bulk we_bulk
we_xml we_xml
marias3
) )

Some files were not shown because too many files have changed in this diff Show More