database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Deep build refactoring phase 2 (#3564)

Browse Source 
* configcpp refactored

* chore(build): massive removals, auto add files to debian install file

* chore(build): configure before autobake

* chore(build): use custom cmake commands for components, mariadb-plugin-columnstore.install generated

* chore(build): install deps as separate step for build-packages

* more deps

* chore(codemanagement, build): build refactoring stage2

* chore(safety): Locked Map for MessageqCpp with a simpler way

 Please enter the commit message for your changes. Lines starting

* chore(codemanagement, ci): better coredumps handling, deps fixed

* Delete build/bootstrap_mcs.py

* Update charset.cpp (add license)
This commit is contained in:
Leonid Fedorov
2025-07-17 16:14:10 +04:00
committed by GitHub
parent d0ee5dae32
commit 449029a827
107 changed files with 354 additions and 3327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -88,8 +88,3 @@ set(COMPONENTS
foreach(component ${COMPONENTS})
add_subdirectory(${component})
endforeach()
add_dependencies(udf_mysql GenError)
add_dependencies(funcexp GenError)
add_dependencies(oamcpp GenError)
add_dependencies(regr_mysql GenError)

19
cmake/ColumnstoreLibrary.cmake
View File

@ -82,6 +82,16 @@ macro(columnstore_shared_library libname)
columnstore_install_target(${libname} ${ENGINE_LIBDIR})
endmacro()
macro(columnstore_static_library libname)
add_definitions(-fPIC -DPIC)
add_library(${libname} STATIC ${ARGN})
endmacro()
macro(columnstore_shared_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${ENGINE_LIBDIR})
endmacro()
macro(columnstore_library libname)
if(COLUMNSTORE_STATIC_LIBRARIES)
columnstore_static_library(${libname} ${ARGN})
@ -99,6 +109,15 @@ macro(columnstore_link libname)
target_link_libraries(${libname} ${ARGN})
endmacro()
macro(columnstore_mysql_plugin_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${MARIADB_PLUGINDIR})
endmacro()
macro(columnstore_link libname)
target_link_libraries(${libname} ${ARGN})
endmacro()
macro(columnstore_executable executable_name)
add_executable(${executable_name} ${ARGN})
columnstore_install_target(${executable_name} ${ENGINE_BINDIR})

21
datatypes/mcs_datatype.h
View File

@ -20,7 +20,7 @@
#include <sstream>
#include <boost/any.hpp>
#include "exceptclasses.h"
#include "conststring.h"
#include "basic/conststring.h"
#include "mcs_datatype_basic.h"
#include "mcs_numeric_limits.h"
#include "mcs_data_condition.h"
@ -796,7 +796,7 @@ class MinMaxPartitionInfo : public MinMaxInfo
uint64_t m_status;
public:
MinMaxPartitionInfo() : m_status(0){};
MinMaxPartitionInfo() : m_status(0) {};
explicit MinMaxPartitionInfo(const BRM::EMEntry& entry);
void set_invalid()
{
@ -1102,7 +1102,8 @@ class TypeHandlerBit : public TypeHandler
idbassert(0); // QQ
return 1;
}
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override
std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{
return "0"; // QQ
}
@ -1135,7 +1136,7 @@ class TypeHandlerBit : public TypeHandler
const ConvertFromStringParam& /*prm*/, const std::string& /*str*/,
bool& /*pushWarning*/) const override;
const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& /*attr*/) const override
const uint8_t* getEmptyValueForType(const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{
idbassert(0);
return nullptr;
@ -2033,7 +2034,8 @@ class TypeHandlerReal : public TypeHandler
{
return {}; // QQ: real types were not handled in IDB_format()
}
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override
std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{
return "0"; // QQ
}
@ -2329,7 +2331,8 @@ class TypeHandlerBlob : public TypeHandlerStr
{
return storeValueToFieldBlobText(row, pos, f);
}
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override
std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{
return "0"; // QQ
}
@ -2355,7 +2358,8 @@ class TypeHandlerText : public TypeHandlerStr
{
return storeValueToFieldBlobText(row, pos, f);
}
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override
std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{
return "0"; // QQ
}
@ -2386,7 +2390,8 @@ class TypeHandlerClob : public TypeHandlerStr
idbassert(0); // QQ
return 1;
}
std::string format(const SimpleValue& /*v*/, const SystemCatalog::TypeAttributesStd& /*attr*/) const override
std::string format(const SimpleValue& /*v*/,
const SystemCatalog::TypeAttributesStd& /*attr*/) const override
{
return "0"; // QQ
}

4
datatypes/mcs_string.h
View File

@ -17,8 +17,8 @@
#pragma once
#include "conststring.h"
#include "collation.h" // class Charset
#include "basic/conststring.h"
#include "mariadb_charset/collation.h" // class Charset
namespace datatypes
{

2
dbcon/ddlpackage/CMakeLists.txt
View File

@ -42,4 +42,4 @@ columnstore_library(
${FLEX_ddl_scan_OUTPUTS}
)
columnstore_link(ddlpackage loggingcpp)
columnstore_link(ddlpackage loggingcpp messageqcpp)

4
dbcon/ddlpackage/sqlparser.h
View File

@ -27,7 +27,7 @@
*/
#include <stdexcept>
#include "collation.h" // CHARSET_INFO
#include "mariadb_charset/collation.h" // CHARSET_INFO
#include "ddlpkg.h"
#include "mariadb_my_sys.h" // myf, MYF()
@ -87,7 +87,7 @@ struct pass_to_bison
myf utf8_flag;
pass_to_bison(ParseTree* pt)
: fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)){};
: fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)) {};
};
class SqlParser

2
dbcon/ddlpackageproc/CMakeLists.txt
View File

@ -13,4 +13,4 @@ set(ddlpackageproc_LIB_SRCS
)
columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS})
columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)
columnstore_link(ddlpackageproc loggingcpp oamcpp messageqcpp)

2
dbcon/dmlpackage/CMakeLists.txt
View File

@ -41,4 +41,4 @@ columnstore_library(
${FLEX_dml_scan_OUTPUTS}
)
columnstore_link(dmlpackage loggingcpp)
columnstore_link(dmlpackage loggingcpp messageqcpp)

2
dbcon/dmlpackageproc/CMakeLists.txt
View File

@ -13,4 +13,4 @@ set(dmlpackageproc_LIB_SRCS
)
columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS})
columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)
columnstore_link(dmlpackageproc loggingcpp oamcpp messageqcpp)

10
dbcon/execplan/CMakeLists.txt
View File

@ -46,4 +46,12 @@ set(execplan_LIB_SRCS
)
columnstore_library(execplan ${execplan_LIB_SRCS})
columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp)
columnstore_link(
execplan
messageqcpp
${ENGINE_DT_LIB}
pron
oamcpp
loggingcpp
querytele
)

18
dbcon/execplan/calpontsystemcatalog.h
View File

@ -49,7 +49,7 @@
#undef max
#include "mcs_datatype.h"
#include "collation.h" // CHARSET_INFO, class Charset
#include "mariadb_charset/collation.h" // CHARSET_INFO, class Charset
#include "nullstring.h"
class ExecPlanTest;
@ -75,7 +75,8 @@ const int32_t IDB_VTABLE_ID = CNX_VTABLE_ID;
/**
* A struct to hold a list of table partitions.
*/
struct Partitions {
struct Partitions
{
std::vector<std::string> fPartNames;
void serialize(messageqcpp::ByteStream& b) const
{
@ -96,12 +97,11 @@ struct Partitions {
b >> t;
fPartNames.push_back(t);
}
}
};
bool operator <(const Partitions& a, const Partitions& b);
bool operator ==(const Partitions& a, const Partitions& b);
bool operator !=(const Partitions& a, const Partitions& b); // for GCC9
bool operator<(const Partitions& a, const Partitions& b);
bool operator==(const Partitions& a, const Partitions& b);
bool operator!=(const Partitions& a, const Partitions& b); // for GCC9
/** The CalpontSystemCatalog class
*
@ -259,9 +259,9 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog
public:
ColType() = default;
ColType(const ColType& rhs);
ColType(int32_t colWidth_, int32_t scale_, int32_t precision_,
const ConstraintType& constraintType_, const DictOID& ddn_, int32_t colPosition_,
int32_t compressionType_, OID columnOID_, const ColDataType& colDataType_);
ColType(int32_t colWidth_, int32_t scale_, int32_t precision_, const ConstraintType& constraintType_,
const DictOID& ddn_, int32_t colPosition_, int32_t compressionType_, OID columnOID_,
const ColDataType& colDataType_);
ColType& operator=(const ColType& rhs);
CHARSET_INFO* getCharset() const;

2
dbcon/execplan/clientrotator.cpp
View File

@ -47,8 +47,6 @@ using namespace logging;
#include "clientrotator.h"
//#include "idb_mysql.h"
/** Debug macro */
#ifdef INFINIDB_DEBUG
#define IDEBUG(x) \

2
dbcon/execplan/predicateoperator.h
View File

@ -39,7 +39,7 @@
#include "returnedcolumn.h"
#include "dataconvert.h"
#include "collation.h" // CHARSET_INFO
#include "mariadb_charset/collation.h" // CHARSET_INFO
namespace messageqcpp
{

2
dbcon/execplan/pseudocolumn.cpp
View File

@ -28,7 +28,7 @@ using namespace std;
#include "basic/string_utils.h"
#include "bytestream.h"
#include "collation.h"
#include "mariadb_charset/collation.h"
using namespace messageqcpp;

10
dbcon/joblist/CMakeLists.txt
View File

@ -68,7 +68,15 @@ columnstore_library(joblist ${joblist_LIB_SRCS})
target_include_directories(
joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR}
)
columnstore_link(joblist loggingcpp)
columnstore_link(
joblist
loggingcpp
boost_thread
oamcpp
querytele
messageqcpp
statistics_manager
)
if(WITH_ORDERBY_UT)
columnstore_executable(job_orderby_tests orderby-tests.cpp)

2
dbcon/joblist/jlf_tuplejoblist.cpp
View File

@ -71,7 +71,7 @@ using namespace dataconvert;
#include "jlf_tuplejoblist.h"
using namespace joblist;
#include "statistics.h"
#include "statistics_manager/statistics.h"
#ifdef __clang__
#pragma clang diagnostic push

4
dbcon/mysql/CMakeLists.txt
View File

@ -63,7 +63,7 @@ if(COMMAND mysql_add_plugin)
${PLUGIN_EXEC_LIBS}
${PLUGIN_WRITE_LIBS}
joblist_server
${NETSNMP_LIBRARIES}
statistics_manager
${MARIADB_CLIENT_LIBS}
${S3API_DEPS}
threadpool
@ -89,10 +89,10 @@ else()
${S3API_DEPS}
${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS}
${NETSNMP_LIBRARIES}
${SERVER_BUILD_DIR}/libservices/libmysqlservices.a
threadpool
loggingcpp
statistics_manager
marias3
)
# define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist)

2
dbcon/mysql/ha_mcs_impl.cpp
View File

@ -128,9 +128,9 @@ using namespace funcexp;
#include "ha_mcs_sysvars.h"
#include "ha_mcs_datatype.h"
#include "statistics.h"
#include "ha_mcs_logging.h"
#include "ha_subquery.h"
#include "statistics_manager/statistics.h"
namespace cal_impl_if
{

15
dbcon/mysql/idb_mysql.h
View File

@ -14,10 +14,17 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// One include file to deal with all the MySQL pollution of the
// global namespace
//
// Don't include ANY mysql headers anywhere except here!
/* One include file to deal with all the MySQL pollution of the
global namespace
Don't include ANY mysql headers anywhere except here!
WARN: if any cmake build target uses this include file,
GenError from server must be added to the target dependencies
to generate mysqld_error.h used below
*/
#pragma once
#ifdef TEST_MCSCONFIG_H

2
ddlproc/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp)
columnstore_executable(DDLProc ${DDLProc_SRCS})
columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp)
columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool loggingcpp)

1
dmlproc/CMakeLists.txt
View File

@ -12,7 +12,6 @@ columnstore_link(
DMLProc
${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS}
${NETSNMP_LIBRARIES}
threadpool
ddlcleanuputil
batchloader

2
oam/oamcpp/CMakeLists.txt
View File

@ -5,6 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp)
columnstore_library(oamcpp ${oamcpp_LIB_SRCS})
columnstore_link(oamcpp loggingcpp)
columnstore_link(oamcpp loggingcpp idbboot)
target_compile_options(oamcpp PRIVATE -Wno-unused-result)

16
oamapps/columnstoreSupport/CMakeLists.txt
View File

@ -1,16 +0,0 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp)
columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS})
target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result)
columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(configReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(logReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR})

39
oamapps/columnstoreSupport/alarmReport.sh
View File

@ -1,39 +0,0 @@
#! /bin/sh
#
# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
SERVER=$1
else
SERVER="localhost"
fi
if [ $2 ] ; then
DATE=$2
else
DATE=" "
fi
#get temp directory
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/logReport.log
{
echo " "
echo "******************** Alarm Report for $SERVER ********************"
echo " "
echo "-- Today's Alarms --"
echo " "
cat /var/log/mariadb/columnstore/alarm.log 2>/dev/null
if test -f /var/log/mariadb/columnstore/archive/alarm.log-$DATE ; then
echo "-- Archived Alarms --"
echo " "
cat /var/log/mariadb/columnstore/archive/alarm.log-$DATE 2>/dev/null
fi
} > ${tmpDir}/logReport.log
exit 0

60
oamapps/columnstoreSupport/approximateRowCount.sh
View File

@ -1,60 +0,0 @@
#!/bin/bash
#
# Estimates the row count for a given table. Uses number of extents * 8M for the estimate.
#
#
# Initialize variables.
#
if [ -z "$MYSQLCMD" ]; then
MYSQLCMD="mysql -u root"
fi
#
# Validate that there are two parameters - schema and table.
#
if [ $# -ne 2 ]; then
echo ""
echo "Reports the approximate row count for the given table."
echo ""
echo "Parameters:"
echo " Schema"
echo " Table"
fi
db=$1
table=$2
#
# Validate that the table exists.
#
sql="select count(*) from systable where \`schema\`='$db' and tablename='$table';"
count=`$MYSQLCMD calpontsys --skip-column-names -e "$sql;"`
if [ $count -le 0 ]; then
echo ""
echo "$db.$table does not exist in Columnstore."
echo ""
exit 1
fi
#
# Grab the objectid and column width for a column in the table.
#
sql="select objectid from syscolumn where \`schema\`='$db' and tablename='$table' limit 1;"
objectid=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
sql="select columnlength from syscolumn where objectid=$objectid;"
colWidth=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
#
# Use editem to count the extents.
#
extentCount=`editem -o $objectid | wc -l`
let extentCount-=2 # Take out the 2 extra rows for header and blank line at end.
let approximateRowCount=$extentCount*8192*1024;
echo ""
echo "Approximate row count for $db.$table is $approximateRowCount."
echo ""
exit 0

30
oamapps/columnstoreSupport/bulklogReport.sh
View File

@ -1,30 +0,0 @@
#! /bin/sh
#
# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
if test -d /var/lib/columnstore/data/bulk ; then
echo " "
echo "-- Check for Errors in Bulk Logs --"
echo " "
echo "################# egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err #################"
echo " "
egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err 2>/dev/null
fi
} >> $OUT_FILE
exit 0

897
oamapps/columnstoreSupport/columnstoreSupport.cpp
View File

@ -1,897 +0,0 @@
/* Copyright (C) 2013 Calpont Corp. */
/* Copyright (C) 2016 MariaDB Corporation */
/******************************************************************************************
* $Id: columnstoreSupport.cpp 64 2006-10-12 22:21:51Z dhill $
*
*
*
******************************************************************************************/
/**
* @file
*/
#include <iterator>
#include <numeric>
#include <deque>
#include <iostream>
#include <ostream>
#include <fstream>
#include <cstdlib>
#include <string>
#include <limits.h>
#include <sstream>
#include <exception>
#include <stdexcept>
#include <vector>
#include "stdio.h"
#include "ctype.h"
#include <netdb.h>
#include <readline.h>
#include <boost/filesystem.hpp>
#include "mcsconfig.h"
#include "liboamcpp.h"
#include "configcpp.h"
#include "installdir.h"
#include "mcsSupportUtil.h"
#include "columnstoreversion.h"
using namespace std;
using namespace oam;
using namespace config;
typedef struct Child_Module_struct
{
std::string moduleName;
std::string moduleIP;
std::string hostName;
} ChildModule;
typedef std::vector<ChildModule> ChildModuleList;
string currentDate;
string systemName;
string localModule;
string localModuleHostName;
ChildModuleList childmodulelist;
ChildModuleList parentmodulelist;
ChildModule childmodule;
string rootPassword = "";
string debug_flag = "0";
string mysqlpw = " ";
string tmpDir;
int runningThreads = 0;
pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
typedef boost::tuple<ChildModuleList::iterator, string> threadInfo_t;
bool LOCAL = false;
void* childReportThread(threadInfo_t* st)
{
assert(st);
ChildModuleList::iterator& list = boost::get<0>(*st);
string reportType = boost::get<1>(*st);
string remoteModuleName = (*list).moduleName;
string remoteModuleIP = (*list).moduleIP;
string remoteHostName = (*list).hostName;
pthread_mutex_lock(&mutex1);
runningThreads++;
// cout << "++ " << runningThreads << endl;
pthread_mutex_unlock(&mutex1);
string outputFile;
if (reportType == "log")
{
outputFile = remoteModuleName + "_" + reportType + "Report.tar.gz";
}
else
{
outputFile = remoteModuleName + "_" + reportType + "Report.txt";
FILE* pOutputFile = fopen(outputFile.c_str(), "a");
if (pOutputFile == NULL)
{
printf("Could not open file: %s", outputFile.c_str());
exit(1);
}
fprintf(pOutputFile,
"********************************************************************************\n"
"\n"
" System %s\n"
" columnstoreSupportReport script ran from Module %s on %s\n"
" SoftwareVersion = %s-%s"
"\n"
"********************************************************************************\n"
"\n"
" %s report\n"
"\n"
"********************************************************************************\n",
systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(),
columnstore_release.c_str(), reportType.c_str());
}
cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl;
string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + ";" + reportType + "Report.sh " +
remoteModuleName + "' " + debug_flag + " - forcetty";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << "Error with running remote_command.sh, exiting..." << endl;
}
cmd = "remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile +
" > /dev/null 2>&1";
rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
cout << "ERROR: failed to retrieve " << tmpDir << "/" << outputFile << " from " + remoteHostName << endl;
pthread_mutex_lock(&mutex1);
runningThreads--;
// cout << "-- " << runningThreads << endl;
pthread_mutex_unlock(&mutex1);
// exit thread
pthread_exit(0);
}
void* reportThread(string* reporttype)
{
assert(reporttype);
string reportType = *reporttype;
Oam oam;
pthread_mutex_lock(&mutex1);
runningThreads++;
// cout << "++ " << runningThreads << endl;
pthread_mutex_unlock(&mutex1);
string outputFile = localModule + "_" + reportType + "Report.txt";
FILE* pOutputFile = fopen(outputFile.c_str(), "a");
if (pOutputFile == NULL)
{
printf("Could not open file: %s", outputFile.c_str());
exit(1);
}
// get local report
fprintf(pOutputFile,
"********************************************************************************\n"
"\n"
" System %s\n"
" columnstoreSupportReport script ran from Module %s on %s\n"
" SoftwareVersion = %s-%s"
"\n"
"********************************************************************************\n"
"\n"
" %s report\n"
"\n"
"********************************************************************************\n",
systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(),
columnstore_release.c_str(), reportType.c_str());
fclose(pOutputFile);
// run on child servers and get report
if (!LOCAL)
{
ChildModuleList::iterator list1 = childmodulelist.begin();
for (; list1 != childmodulelist.end(); list1++)
{
threadInfo_t* st = new threadInfo_t;
*st = boost::make_tuple(list1, reportType);
pthread_t childreportthread;
int status = pthread_create(&childreportthread, NULL, (void* (*)(void*)) & childReportThread, st);
if (status != 0)
{
cout << "ERROR: childreportthread: pthread_create failed, return status = " + oam.itoa(status)
<< endl;
}
sleep(1);
}
}
if (reportType == "log")
{
// run log config on local server
cout << "Get log config data for " + localModule << endl;
string cmd = "logReport.sh " + localModule + " " + outputFile;
system(cmd.c_str());
}
else
{
string cmd = reportType + "Report.sh " + localModule + " " + outputFile;
system(cmd.c_str());
if (reportType == "config")
{
pOutputFile = fopen(outputFile.c_str(), "a");
if (pOutputFile == NULL)
{
printf("Could not open file: %s", outputFile.c_str());
exit(1);
}
fprintf(pOutputFile,
"\n******************** System Network Configuration ******************************\n\n");
getSystemNetworkConfig(pOutputFile);
fprintf(pOutputFile,
"\n******************** System Module Configure **********************************\n\n");
getModuleTypeConfig(pOutputFile);
fprintf(pOutputFile,
"\n******************** System Storage Configuration *****************************\n\n");
getStorageConfig(pOutputFile);
fprintf(pOutputFile,
"\n******************** System Storage Status ************************************\n\n");
getStorageStatus(pOutputFile);
// BT: most of this is tedious to collect and can be manually looked up in the debug.log file
// fprintf(pOutputFile,"\n******************** System Status
// ********************************************\n\n"); printSystemStatus(pOutputFile);
// printProcessStatus(pOutputFile);
// printAlarmSummary(pOutputFile);
//
// fprintf(pOutputFile,"\n******************** System Directories
// ***************************************\n\n"); getSystemDirectories(pOutputFile);
boost::filesystem::path configFile =
std::string(MCSSYSCONFDIR) + std::string("/columnstore/Columnstore.xml");
boost::filesystem::copy_file(configFile, "./Columnstore.xml",
boost::filesystem::copy_options::overwrite_existing);
boost::filesystem::path SMconfigFile =
std::string(MCSSYSCONFDIR) + std::string("/columnstore/storagemanager.cnf");
boost::filesystem::copy_file(SMconfigFile, "./storagemanager.cnf",
boost::filesystem::copy_options::overwrite_existing);
system("sed -i 's/.*aws_access_key_id.*/aws_access_key_id={PRIVATE}/' ./storagemanager.cnf");
system("sed -i 's/.*aws_secret_access_key.*/aws_secret_access_key={PRIVATE}/' ./storagemanager.cnf");
fclose(pOutputFile);
}
/*
// TODO: This can be ported from mcsadmin if needed most info included does not seem useful at this time
if (reportType == "resource" )
{
if (LOCAL)
{
fprintf(pOutputFile,"\n******************** mcsadmin getModuleResourceUsage
**************************\n\n"); string cmd = "mcsadmin getModuleResourceUsage " + localModule + " >> " +
outputFile; system(cmd.c_str());
}
else
{
fprintf(pOutputFile,"\n******************** mcsadmin getSystemResourceUsage
**************************\n\n"); string cmd = "mcsadmin getSystemResourceUsage >> " + outputFile;
system(cmd.c_str());
}
}*/
}
// exit thread
pthread_mutex_lock(&mutex1);
runningThreads--;
// cout << "-- " << runningThreads << endl;
pthread_mutex_unlock(&mutex1);
pthread_exit(0);
}
int main(int argc, char* argv[])
{
Oam oam;
Config* sysConfig = Config::makeConfig();
string SystemSection = "SystemConfig";
string InstallSection = "Installation";
bool HARDWARE = false;
bool CONFIG = false;
bool DBMS = false;
bool RESOURCE = false;
bool LOG = false;
bool BULKLOG = false;
bool HADOOP = false;
// get current time and date
time_t now;
now = time(NULL);
struct tm tm;
localtime_r(&now, &tm);
char timestamp[200];
strftime(timestamp, 200, "%m:%d:%y-%H:%M:%S", &tm);
currentDate = timestamp;
char helpArg[3] = "-h";
// Get System Name
try
{
oam.getSystemConfig("SystemName", systemName);
}
catch (...)
{
systemName = "unassigned";
}
// get Local Module Name and Server Install Indicator
string singleServerInstall = "n";
oamModuleInfo_t st;
try
{
st = oam.getModuleInfo();
localModule = boost::get<0>(st);
}
catch (...)
{
cout << endl << "**** Failed : Failed to read Local Module Name" << endl;
exit(-1);
}
if (argc == 1)
{
argv[1] = &helpArg[0];
argc = 2;
}
string DataFilePlugin;
try
{
DataFilePlugin = sysConfig->getConfig(SystemSection, "DataFilePlugin");
}
catch (...)
{
cout << "ERROR: Problem accessing Columnstore configuration file" << endl;
exit(-1);
}
tmpDir = startup::StartUp::tmpDir();
for (int i = 1; i < argc; i++)
{
if (string("-h") == argv[i])
{
cout << endl;
cout << "'columnstoreSupport' generates a Set of System Support Report Files in a tar file" << endl;
cout << "called columnstoreSupportReport.'system-name'.tar.gz in the local directory." << endl;
cout << "It should be run on the server with the DBRM front-end." << endl;
cout << "Check the Admin Guide for additional information." << endl;
cout << endl;
cout << "Usage: columnstoreSupport [-h][-a][-hw][-s][-c][-db][-r][-l][-bl][-lc][-p "
"'root-password'][-de]";
cout << endl;
cout << " -h help" << endl;
cout << " -a Output all Reports (excluding Bulk Logs Reports)" << endl;
cout << " -hw Output Hardware Reports only" << endl;
cout << " -c Output Configuration/Status Reports only" << endl;
cout << " -db Output DBMS Reports only" << endl;
cout << " -r Output Resource Reports only" << endl;
cout << " -l Output Columnstore Log/Alarms Reports only" << endl;
cout << " -bl Output Columnstore Bulk Log Reports only" << endl;
cout << " -lc Output Reports for Local Server only" << endl;
cout << " -p password (multi-server systems), root-password or 'ssh' to use 'ssh keys'"
<< endl;
cout << " -de Debug Flag" << endl;
exit(0);
}
else
{
if (string("-a") == argv[i])
{
HARDWARE = true;
CONFIG = true;
DBMS = true;
RESOURCE = true;
LOG = true;
HADOOP = (DataFilePlugin.empty() ? false : true);
}
else if (string("-hw") == argv[i])
HARDWARE = true;
else if (string("-c") == argv[i])
CONFIG = true;
else if (string("-db") == argv[i])
DBMS = true;
else if (string("-r") == argv[i])
RESOURCE = true;
else if (string("-l") == argv[i])
LOG = true;
else if (string("-bl") == argv[i])
BULKLOG = true;
else if (string("-lc") == argv[i])
LOCAL = true;
else if (string("-p") == argv[i])
{
i++;
if (argc == i)
{
cout << "ERROR: missing root password argument" << endl;
exit(-1);
}
rootPassword = argv[i];
// add single quote for special characters
if (rootPassword != "ssh")
{
rootPassword = "'" + rootPassword + "'";
}
}
else if (string("-mp") == argv[i])
{
i++;
if (argc == i)
{
cout << "ERROR: missing MariaDB Columnstore root user password argument" << endl;
exit(-1);
}
mysqlpw = argv[i];
mysqlpw = "'" + mysqlpw + "'";
}
else if (string("-de") == argv[i])
debug_flag = "1";
else if (string("-hd") == argv[i])
{
HADOOP = (DataFilePlugin.empty() ? false : true);
}
else
{
cout << "Invalid Option of '" << argv[i] << "', run with '-h' for help" << endl;
exit(1);
}
}
}
// default to -a if nothing is set
if (!HARDWARE && !CONFIG && !DBMS && !RESOURCE && !LOG && !BULKLOG && !HADOOP)
{
HARDWARE = true;
CONFIG = true;
DBMS = true;
RESOURCE = true;
LOG = true;
HADOOP = (DataFilePlugin.empty() ? false : true);
}
// get Parent OAM Module Name and setup of it's Custom OS files
string PrimaryUMModuleName;
try
{
PrimaryUMModuleName = sysConfig->getConfig(SystemSection, "PrimaryUMModuleName");
}
catch (...)
{
cout << "ERROR: Problem getting Parent OAM Module Name" << endl;
exit(-1);
}
if (PrimaryUMModuleName == "unassigned")
PrimaryUMModuleName = localModule;
if ((localModule != PrimaryUMModuleName) && DBMS)
{
char* pcommand = 0;
char* p;
string argument = "n";
while (true)
{
cout << endl << "You selected to get the DBMS data." << endl;
cout << "You need to run the columnstoreSupport command on module '" << PrimaryUMModuleName
<< "' to get that information." << endl;
cout << "Or you can proceed on to get all data except the DBMS." << endl;
pcommand = readline(" Do you want to proceed: (y or n) [n]: ");
if (pcommand && *pcommand)
{
p = strtok(pcommand, " ");
argument = p;
free(pcommand);
pcommand = 0;
}
if (pcommand)
{
free(pcommand);
pcommand = 0;
}
if (argument == "y")
{
cout << endl;
break;
}
else if (argument == "n")
exit(1);
}
}
// get number of worker-nodes, will tell us if a single server system
// get Parent OAM Module Name and setup of it's Custom OS files
try
{
string NumWorkers = sysConfig->getConfig("DBRM_Controller", "NumWorkers");
if (NumWorkers == "1")
singleServerInstall = "y";
}
catch (...)
{
}
if (singleServerInstall == "n" && !LOCAL)
if (HARDWARE || CONFIG || RESOURCE || LOG || HADOOP)
if (rootPassword.empty())
{
cout << "ERROR: Multi-Module System, Password Argument required or use '-lc' option, check help for "
"more information"
<< endl;
exit(-1);
}
// get Parent OAM Module Name and setup of it's Custom OS files
// string parentOAMModuleName;
ChildModule parentOAMModule;
try
{
parentOAMModule.moduleName = sysConfig->getConfig(SystemSection, "ParentOAMModuleName");
}
catch (...)
{
cout << "ERROR: Problem getting Parent OAM Module Name" << endl;
exit(-1);
}
// Get list of configured system modules
SystemModuleTypeConfig sysModuleTypeConfig;
try
{
oam.getSystemConfig(sysModuleTypeConfig);
}
catch (...)
{
cout << "ERROR: Problem reading the Columnstore System Configuration file" << endl;
exit(-1);
}
string ModuleSection = "SystemModuleConfig";
for (unsigned int i = 0; i < sysModuleTypeConfig.moduletypeconfig.size(); i++)
{
string moduleType = sysModuleTypeConfig.moduletypeconfig[i].ModuleType;
int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount;
if (moduleCount == 0)
// no modules equipped for this Module Type, skip
continue;
// get IP addresses and Host Names
DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin();
for (; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end(); listPT++)
{
string moduleName = (*listPT).DeviceName;
HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin();
string moduleIPAddr = (*pt1).IPAddr;
string moduleHostName = (*pt1).HostName;
if (moduleName == localModule)
{
localModuleHostName = moduleHostName;
}
// save Child modules
if (moduleName != localModule && moduleType != "xm")
{
childmodule.moduleName = moduleName;
childmodule.moduleIP = moduleIPAddr;
childmodule.hostName = moduleHostName;
childmodulelist.push_back(childmodule);
}
if (moduleName == parentOAMModule.moduleName)
{
parentOAMModule.moduleIP = moduleIPAddr;
parentOAMModule.hostName = moduleHostName;
parentOAMModule.moduleName = moduleName;
}
}
} // end of i for loop
// create a clean Columnstore Support Report
system("rm -f *_configReport.txt");
system("rm -f *_dbmsReport.txt");
system("rm -f *_hardwareReport.txt");
system("rm -f *_logReport.txt");
system("rm -f *_bulklogReport.txt");
system("rm -f *_resourceReport.txt");
//
// Configuration
//
if (CONFIG)
{
string reportType = "config";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// Alarms and Columnstore Logs
//
if (LOG)
{
string reportType = "log";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// Bulk Logs
//
if (BULKLOG)
{
string reportType = "bulklog";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// Hardware
//
if (HARDWARE)
{
string reportType = "hardware";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// Resources
//
if (RESOURCE)
{
string reportType = "resource";
cout << "Get " + reportType + " report data for " + localModule << endl;
pthread_t reportthread;
int status = pthread_create(&reportthread, NULL, (void* (*)(void*)) & reportThread, &reportType);
if (status != 0)
{
cout << "ERROR: reportthread: pthread_create failed, return status = " + oam.itoa(status);
}
sleep(1);
}
//
// DBMS
//
if (DBMS)
{
cout << "Get dbms report data for " << localModule << endl;
string outputFile = localModule + "_dbmsReport.txt";
FILE* pOutputFile = fopen(outputFile.c_str(), "w");
if (pOutputFile == NULL)
{
cout << "Could not open file: " + outputFile << endl;
exit(1);
}
fprintf(pOutputFile,
"********************************************************************************\n"
"\n"
" System %s\n"
" columnstoreSupportReport script ran from Module %s on %s\n"
" SoftwareVersion = %s-%s"
"\n"
"********************************************************************************\n"
"\n"
" DBMS report\n"
"\n"
"********************************************************************************\n",
systemName.c_str(), localModule.c_str(), currentDate.c_str(), columnstore_version.c_str(),
columnstore_release.c_str());
fclose(pOutputFile);
// run DBMS report on local server
bool FAILED = false;
if (localModule != PrimaryUMModuleName)
{
cout << " FAILED: run columnstoreSupport on '" << PrimaryUMModuleName << "' to get the dbrm report"
<< endl;
FAILED = true;
}
else
{
// check if mysql is supported and get info
string logFile = tmpDir + "/idbmysql.log";
string columnstoreMysql = "mysql -u root ";
string cmd = columnstoreMysql + " -e 'status' > " + logFile + " 2>&1";
system(cmd.c_str());
// check for mysql password set
string pwprompt = " ";
if (checkLogStatus(logFile, "ERROR 1045"))
{
cout << "NOTE: MariaDB Columnstore root user password is set" << endl;
// needs a password, was password entered on command line
if (mysqlpw == " ")
{
// go check columnstore.cnf
string file = std::string(MCSMYCNFDIR) + "/columnstore.cnf";
ifstream oldFile(file.c_str());
vector<string> lines;
char line[200];
string buf;
while (oldFile.getline(line, 200))
{
buf = line;
string::size_type pos = buf.find("password", 0);
if (pos != string::npos)
{
string::size_type pos1 = buf.find("=", 0);
if (pos1 != string::npos)
{
pos = buf.find("#", 0);
if (pos == string::npos)
{
// password arg in columnstore.cnf, go get password
cout << "NOTE: Using password from columnstore.cnf" << endl;
mysqlpw = buf.substr(pos1 + 1, 80);
cout << mysqlpw << endl;
break;
}
}
}
}
oldFile.close();
if (mysqlpw == " ")
{
cout << "NOTE: No password provide on command line or found uncommented in columnstore.cnf"
<< endl;
cout << endl;
string prompt = " *** Enter MariaDB Columnstore password > ";
mysqlpw = getpass(prompt.c_str());
}
}
// check for mysql password set
pwprompt = "--password=" + mysqlpw;
string cmd = columnstoreMysql + pwprompt + " -e 'status' > " + logFile + " 2>&1";
system(cmd.c_str());
if (checkLogStatus(logFile, "ERROR 1045"))
{
cout << "FAILED: Failed login using MariaDB Columnstore root user password '" << mysqlpw << "'"
<< endl;
FAILED = true;
}
}
if (!FAILED)
{
string cmd = "dbmsReport.sh " + localModule + " " + outputFile + " " + std::string(MCSSUPPORTDIR) +
" " + pwprompt;
system(cmd.c_str());
}
}
/*
BT: This doesn't appear to do anything
fprintf(pOutputFile,"\n******************** Database Size Report
*************************************\n\n"); getStorageStatus(pOutputFile);
string file = "databaseSizeReport";
ifstream File (file.c_str());
if (File)
{
string cmd = "databaseSizeReport >> " + outputFile;
system(cmd.c_str());
}
*/
boost::filesystem::path configFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf";
boost::filesystem::copy_file(configFile, "./columnstore.cnf",
boost::filesystem::copy_options::overwrite_existing);
}
int wait = 0;
while (true)
{
// cout << "check " << runningThreads << endl;
if (runningThreads < 1)
break;
sleep(2);
wait++;
// give it 60 minutes to complete
if (wait >= 3600 * 5)
{
cout << "Timed out (60 minutes) waiting for Requests to complete" << endl;
}
}
system("unix2dos *Report.txt > /dev/null 2>&1");
system(
"rm -rf columnstoreSupportReport;"
"mkdir columnstoreSupportReport;"
"mv *Report.txt columnstoreSupportReport/. > /dev/null 2>&1;"
"mv Columnstore.xml columnstoreSupportReport/. > /dev/null 2>&1;"
"mv columnstore.cnf columnstoreSupportReport/. > /dev/null 2>&1;"
"mv storagemanager.cnf columnstoreSupportReport/. > /dev/null 2>&1;"
"mv *Report.tar.gz columnstoreSupportReport/. > /dev/null 2>&1");
string cmd = "tar -zcf columnstoreSupportReport." + systemName + ".tar.gz columnstoreSupportReport/*";
system(cmd.c_str());
cout << endl
<< "Columnstore Support Script Successfully completed, files located in columnstoreSupportReport." +
systemName + ".tar.gz"
<< endl;
}

76
oamapps/columnstoreSupport/configReport.sh
View File

@ -1,76 +0,0 @@
#!/bin/bash
#
# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
echo " "
echo "******************** Configuration/Status Report for ${MODULE} ********************"
echo " "
chkconfig=`which chkconfig 2>/dev/null`
if [ -n "$chkconfig" ]; then
echo "-- chkconfig configuration --"
echo " "
echo "################# chkconfig --list | grep columnstore #################"
echo " "
chkconfig --list | grep columnstore 2>/dev/null
fi
systemctl=`which systemctl 2>/dev/null`
if [ -n "$systemctl" ]; then
echo "-- systemctl configuration --"
echo " "
echo "################# systemctl list-unit-files --type=service | grep columnstore #################"
echo " "
systemctl list-unit-files --type=service | grep columnstore 2>/dev/null
echo "################# systemctl list-unit-files --type=service | grep mariadb #################"
echo " "
systemctl list-unit-files --type=service | grep mariadb 2>/dev/null
fi
updaterc=`which update-rc.d 2>/dev/null`
if [ -n "$updaterc" ]; then
echo "-- services configuration --"
echo " "
echo "################# service --status-all | grep columnstore #################"
echo " "
service --status-all | grep columnstore 2>/dev/null
fi
echo " "
echo "-- fstab Configuration --"
echo " "
echo "################# cat /etc/fstab #################"
echo " "
cat /etc/fstab 2>/dev/null
echo " "
echo "-- Server Processes --"
echo " "
echo "################# ps axu #################"
echo " "
ps axu
echo " "
echo "-- Server Processes with resource usage --"
echo " "
echo "################# top -b -n 1 #################"
echo " "
top -b -n 1
} >> $OUT_FILE
exit 0

64
oamapps/columnstoreSupport/dbmsReport.sh
View File

@ -1,64 +0,0 @@
#! /bin/sh
#
# $Id: dbmsReport.sh
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
if [ $3 ] ; then
MCSSUPPORTDIR=$3
else
MCSSUPPORTDIR="/usr/share/columnstore"
fi
if [ $4 ] ; then
PW_PROMPT=$4
else
PW_PROMPT=""
fi
{
columnstoreMysql="mysql -u root ${PW_PROMPT} "
if ${columnstoreMysql} -V > /dev/null 2>&1; then
echo " "
echo "******************** DBMS Columnstore Version *********************************"
echo " "
${columnstoreMysql} -e 'status'
echo " "
echo "******************** DBMS Columnstore System Column ***************************"
echo " "
${columnstoreMysql} -e 'desc calpontsys.syscolumn;'
echo " "
echo "******************** DBMS Columnstore System Table ****************************"
echo " "
${columnstoreMysql} -e 'desc calpontsys.systable;'
echo " "
echo "******************** DBMS Columnstore System Catalog Data *********************"
echo " "
${columnstoreMysql} calpontsys < $MCSSUPPORTDIR/dumpcat_mysql.sql
echo " "
echo "******************** DBMS Columnstore System Table Data ***********************"
echo "******************** DBMS Columnstore Databases *******************************"
echo " "
${columnstoreMysql} -e 'show databases;'
echo " "
echo "******************** DBMS Columnstore variables *******************************"
echo " "
${columnstoreMysql} -e 'show variables;'
echo " "
fi
} >> $OUT_FILE
exit 0

130
oamapps/columnstoreSupport/getMinMax.sh
View File

@ -1,130 +0,0 @@
#!/bin/bash
#
# Reports the max value from the extent map for the given column.
#
#
# Initialize variables.
#
if [ -z "$MYSQLCMD" ]; then
MYSQLCMD="mysql -u root"
fi
#
# Validate that there are three parameters - schema and table and columnname.
#
if [ $# -ne 3 ]; then
echo ""
echo "Reports the max value for the given column."
echo ""
echo "Parameters:"
echo " Schema"
echo " Table"
echo " Column"
exit 1
fi
db=$1
table=$2
column=$3
#
# Validate that the column exists.
#
sql="select count(*) from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';"
count=`$MYSQLCMD calpontsys --skip-column-names -e "$sql;"`
if [ $count -le 0 ]; then
echo ""
echo "$db.$table.$column does not exist in Columnstore."
echo ""
exit 1
fi
#
# Validate that the column type is one that this script supports.
# Supported Types:
# 6 int
# 8 date
# 9 bigint
# 11 datetime
sql="select datatype from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';"
dataType=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
if [ $dataType -ne 6 ] && [ $dataType -ne 8 ] && [ $dataType -ne 9 ] && [ $dataType -ne 11 ]; then
echo ""
echo "The column data type must be an int, bigint, date, or datetime."
echo ""
exit 1
fi
#
# Grab the objectid for the column.
#
sql="select objectid from syscolumn where \`schema\`='$db' and tablename='$table' and columnname='$column';"
objectid=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
#
# Set the editem specific parameter if the column is a date or datetime.
#
if [ $dataType -eq 8 ]; then
parm="-t"
elif [ $dataType -eq 11 ]; then
parm="-s"
fi
#
# Use the editem utility to get the min and max value.
#
editem -o $objectid $parm | grep max | awk -v dataType=$dataType '
BEGIN {
allValid=1;
foundValidExtent=0;
}
{
if(dataType == 11) {
state=substr($14, 1, length($14)-1); # Datetime has date and time as two fields.
thisMin=$6 " " substr($7, 1, length($7)-1);
thisMax=$9 " " substr($10, 1, length($10)-1);
}
else {
state=substr($12, 1, length($12)-1);
thisMin=substr($6, 1, length($6)-1);
thisMax=substr($8, 1, length($8)-1);
}
if(state == "valid") {
if(!foundValidExtent) {
min=thisMin;
max=thisMax;
foundValidExtent=1;
}
else {
if(thisMin < min) {
min=thisMin;
}
if(thisMax > max) {
max=thisMax;
}
}
}
else {
allValid=0;
}
}
END {
if(foundValidExtent == 1) {
print "";
print "Min=" min;
print "Max=" max;
print "";
if(allValid == 0) {
print "Not all extents had min and max values set. Answer is incomplete."
}
}
else {
print "";
print "There were not any extents with valid min/max values. Unable to provide answer.";
print "";
}
}'
exit 0

79
oamapps/columnstoreSupport/hardwareReport.sh
View File

@ -1,79 +0,0 @@
#! /bin/sh
#
# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
echo " "
echo "******************** Hardware Report for ${MODULE} ********************"
echo " "
echo "-- Server OS Version --"
echo " "
echo "################# cat /proc/version #################"
echo " "
cat /proc/version 2>/dev/null
echo " "
echo "################# uname -a #################"
echo " "
uname -a
echo " "
echo "################# cat /etc/issue #################"
echo " "
cat /etc/issue 2>/dev/null
echo " "
echo "run columnstore_os_check.sh"
echo " "
echo "################# /bin/columnstore_os_check.sh #################"
echo " "
columnstore_os_check.sh 2>/dev/null
echo " "
echo "-- Server Uptime --"
echo " "
echo "################# uptime #################"
echo " "
uptime
echo " "
echo "-- Server cpu-info --"
echo " "
echo "################# cat /proc/cpuinfo #################"
echo " "
cat /proc/cpuinfo 2>/dev/null
echo " "
echo "-- Server memory-info --"
echo " "
echo "################# cat /proc/meminfo #################"
echo " "
$cat /proc/meminfo 2>/dev/null
echo " "
echo "-- Server mounts --"
echo " "
echo "################# cat /proc/mounts #################"
echo " "
cat /proc/mounts 2>/dev/null
echo " "
echo "-- Server Ethernet Configuration --"
echo " "
echo "################# ifconfig -a #################"
echo " "
ifconfig -a 2>/dev/null
} >> $OUT_FILE
exit 0

52
oamapps/columnstoreSupport/logReport.sh
View File

@ -1,52 +0,0 @@
#! /bin/sh
#
# $Id: logReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
#get temp directory
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/${MODULE}_logReport.tar.gz
tar -zcf ${tmpDir}/${MODULE}_logReport.tar.gz /var/log/mariadb/columnstore > /dev/null 2>&1
cp ${tmpDir}/${MODULE}_logReport.tar.gz .
tar -zcf ${MODULE}_mysqllogReport.tar.gz /var/log/mysql/*.err 2>/dev/null
echo '******************** Log Configuration ********************' >> $OUT_FILE
echo '' >> $OUT_FILE
echo 'MariaDB ColumnStore System Log Configuration Data' >> $OUT_FILE
echo '' >> $OUT_FILE
configFileName=`mcsGetConfig Installation SystemLogConfigFile`
echo 'System Logging Configuration File being used: '${configFileName} >> $OUT_FILE
echo '' >> $OUT_FILE
echo -e 'Module\tConfigured Log Levels' >> $OUT_FILE
echo -e '------\t---------------------------------------' >> $OUT_FILE
moduleConfig=''
if grep -q '/var/log/mariadb/columnstore/crit.log' ${configFileName}; then
moduleConfig=${moduleConfig}' CRITICAL'
fi
if grep -q '/var/log/mariadb/columnstore/err.log' ${configFileName}; then
moduleConfig=${moduleConfig}' ERROR'
fi
if grep -q '/var/log/mariadb/columnstore/warning.log' ${configFileName}; then
moduleConfig=${moduleConfig}' WARNING'
fi
if grep -q '/var/log/mariadb/columnstore/info.log' ${configFileName}; then
moduleConfig=${moduleConfig}' INFO'
fi
if grep -q '/var/log/mariadb/columnstore/debug.log' ${configFileName}; then
moduleConfig=${moduleConfig}' DEBUG'
fi
echo -e ${MODULE}'\t'${moduleConfig} >> $OUT_FILE
exit 0

621
oamapps/columnstoreSupport/mcsSupportUtil.cpp
View File

@ -1,621 +0,0 @@
/* Copyright (C) 2019 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "mcsSupportUtil.h"
using namespace std;
using namespace oam;
using namespace config;
void getSystemNetworkConfig(FILE* pOutputFile)
{
Oam oam;
// get and display Module Network Config
SystemModuleTypeConfig systemmoduletypeconfig;
systemmoduletypeconfig.moduletypeconfig.clear();
// get max length of a host name for header formatting
int maxSize = 9;
try
{
oam.getSystemConfig(systemmoduletypeconfig);
for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++)
{
if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty())
// end of list
break;
int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount;
string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType;
string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc;
if (moduleCount > 0)
{
DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin();
for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++)
{
HostConfigList::iterator pt1 = (*pt).hostConfigList.begin();
for (; pt1 != (*pt).hostConfigList.end(); pt1++)
{
if (maxSize < (int)(*pt1).HostName.size())
maxSize = (*pt1).HostName.size();
}
}
}
}
}
catch (exception& e)
{
fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what());
}
fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "Module Name", "Module Description", "NIC ID",
"Host Name", "IP Address");
fprintf(pOutputFile, "%-15s%-30s%-10s%-14s%-20s\n", "-----------", "-------------------------", "------",
"---------", "---------------");
try
{
oam.getSystemConfig(systemmoduletypeconfig);
for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++)
{
if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty())
// end of list
break;
int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount;
string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType;
string moduletypedesc = systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc;
if (moduleCount > 0)
{
DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin();
for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++)
{
string modulename = (*pt).DeviceName;
string moduleID = modulename.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE);
string modulenamedesc = moduletypedesc + " #" + moduleID;
fprintf(pOutputFile, "%-15s%-30s", modulename.c_str(), modulenamedesc.c_str());
HostConfigList::iterator pt1 = (*pt).hostConfigList.begin();
for (; pt1 != (*pt).hostConfigList.end(); pt1++)
{
/* MCOL-1607. IPAddr may be a host name here b/c it is read straight
from the config file. */
string tmphost = getIPAddress(pt1->IPAddr);
string ipAddr;
if (tmphost.empty())
ipAddr = pt1->IPAddr;
else
ipAddr = tmphost;
string hostname = (*pt1).HostName;
string nicID = oam.itoa((*pt1).NicID);
if (nicID != "1")
{
fprintf(pOutputFile, "%-45s", "");
}
fprintf(pOutputFile, "%-13s%-14s%-20s\n", nicID.c_str(), hostname.c_str(), ipAddr.c_str());
}
}
}
}
}
catch (exception& e)
{
fprintf(pOutputFile, "**** getNetworkConfig Failed = %s\n\n", e.what());
}
}
void getModuleTypeConfig(FILE* pOutputFile)
{
Oam oam;
SystemModuleTypeConfig systemmoduletypeconfig;
ModuleTypeConfig moduletypeconfig;
ModuleConfig moduleconfig;
systemmoduletypeconfig.moduletypeconfig.clear();
try
{
oam.getSystemConfig(systemmoduletypeconfig);
fprintf(pOutputFile, "Module Type Configuration\n\n");
for (unsigned int i = 0; i < systemmoduletypeconfig.moduletypeconfig.size(); i++)
{
if (systemmoduletypeconfig.moduletypeconfig[i].ModuleType.empty())
// end of list
break;
int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount;
if (moduleCount < 1)
continue;
string moduletype = systemmoduletypeconfig.moduletypeconfig[i].ModuleType;
fprintf(pOutputFile, "ModuleType '%s' Configuration information\n", moduletype.c_str());
fprintf(pOutputFile, "ModuleDesc = %s\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleDesc.c_str());
fprintf(pOutputFile, "ModuleCount = %i\n", moduleCount);
if (moduleCount > 0)
{
DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin();
for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++)
{
string modulename = (*pt).DeviceName;
HostConfigList::iterator pt1 = (*pt).hostConfigList.begin();
for (; pt1 != (*pt).hostConfigList.end(); pt1++)
{
string ipAddr = (*pt1).IPAddr;
string servername = (*pt1).HostName;
fprintf(pOutputFile, "ModuleHostName and ModuleIPAddr for NIC ID %u on module '%s' = %s , %s\n",
(*pt1).NicID, modulename.c_str(), servername.c_str(), ipAddr.c_str());
}
}
}
DeviceDBRootList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.begin();
for (; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleDBRootList.end(); pt++)
{
if ((*pt).dbrootConfigList.size() > 0)
{
fprintf(pOutputFile, "DBRootIDs assigned to module 'pm%u' = ", (*pt).DeviceID);
DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin();
for (; pt1 != (*pt).dbrootConfigList.end();)
{
fprintf(pOutputFile, "%u", *pt1);
pt1++;
if (pt1 != (*pt).dbrootConfigList.end())
fprintf(pOutputFile, ", ");
}
}
fprintf(pOutputFile, "\n");
}
fprintf(pOutputFile, "ModuleCPUCriticalThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUCriticalThreshold);
fprintf(pOutputFile, "ModuleCPUMajorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMajorThreshold);
fprintf(pOutputFile, "ModuleCPUMinorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorThreshold);
fprintf(pOutputFile, "ModuleCPUMinorClearThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleCPUMinorClearThreshold);
fprintf(pOutputFile, "ModuleDiskCriticalThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskCriticalThreshold);
fprintf(pOutputFile, "ModuleDiskMajorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMajorThreshold);
fprintf(pOutputFile, "ModuleDiskMinorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleDiskMinorThreshold);
fprintf(pOutputFile, "ModuleMemCriticalThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleMemCriticalThreshold);
fprintf(pOutputFile, "ModuleMemMajorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMajorThreshold);
fprintf(pOutputFile, "ModuleMemMinorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleMemMinorThreshold);
fprintf(pOutputFile, "ModuleSwapCriticalThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapCriticalThreshold);
fprintf(pOutputFile, "ModuleSwapMajorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMajorThreshold);
fprintf(pOutputFile, "ModuleSwapMinorThreshold = %u%%\n",
systemmoduletypeconfig.moduletypeconfig[i].ModuleSwapMinorThreshold);
DiskMonitorFileSystems::iterator pt2 = systemmoduletypeconfig.moduletypeconfig[i].FileSystems.begin();
int id = 1;
for (; pt2 != systemmoduletypeconfig.moduletypeconfig[i].FileSystems.end(); pt2++)
{
string fs = *pt2;
fprintf(pOutputFile, "ModuleDiskMonitorFileSystem#%i = %s\n", id, fs.c_str());
++id;
}
fprintf(pOutputFile, "\n");
}
}
catch (exception& e)
{
cout << endl << "**** getModuleTypeConfig Failed = " << e.what() << endl;
}
}
void getStorageConfig(FILE* pOutputFile)
{
Oam oam;
try
{
systemStorageInfo_t t;
t = oam.getStorageConfig();
string cloud;
try
{
oam.getSystemConfig("Cloud", cloud);
}
catch (...)
{
}
string::size_type pos = cloud.find("amazon", 0);
if (pos != string::npos)
cloud = "amazon";
fprintf(pOutputFile, "System Storage Configuration\n");
fprintf(pOutputFile, "Performance Module (DBRoot) Storage Type = %s\n", boost::get<0>(t).c_str());
if (cloud == "amazon")
fprintf(pOutputFile, "User Module Storage Type = %s\n", boost::get<3>(t).c_str());
fprintf(pOutputFile, "System Assigned DBRoot Count = %i\n", boost::get<1>(t));
DeviceDBRootList moduledbrootlist = boost::get<2>(t);
typedef std::vector<int> dbrootList;
dbrootList dbrootlist;
DeviceDBRootList::iterator pt = moduledbrootlist.begin();
for (; pt != moduledbrootlist.end(); pt++)
{
fprintf(pOutputFile, "DBRoot IDs assigned to 'pm%u' = ", (*pt).DeviceID);
DBRootConfigList::iterator pt1 = (*pt).dbrootConfigList.begin();
for (; pt1 != (*pt).dbrootConfigList.end();)
{
fprintf(pOutputFile, "%u", *pt1);
dbrootlist.push_back(*pt1);
pt1++;
if (pt1 != (*pt).dbrootConfigList.end())
fprintf(pOutputFile, ", ");
}
fprintf(pOutputFile, "\n");
}
// get any unassigned DBRoots
/*DBRootConfigList undbrootlist;
try
{
oam.getUnassignedDbroot(undbrootlist);
}
catch (...) {}
if ( !undbrootlist.empty() )
{
fprintf(pOutputFile,"DBRoot IDs unassigned = ");
DBRootConfigList::iterator pt1 = undbrootlist.begin();
for ( ; pt1 != undbrootlist.end() ;)
{
fprintf(pOutputFile,"%u",*pt1);
pt1++;
if (pt1 != undbrootlist.end())
fprintf(pOutputFile,", ");
}
fprintf(pOutputFile,"\n");
}*/
fprintf(pOutputFile, "\n");
// um volumes
if (cloud == "amazon" && boost::get<3>(t) == "external")
{
ModuleTypeConfig moduletypeconfig;
oam.getSystemConfig("um", moduletypeconfig);
for (int id = 1; id < moduletypeconfig.ModuleCount + 1; id++)
{
string volumeNameID = "UMVolumeName" + oam.itoa(id);
string volumeName = oam::UnassignedName;
string deviceNameID = "UMVolumeDeviceName" + oam.itoa(id);
string deviceName = oam::UnassignedName;
try
{
oam.getSystemConfig(volumeNameID, volumeName);
oam.getSystemConfig(deviceNameID, deviceName);
}
catch (...)
{
}
fprintf(pOutputFile, "Amazon EC2 Volume Name/Device Name for 'um%i': %s, %s", id, volumeName.c_str(),
deviceName.c_str());
}
}
// pm volumes
if (cloud == "amazon" && boost::get<0>(t) == "external")
{
fprintf(pOutputFile, "\n");
DBRootConfigList dbrootConfigList;
try
{
oam.getSystemDbrootConfig(dbrootConfigList);
DBRootConfigList::iterator pt = dbrootConfigList.begin();
for (; pt != dbrootConfigList.end(); pt++)
{
string volumeNameID = "PMVolumeName" + oam.itoa(*pt);
string volumeName = oam::UnassignedName;
string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt);
string deviceName = oam::UnassignedName;
try
{
oam.getSystemConfig(volumeNameID, volumeName);
oam.getSystemConfig(deviceNameID, deviceName);
}
catch (...)
{
continue;
}
}
}
catch (exception& e)
{
cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl;
}
// print un-assigned dbroots
/*DBRootConfigList::iterator pt1 = undbrootlist.begin();
for ( ; pt1 != undbrootlist.end() ; pt1++)
{
string volumeNameID = "PMVolumeName" + oam.itoa(*pt1);
string volumeName = oam::UnassignedName;
string deviceNameID = "PMVolumeDeviceName" + oam.itoa(*pt1);
string deviceName = oam::UnassignedName;
try
{
oam.getSystemConfig( volumeNameID, volumeName);
oam.getSystemConfig( deviceNameID, deviceName);
}
catch (...)
{
continue;
}
}*/
}
string DataRedundancyConfig;
int DataRedundancyCopies;
string DataRedundancyStorageType;
try
{
oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig);
oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies);
oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType);
}
catch (...)
{
}
if (DataRedundancyConfig == "y")
{
fprintf(pOutputFile, "\nData Redundant Configuration\n\n");
fprintf(pOutputFile, "Copies Per DBroot = %i", DataRedundancyCopies);
oamModuleInfo_t st;
string moduleType;
try
{
st = oam.getModuleInfo();
moduleType = boost::get<1>(st);
}
catch (...)
{
}
if (moduleType != "pm")
return;
try
{
DBRootConfigList dbrootConfigList;
oam.getSystemDbrootConfig(dbrootConfigList);
DBRootConfigList::iterator pt = dbrootConfigList.begin();
for (; pt != dbrootConfigList.end(); pt++)
{
fprintf(pOutputFile, "DBRoot #%u has copies on PMs = ", *pt);
string pmList = "";
try
{
string errmsg;
// oam.glusterctl(oam::GLUSTER_WHOHAS, oam.itoa(*pt), pmList, errmsg);
}
catch (...)
{
}
boost::char_separator<char> sep(" ");
boost::tokenizer<boost::char_separator<char> > tokens(pmList, sep);
for (boost::tokenizer<boost::char_separator<char> >::iterator it = tokens.begin();
it != tokens.end(); ++it)
{
fprintf(pOutputFile, "%s ", (*it).c_str());
}
fprintf(pOutputFile, "\n");
}
fprintf(pOutputFile, "\n");
}
catch (exception& e)
{
cout << endl << "**** getSystemDbrootConfig Failed : " << e.what() << endl;
}
}
}
catch (exception& e)
{
cout << endl << "**** getStorageConfig Failed : " << e.what() << endl;
}
}
void getStorageStatus(FILE* pOutputFile)
{
Oam oam;
fprintf(pOutputFile, "System External DBRoot Storage Statuses\n\n");
fprintf(pOutputFile, "Component Status Last Status Change\n");
fprintf(pOutputFile, "------------ -------------------------- ------------------------\n");
/*try
{
oam.getSystemStatus(systemstatus, false);
if ( systemstatus.systemdbrootstatus.dbrootstatus.size() == 0 )
{
fprintf(pOutputFile," No External DBRoot Storage Configured\n\n");
return;
}
for ( unsigned int i = 0 ; i < systemstatus.systemdbrootstatus.dbrootstatus.size(); i++)
{
if ( systemstatus.systemdbrootstatus.dbrootstatus[i].Name.empty() )
// end of list
break;
int state = systemstatus.systemdbrootstatus.dbrootstatus[i].OpState;
string stime = systemstatus.systemdbrootstatus.dbrootstatus[i].StateChangeDate ;
stime = stime.substr (0, 24);
fprintf(pOutputFile,"DBRoot%s%-29s%-24s\n",
systemstatus.systemdbrootstatus.dbrootstatus[i].Name.c_str(),
oamState[state].c_str(),
stime.c_str());
}
fprintf(pOutputFile,"\n");
}
catch (exception& e)
{
cout << endl << "**** getSystemStatus Failed = " << e.what() << endl;
}*/
string DataRedundancyConfig;
int DataRedundancyCopies;
string DataRedundancyStorageType;
try
{
oam.getSystemConfig("DataRedundancyConfig", DataRedundancyConfig);
oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies);
oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType);
}
catch (...)
{
}
}
/********************************************************************
*
* checkLogStatus - Check for a phrase in a log file and return status
*
********************************************************************/
bool checkLogStatus(std::string fileName, std::string phrase)
{
ifstream file(fileName.c_str());
if (!file.is_open())
{
return false;
}
string buf;
while (getline(file, buf))
{
string::size_type pos = buf.find(phrase, 0);
if (pos != string::npos)
// found phrase
return true;
}
if (file.bad())
{
return false;
}
file.close();
return false;
}
/******************************************************************************************
* @brief Get Network IP Address for Host Name
*
* purpose: Get Network IP Address for Host Name
*
******************************************************************************************/
string getIPAddress(string hostName)
{
static uint32_t my_bind_addr;
struct hostent* ent;
string IPAddr = "";
Oam oam;
ent = gethostbyname(hostName.c_str());
if (ent != 0)
{
my_bind_addr = (uint32_t)((in_addr*)ent->h_addr_list[0])->s_addr;
uint8_t split[4];
uint32_t ip = my_bind_addr;
split[0] = (ip & 0xff000000) >> 24;
split[1] = (ip & 0x00ff0000) >> 16;
split[2] = (ip & 0x0000ff00) >> 8;
split[3] = (ip & 0x000000ff);
IPAddr =
oam.itoa(split[3]) + "." + oam.itoa(split[2]) + "." + oam.itoa(split[1]) + "." + oam.itoa(split[0]);
}
return IPAddr;
}

34
oamapps/columnstoreSupport/mcsSupportUtil.h
View File

@ -1,34 +0,0 @@
/* Copyright (C) 2020 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <iostream>
#include <fstream>
#include <cstdlib>
#include <string>
#include <limits.h>
#include <sstream>
#include "configcpp.h"
#include "liboamcpp.h"
void getSystemNetworkConfig(FILE* pOutputFile);
void getModuleTypeConfig(FILE* pOutputFile);
void getStorageConfig(FILE* pOutputFile);
void getStorageStatus(FILE* pOutputFile);
bool checkLogStatus(std::string filename, std::string phase);
std::string getIPAddress(std::string hostName);

66
oamapps/columnstoreSupport/resourceReport.sh
View File

@ -1,66 +0,0 @@
#! /bin/sh
#
# $Id: resourceReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
echo " "
echo "******************** Resource Usage Report for ${MODULE} ********************"
echo " "
echo " "
echo "-- Shared Memory --"
echo " "
echo "################# ipcs -l #################"
echo " "
ipcs -l
echo "################# clearShm -n #################"
echo " "
clearShm -n
echo " "
echo "-- Disk Usage --"
echo " "
echo "################# df -k #################"
echo " "
df -k
echo " "
echo "-- Disk BRM Data files --"
echo " "
ls -l /var/lib/columnstore/data1/systemFiles/dbrm 2> /dev/null
ls -l /var/lib/columnstore/dbrm 2> /dev/null
echo "################# cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current #################"
echo " "
cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current 2> /dev/null
echo " "
echo "-- View Table Locks --"
echo " "
echo "################# cat bin/viewtablelock #################"
echo " "
viewtablelock 2> /dev/null
echo " "
echo "-- BRM Extent Map --"
echo " "
echo "################# bin/editem -i #################"
echo " "
editem -i 2>/dev/null
} >> $OUT_FILE
exit 0

30
oamapps/columnstoreSupport/softwareReport.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#
# $Id: hardwareReport.sh 421 2007-04-05 15:46:55Z dhill $
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
OUT_FILE=$2
else
OUT_FILE=${MODULE}_logReport.txt
fi
{
echo " "
echo "******************** Software Report for ${MODULE} ********************"
echo " "
echo " "
echo "-- Columnstore Package Details --"
echo " "
rpm -qi MariaDB-columnstore-engine
echo " "
} >> $OUT_FILE
exit 0

34
oamapps/replayTransactionLog/CMakeLists.txt
View File

@ -1,34 +0,0 @@
#
# Not used
#
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = ReplayTransactionLog ReplayTransactionLog_SOURCES = replaytransactionlog.cpp
# ReplayTransactionLog_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) ReplayTransactionLog_LDFLAGS =
# @idb_common_ldflags@ @idb_exec_libs@ -lreplaytxnlog $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

156
oamapps/replayTransactionLog/replaytransactionlog.cpp
View File

@ -1,156 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// WWW - Add header comment.
#include <iostream>
#include <string>
#include <sstream>
#include "liboamcpp.h"
using namespace std;
using namespace oam;
#include "replaytxnlog.h"
namespace
{
void usage(char* prog)
{
cout << endl;
cout << "Usage: " << prog << " [options]" << endl;
cout << endl;
cout << "This utility can be used after a backup is restored to report transactions that " << endl;
cout << "occurred after the backup. It begins with the first transaction that was committed " << endl;
cout << "after the backup and reports DDL and DML statements as well as imports." << endl;
cout << endl;
cout << "Options:" << endl;
/*
cout << "-u <user> Database user id." << endl << endl;
cout << "-p <password> Password." << endl << endl;
*/
cout << "-d <stop date> Stop date and time as mm/dd/yy@hh:mm:ss or 'Now'." << endl;
cout << " Only transactions committed before this date and time will be reported." << endl;
cout << " The current date and time will be used if 'Now'." << endl << endl;
/*
cout << "-i Ignore bulk load log entries." << endl;
cout << " The program will pause and prompt at bulk load entries by default." << endl <<
endl;
cout << "-e Report mode. The sql statements will be displayed to the console only. No" <<
endl; cout << " transactions will be processed. The user and password will be ignored." <<
endl << endl;
*/
cout << "-h Display this help." << endl << endl;
}
bool isRunningOnPm()
{
Oam oam;
oamModuleInfo_t t;
string moduleType;
int installType = -1;
char* csc_ident = getenv("CALPONT_CSC_IDENT");
if (csc_ident == 0 || *csc_ident == 0)
{
// get local module info valdiate running on a pm
try
{
t = oam.getModuleInfo();
moduleType = boost::get<1>(t);
installType = boost::get<5>(t);
}
catch (exception& e)
{
moduleType = "pm";
}
}
else
moduleType = csc_ident;
if (installType != oam::INSTALL_COMBINE_DM_UM_PM)
{
if (moduleType != "pm")
{
cerr << "Exiting, ReplayTransactionLog can only be run on a performance module (pm)" << endl;
return false;
}
}
return true;
}
} // namespace
int main(int argc, char** argv)
{
string user;
string password;
string stopDate;
bool ignoreBulk = false;
bool reportMode = false;
char c;
// Invokes member function `int operator ()(void);'
while ((c = getopt(argc, argv, "u:p:d:ihe")) != -1)
{
switch (c)
{
/*
case 'u':
user = optarg;
break;
case 'p':
password = optarg;
break;
*/
case 'd': stopDate = optarg; break;
/*
case 'i':
ignoreBulk = true;
break;
case 'e':
reportMode = true;
break;
*/
case 'h':
usage(argv[0]);
return 0;
break;
default:
usage(argv[0]);
return 1;
break;
}
}
if (!isRunningOnPm())
{
return 0;
}
ReplayTxnLog replayTxnLog(user, password, stopDate, ignoreBulk, reportMode);
replayTxnLog.process();
return 0;
}

66
oamapps/replayTransactionLog/tdriver.cpp
View File

@ -1,66 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
* wweeks@calpont.com *
* *
***************************************************************************/
using namespace std;
#include <iostream>
#include <cstdlib>
#include <sys/types.h>
#include <unistd.h>
#include "sessionmanager.h"
#include <cppunit/extensions/HelperMacros.h>
using namespace execplan;
int maxNewTxns = 1000;
int maxTxns = 1000;
class ExecPlanTest : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(ExecPlanTest);
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp()
{
}
void tearDown()
{
}
}; // test suite
CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest);
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main(int argc, char* argv[])
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest(registry.makeTest());
bool wasSuccessful = runner.run("", false);
return (wasSuccessful ? 0 : 1);
}

34
oamapps/sessionWalker/CMakeLists.txt
View File

@ -1,34 +0,0 @@
#
# Not used
#
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = sessionWalker sessionWalker_SOURCES = sessionwalker.cpp sessionWalker_CPPFLAGS = @idb_common_includes@
# $(AM_CPPFLAGS) sessionWalker_LDFLAGS = @idb_common_ldflags@ @idb_common_libs@ @idb_write_libs@ @netsnmp_libs@
# $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

135
oamapps/sessionWalker/sessionwalker.cpp
View File

@ -1,135 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
* $Id: sessionwalker.cpp 3072 2013-04-04 19:04:45Z rdempsey $
*
* jrodriguez@calpont.com
* *
***************************************************************************/
#include <iostream>
using namespace std;
#include "sessionmonitor.h"
using namespace execplan;
#include "vendordmlstatement.h"
#include "calpontdmlpackage.h"
#include "calpontdmlfactory.h"
using namespace dmlpackage;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
namespace
{
void usage()
{
cout << "sessionwalker [-d|-h]" << endl
<< " -r rollback all transactions found" << endl
<< " -h display this help" << endl;
}
void rollback(const SessionMonitor::MonSIDTIDEntry& txn)
{
VendorDMLStatement dmlStmt("ROLLBACK;", txn.sessionid);
CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
if (pDMLPackage == 0)
{
return;
}
ByteStream bytestream;
pDMLPackage->write(bytestream);
delete pDMLPackage;
MessageQueueClient mq("DMLProc");
try
{
cout << "sending ROLLBACK for sessionID " << txn.sessionid << endl;
mq.write(bytestream);
bytestream = mq.read();
}
catch (...)
{
}
}
} // namespace
int main(int argc, char** argv)
{
bool rflg = false;
opterr = 0;
int c;
while ((c = getopt(argc, argv, "rh")) != EOF)
switch (c)
{
case 'r': rflg = true; break;
case 'h':
usage();
return 0;
break;
default:
usage();
return 1;
break;
}
vector<SessionMonitor::MonSIDTIDEntry*> toTxns;
SessionMonitor* monitor = new SessionMonitor();
toTxns.clear();
toTxns = monitor->timedOutTxns(); // get timed out txns
vector<SessionMonitor::MonSIDTIDEntry*>::iterator iter = toTxns.begin();
vector<SessionMonitor::MonSIDTIDEntry*>::iterator end = toTxns.end();
vector<SessionMonitor::MonSIDTIDEntry*> tmp;
while (iter != end)
{
if ((*iter)->sessionid > 0)
tmp.push_back(*iter);
++iter;
}
toTxns.swap(tmp);
cout << toTxns.size() << " timed out transactions." << endl;
for (unsigned idx = 0; idx < toTxns.size(); idx++)
{
monitor->printTxns(*toTxns[idx]);
if (rflg)
{
rollback(*toTxns[idx]);
}
}
delete monitor;
return 0;
}

189
oamapps/sessionWalker/tdriver.cpp
View File

@ -1,189 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
* jrodriguez@calpont.com *
* *
***************************************************************************/
using namespace std;
#include <iostream>
#include <cstdlib>
#include <sys/types.h>
#include <unistd.h>
#include "sessionmonitor.h"
#include "sessionmanager.h"
#include <cppunit/extensions/HelperMacros.h>
using namespace execplan;
int maxNewTxns = 1000;
int maxTxns = 1000;
class ExecPlanTest : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(ExecPlanTest);
CPPUNIT_TEST(MonitorTestPlan_1);
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp()
{
}
void tearDown()
{
}
int verifyLen;
SessionManager* manager;
SessionManager::TxnID managerTxns[1000];
int createTxns(const int& start, const int& end)
{
int first = start;
int last = end;
int newTxns = 0;
verifyLen = manager->verifySize();
for (int idx = first; idx < last && verifyLen < maxNewTxns; idx++)
{
managerTxns[idx] = manager->newTxnID((uint32_t)idx + 1000);
CPPUNIT_ASSERT(managerTxns[idx].id > 0);
CPPUNIT_ASSERT(managerTxns[idx].valid == true);
verifyLen = manager->verifySize();
CPPUNIT_ASSERT(verifyLen > 0);
newTxns++;
}
CPPUNIT_ASSERT(newTxns == last - first);
return newTxns;
}
int closeTxns(const int& start, const int& end)
{
int first = start;
int last = end;
int totalClosed = 0;
for (int idx = first; idx < last; idx++)
{
try
{
SessionManager::TxnID tmp = manager->getTxnID(idx + 1000);
if (tmp.valid == true)
{
manager->committed(tmp);
CPPUNIT_ASSERT(tmp.valid == false);
totalClosed++;
}
}
catch (exception& e)
{
cerr << e.what() << endl;
continue;
}
}
return totalClosed;
} // closeTxns
void MonitorTestPlan_1()
{
int currStartTxn = 0;
int currEndTxn = 5;
int txnCntIncr = 5;
const int sleepTime = 1;
const int iterMax = 1;
vector<SessionMonitor::MonSIDTIDEntry*> toTxns;
manager = new SessionManager();
// CPPUNIT_ASSERT(manager->verifySize()==0);
SessionMonitor* monitor = NULL;
for (int jdx = 0; jdx < iterMax; jdx++)
{
// store the current state of the SessionManager
monitor = new SessionMonitor();
monitor->AgeLimit(sleepTime);
delete monitor;
int idx = 0;
int grpStart = currStartTxn;
for (idx = 0; idx < 3; idx++)
{
createTxns(currStartTxn, currEndTxn);
// CPPUNIT_ASSERT(manager->verifySize()==(idx+1)*txnCntIncr);
currStartTxn += txnCntIncr;
currEndTxn += txnCntIncr;
sleep(sleepTime + 1); // make sessions time out
monitor = new SessionMonitor(); // read Monitor data
monitor->AgeLimit(sleepTime);
toTxns.clear();
toTxns = monitor->timedOutTxns(); // get timed out txns
CPPUNIT_ASSERT(toTxns.size() == (uint32_t)txnCntIncr * idx);
delete monitor;
}
int grpEnd = currEndTxn;
monitor = new SessionMonitor();
monitor->AgeLimit(sleepTime);
closeTxns(grpStart, grpEnd); // close this iteration of txns
// CPPUNIT_ASSERT(manager->verifySize()==0);
toTxns = monitor->timedOutTxns(); // get timed out txns
CPPUNIT_ASSERT(toTxns.size() == 0);
delete monitor;
}
monitor = new SessionMonitor(); // readload Monitor data
monitor->AgeLimit(sleepTime - 1);
toTxns.clear();
toTxns = monitor->timedOutTxns(); // get timed out txns
CPPUNIT_ASSERT(toTxns.size() == 0);
delete monitor;
// CPPUNIT_ASSERT(manager->verifySize()==0);
delete manager;
}
}; // test suite
CPPUNIT_TEST_SUITE_REGISTRATION(ExecPlanTest);
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main(int argc, char* argv[])
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest(registry.makeTest());
bool wasSuccessful = runner.run("", false);
return (wasSuccessful ? 0 : 1);
}

2
primitives/blockcache/CMakeLists.txt
View File

@ -14,4 +14,4 @@ set(dbbc_STAT_SRCS
fsutils.cpp
)
columnstore_static_library(dbbc ${dbbc_STAT_SRCS})
columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp)
columnstore_link(dbbc loggingcpp)

2
primitives/linux-port/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc)
set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp)
columnstore_static_library(processor ${processor_STAT_SRCS})
columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp)
columnstore_link(processor loggingcpp)

1
primitives/linux-port/column.cpp
View File

@ -40,7 +40,6 @@ using namespace boost;
#include "simd_sse.h"
#include "simd_arm.h"
#include "utils/common/columnwidth.h"
#include "utils/common/bit_cast.h"
#include "exceptclasses.h"

13
primitives/primproc/CMakeLists.txt
View File

@ -3,25 +3,26 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../linux-port)
# ########## next target ###############
set(PrimProc_SRCS
primproc.cpp
activestatementcounter.cpp
batchprimitiveprocessor.cpp
bppseeder.cpp
bppsendthread.cpp
columncommand.cpp
command.cpp
dictstep.cpp
femsghandler.cpp
filtercommand.cpp
logger.cpp
passthrucommand.cpp
primitiveserver.cpp
primproc.cpp
pseudocc.cpp
rssmonfcn.cpp
rtscommand.cpp
umsocketselector.cpp
samenodepseudosocket.cpp
serviceexemgr.cpp
sqlfrontsessionthread.cpp
rssmonfcn.cpp
activestatementcounter.cpp
femsghandler.cpp
umsocketselector.cpp
../../utils/common/crashtrace.cpp
)
@ -31,11 +32,11 @@ target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS})
columnstore_link(
PrimProc
${ENGINE_LDFLAGS}
${NETSNMP_LIBRARIES}
${ENGINE_WRITE_LIBS}
threadpool
cacheutils
dbbc
processor
loggingcpp
statistics_manager
)

0
utils/messageqcpp/samenodepseudosocket.cpp → primitives/primproc/samenodepseudosocket.cpp
View File

0
utils/messageqcpp/samenodepseudosocket.h → primitives/primproc/samenodepseudosocket.h
View File

2
primitives/primproc/serviceexemgr.cpp
View File

@ -78,7 +78,7 @@
#include "dbrm.h"
#include "mariadb_my_sys.h"
#include "statistics.h"
#include "statistics_manager/statistics.h"
#include "serviceexemgr.h"
#include "sqlfrontsessionthread.h"

4
primitives/primproc/serviceexemgr.h
View File

@ -58,7 +58,7 @@
#include "dbrm.h"
#include "mariadb_my_sys.h"
#include "statistics.h"
#include "statistics_manager/statistics.h"
namespace exemgr
{
@ -69,7 +69,7 @@ class Opt
int m_debug;
bool m_e;
bool m_fg;
Opt() : m_debug(0), m_e(false), m_fg(false){};
Opt() : m_debug(0), m_e(false), m_fg(false) {};
Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false)
{
int c;

11
primitives/primproc/sqlfrontsessionthread.h
View File

@ -56,13 +56,13 @@
#include "dbrm.h"
#include "mariadb_my_sys.h"
#include "statistics.h"
#include "statistics_manager/statistics.h"
#include "serviceexemgr.h"
namespace exemgr
{
class SQLFrontSessionThread
{
class SQLFrontSessionThread
{
public:
SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec,
joblist::ResourceManager* rm)
@ -125,7 +125,8 @@ namespace exemgr
void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted);
void analyzeTableHandleStats(messageqcpp::ByteStream& bs);
uint64_t roundMB(uint64_t value) const;
public:
void operator()();
};
}
};
} // namespace exemgr

2
storage-manager/CMakeLists.txt
View File

@ -64,7 +64,7 @@ configure_file(
link_directories(${CMAKE_BINARY_DIR}/lib)
set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
columnstore_library(storagemanager SHARED ${storagemanager_SRCS})
columnstore_library(storagemanager ${storagemanager_SRCS})
add_dependencies(storagemanager marias3 external_boost)
target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)

23
tools/CMakeLists.txt
View File

@ -1,14 +1,15 @@
add_subdirectory(dbbuilder)
add_subdirectory(editem)
add_subdirectory(dbloadxml)
add_subdirectory(getConfig)
add_subdirectory(cplogger)
add_subdirectory(clearShm)
add_subdirectory(cleartablelock)
add_subdirectory(configMgt)
add_subdirectory(cplogger)
add_subdirectory(dbbuilder)
add_subdirectory(dbloadxml)
add_subdirectory(ddlcleanup)
add_subdirectory(editem)
add_subdirectory(getConfig)
add_subdirectory(idbmeminfo)
add_subdirectory(passwd)
add_subdirectory(rebuildEM)
add_subdirectory(rgprint)
add_subdirectory(setConfig)
add_subdirectory(viewtablelock)
add_subdirectory(cleartablelock)
add_subdirectory(ddlcleanup)
add_subdirectory(idbmeminfo)
add_subdirectory(rebuildEM)
add_subdirectory(passwd)
add_subdirectory(configMgt)

2
tools/cleartablelock/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp)
columnstore_executable(cleartablelock ${cleartablelock_SRCS})
columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})

2
tools/configMgt/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(autoConfigure_SRCS autoConfigure.cpp)
add_executable(autoConfigure ${autoConfigure_SRCS})
columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

2
tools/dbbuilder/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp)
columnstore_executable(dbbuilder ${dbbuilder_SRCS})
columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})

2
tools/ddlcleanup/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(ddlcleanup_SRCS ddlcleanup.cpp)
columnstore_executable(ddlcleanup ${ddlcleanup_SRCS})
columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)
columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ddlcleanuputil)

2
tools/editem/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(editem_SRCS editem.cpp)
columnstore_executable(editem ${editem_SRCS})
columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(editem ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

2
tools/rgprint/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(rgprint_SRCS rgprint.cpp)
columnstore_executable(rgprint ${rgprint_SRCS})
columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
columnstore_link(rgprint ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS})

2
tools/setConfig/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(setConfig_SRCS main.cpp)
columnstore_executable(mcsSetConfig ${setConfig_SRCS})
columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

44
utils/CMakeLists.txt
View File

@ -1,26 +1,28 @@
# add_subdirectory(boost_idb)
add_subdirectory(startup)
add_subdirectory(common)
add_subdirectory(configcpp)
add_subdirectory(loggingcpp)
add_subdirectory(messageqcpp)
add_subdirectory(threadpool)
add_subdirectory(rwlock)
add_subdirectory(dataconvert)
add_subdirectory(joiner)
add_subdirectory(rowgroup)
add_subdirectory(cacheutils)
add_subdirectory(funcexp)
add_subdirectory(udfsdk)
add_subdirectory(compress)
add_subdirectory(batchloader)
add_subdirectory(ddlcleanup)
add_subdirectory(querystats)
add_subdirectory(windowfunction)
add_subdirectory(idbdatafile)
add_subdirectory(querytele)
add_subdirectory(libmysql_client)
add_subdirectory(regr)
add_subdirectory(cacheutils)
add_subdirectory(cloudio)
add_subdirectory(common)
add_subdirectory(compress)
add_subdirectory(configcpp)
add_subdirectory(dataconvert)
add_subdirectory(ddlcleanup)
add_subdirectory(funcexp)
add_subdirectory(idbdatafile)
add_subdirectory(joiner)
add_subdirectory(libmarias3)
add_subdirectory(libmysql_client)
add_subdirectory(loggingcpp)
add_subdirectory(mariadb_charset)
add_subdirectory(messageqcpp)
add_subdirectory(pron)
add_subdirectory(querystats)
add_subdirectory(querytele)
add_subdirectory(regr)
add_subdirectory(rowgroup)
add_subdirectory(rwlock)
add_subdirectory(startup)
add_subdirectory(statistics_manager)
add_subdirectory(threadpool)
add_subdirectory(udfsdk)
add_subdirectory(windowfunction)

0
utils/common/conststring.h → utils/basic/conststring.h
View File

2
utils/batchloader/CMakeLists.txt
View File

@ -6,4 +6,4 @@ set(batchloader_LIB_SRCS batchloader.cpp)
columnstore_library(batchloader ${batchloader_LIB_SRCS})
columnstore_link(batchloader ${NETSNMP_LIBRARIES} loggingcpp)
columnstore_link(batchloader oamcpp loggingcpp)

2
utils/cacheutils/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(cacheutils_LIB_SRCS cacheutils.cpp)
columnstore_library(cacheutils ${cacheutils_LIB_SRCS})
columnstore_link(cacheutils messageqcpp)
columnstore_link(cacheutils messageqcpp boost_thread)

2
utils/cloudio/CMakeLists.txt
View File

@ -14,7 +14,7 @@ columnstore_library(cloudio ${cloudio_LIB_SRCS})
# IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for
# idbdatafile, we should be able to reverse the dependency like so:
columnstore_link(cloudio idbdatafile messageqcpp)
columnstore_link(cloudio idbdatafile messageqcpp boost_thread)
add_executable(cloudio_component_test component_test.cpp)
add_executable(cloudio_end_to_end_test end_to_end_test.cpp)

13
utils/common/CMakeLists.txt
View File

@ -9,10 +9,17 @@ set(common_LIB_SRCS
MonitorProcMem.cpp
nullvaluemanip.cpp
threadnaming.cpp
utils_utf8.cpp
statistics.cpp
string_prefixes.cpp
)
columnstore_library(common ${common_LIB_SRCS})
columnstore_link(common boost_filesystem configcpp loggingcpp messageqcpp)
columnstore_link(
common
PRIVATE
boost_filesystem
configcpp
loggingcpp
messageqcpp
idbdatafile
mariadb_charset
)

39
utils/common/bit_cast.h
View File

@ -1,39 +0,0 @@
/* Copyright (C) 2020 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <type_traits>
#include <cstring>
namespace utils
{
template <class To, class From>
std::enable_if_t<
sizeof(To) == sizeof(From) && std::is_trivially_copyable_v<From> && std::is_trivially_copyable_v<To>, To>
// constexpr support needs compiler magic
bitCast(const From& src) noexcept
{
static_assert(std::is_trivially_constructible_v<To>,
"This implementation additionally requires "
"destination type to be trivially constructible");
To dst;
std::memcpy(&dst, &src, sizeof(To));
return dst;
}
} // namespace utils

6
utils/common/branchpred.h
View File

@ -19,12 +19,6 @@
#pragma once
#if !defined(__GNUC__) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
#ifndef __builtin_expect
#define __builtin_expect(x, expected_value) (x)
#endif
#endif
#ifndef LIKELY
#define LIKELY(x) __builtin_expect((x), 1)
#define UNLIKELY(x) __builtin_expect((x), 0)

3
utils/common/columnwidth.h
View File

@ -17,6 +17,9 @@
#pragma once
#include <cstdint>
#include <new>
#include "branchpred.h"
namespace utils

2
utils/common/genericparser.h
View File

@ -17,7 +17,7 @@
#pragma once
#include "conststring.h"
#include "basic/conststring.h"
namespace genericparser
{

2
utils/common/hashfamily.h
View File

@ -18,7 +18,7 @@
#pragma once
#include "hasher.h"
#include "collation.h"
#include "mariadb_charset/collation.h"
namespace utils
{

2
utils/common/nullstring.h
View File

@ -25,7 +25,7 @@
#include <iostream>
#include <memory>
#include "exceptclasses.h"
#include "conststring.h"
#include "basic/conststring.h"
#include "mcs_datatype_basic.h"
namespace utils

2
utils/common/string_prefixes.cpp
View File

@ -18,7 +18,7 @@
/* handling of the conversion of string prefixes to int64_t for quick range checking */
#include "collation.h"
#include "mariadb_charset/collation.h"
#include "joblisttypes.h"
#include "string_prefixes.h"

157
utils/common/syncstream.h
View File

@ -1,157 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/** @file */
/*
* classes isyncstream and osyncstream provide a C++ iostream interface
* for C stdio FILE* streams. The current implementation does not provide
* the necessary methods to support seeking. The I/O buffering of the
* input FILE* is used. The C++ iostream library calls syncbuf::sync()
* for every line, so output buffering is line-by-line.
* */
/*
#include "syncstream.h"
void copyStream(istream& iss, ostream& oss)
{
string line;
getline(iss, line);
while (iss.good())
{
oss << line << endl;
getline(iss, line);
}
}
main()
{
FILE* ifp;
FILE* ofp;
...
isyncstream iss(ifp);
osyncstream oss(ofp);
copyStream(iss, oss);
...
}
*/
#pragma once
#include <iostream>
#include <cstdio>
namespace syncstream
{
/** A streambuf implementation for C stdio FILE* streams.
*
* Adapted from http://www.drdobbs.com/184401305
*/
class syncbuf : public std::streambuf
{
public:
/** ctor */
syncbuf(FILE* f) : std::streambuf(), fptr(f)
{
}
protected:
/** Write character in the case of overflow */
virtual int overflow(int c = EOF)
{
return (c != EOF ? fputc(c, fptr) : EOF);
}
/** Get character in the case of overflow */
virtual int underflow()
{
int c = getc(fptr);
if (c != EOF)
ungetc(c, fptr);
return c;
}
/** Get character in the case of overflow and advance get pointer */
virtual int uflow()
{
return getc(fptr);
}
/** put character back in the case of backup underflow */
virtual int pbackfail(int c = EOF)
{
return (c != EOF ? ungetc(c, fptr) : EOF);
}
/** Synchronize stream buffer */
virtual int sync()
{
return fflush(fptr);
}
private:
FILE* fptr;
};
/** An istream adaptor for input FILE* streams */
class isyncstream : public std::istream
{
public:
/** ctor */
isyncstream() : istream(&buf), buf(0)
{
}
/** ctor */
isyncstream(FILE* fptr) : istream(&buf), buf(fptr)
{
}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
};
/** An ostream adaptor for output FILE* streams */
class osyncstream : public std::ostream
{
public:
/** ctor */
osyncstream() : ostream(&buf), buf(0)
{
}
/** ctor */
osyncstream(FILE* fptr) : ostream(&buf), buf(fptr)
{
}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
};
} // namespace syncstream

8
utils/common/utils_utf8.h
View File

@ -20,16 +20,10 @@
#pragma once
#include <cstdint>
#include <string>
#if defined(__FreeBSD__)
//#include <cstdlib>
#else
#include <alloca.h>
#endif
#include <cstdlib>
#include <clocale>
#include "liboamcpp.h"
// Change the name from utf8. Even change the file name to something resembling char helper
namespace utf8

2
utils/ddlcleanup/CMakeLists.txt
View File

@ -2,4 +2,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(ddlcleanuputil_LIB_SRCS ddlcleanuputil.cpp)
columnstore_library(ddlcleanuputil ${ddlcleanuputil_LIB_SRCS})
columnstore_link(ddlcleanuputil PRIVATE loggingcpp ${NETSNMP_LIBRARIES})
columnstore_link(ddlcleanuputil PRIVATE loggingcpp)

5
utils/funcexp/CMakeLists.txt
View File

@ -150,6 +150,7 @@ columnstore_link(
pron
loggingcpp
dataconvert
${MARIADB_STRING_LIBS}
${NETSNMP_LIBRARIES}
mariadb_charset
)
columnstore_link(funcexp PRIVATE ${MARIADB_STRING_LIBS})

2
utils/funcexp/functor_json.h
View File

@ -7,7 +7,7 @@
#include <my_sys.h>
#include <json_lib.h>
#include "collation.h"
#include "mariadb_charset/collation.h"
#include "functor_bool.h"
#include "functor_int.h"
#include "functor_str.h"

4
utils/funcexp/jsonhelpers.h
View File

@ -10,10 +10,10 @@
#include <my_sys.h>
// #include <json_lib.h>
#include "collation.h"
#include "mariadb_charset/collation.h"
#include "functor_json.h"
#include "functor_str.h"
#include "collation.h"
#include "mariadb_charset/collation.h"
#include "rowgroup.h"
#include "treenode.h"
#include "functioncolumn.h"

2
utils/idbdatafile/CMakeLists.txt
View File

@ -14,4 +14,4 @@ set(idbdatafile_LIB_SRCS
)
columnstore_library(idbdatafile ${idbdatafile_LIB_SRCS})
columnstore_link(idbdatafile PRIVATE ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} boost_filesystem boost_system)
columnstore_link(idbdatafile PRIVATE ${ENGINE_OAM_LIBS} boost_filesystem boost_system compress)

2
utils/joiner/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(joiner_LIB_SRCS tuplejoiner.cpp joinpartition.cpp)
columnstore_library(joiner ${joiner_LIB_SRCS})
columnstore_link(joiner PRIVATE loggingcpp)
columnstore_link(joiner PRIVATE loggingcpp rowgroup datatypes compress)

7
utils/mariadb_charset/CMakeLists.txt Normal file
View File

@ -0,0 +1,7 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
columnstore_static_library(mariadb_charset charset.cpp)
columnstore_link(mariadb_charset PRIVATE ${MARIADB_STRING_LIBS})
add_dependencies(mariadb_charset loggingcpp)

17
utils/common/utils_utf8.cpp → utils/mariadb_charset/charset.cpp
View File

@ -1,8 +1,9 @@
/* Copyright (C) 2020 MariaDB Corporation.
/* Copyright (C) 2025 MariaDB Corporation
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -11,11 +12,12 @@
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */
#include "utils_utf8.h"
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "mariadb_charset/collation.h"
#include "mariadb_my_sys.h"
#include "mcs_datatype.h"
namespace datatypes
{
@ -35,4 +37,3 @@ void Charset::setCharset(uint32_t charsetNumber)
}
} // namespace datatypes

2
utils/common/collation.h → utils/mariadb_charset/collation.h
View File

@ -35,7 +35,7 @@
#include "mcsconfig.h"
#include "exceptclasses.h"
#include "conststring.h"
#include "basic/conststring.h"
/*
Redefine definitions used by MariaDB m_ctype.h.

1
utils/messageqcpp/CMakeLists.txt
View File

@ -8,7 +8,6 @@ set(messageqcpp_LIB_SRCS
iosocket.cpp
messagequeue.cpp
messagequeuepool.cpp
samenodepseudosocket.cpp
socketparms.cpp
)

92
utils/messageqcpp/messagequeuepool.cpp
View File

@ -25,40 +25,57 @@
#include <new>
#include <type_traits>
namespace messageqcpp
{
using ClientMapType = std::multimap<std::string, std::unique_ptr<ClientObject>>;
struct LockedClientMap
template <class T>
struct Immortal
{
LockedClientMap()
template <class... Args>
Immortal(Args&&... args)
{
::new (space) T(std::forward<Args>(args)...);
}
~LockedClientMap()
operator T&() & noexcept
{
return reinterpret_cast<T&>(space);
}
ClientMapType clientMap;
std::mutex queueMutex;
private:
alignas(T) unsigned char space[sizeof(T)];
};
static int clientMapNiftyCounter;
static typename std::aligned_storage<sizeof(LockedClientMap), alignof(LockedClientMap)>::type clientMapBuf;
auto& lockedMap = reinterpret_cast<LockedClientMap&>(clientMapBuf);
LockedClientMapInitilizer::LockedClientMapInitilizer ()
class LockedClientMap
{
if (clientMapNiftyCounter++ == 0) new (&lockedMap) LockedClientMap (); // placement new
}
LockedClientMapInitilizer::~LockedClientMapInitilizer ()
{
if (--clientMapNiftyCounter == 0) (&lockedMap)->~LockedClientMap();
}
struct KeyToUsePrivateCtor
{
explicit KeyToUsePrivateCtor() = default;
};
public:
LockedClientMap(const LockedClientMap&) = delete;
LockedClientMap& operator=(const LockedClientMap&) = delete;
~LockedClientMap() = delete;
static LockedClientMap& getInstance()
{
static Immortal<LockedClientMap> instance(KeyToUsePrivateCtor{});
return instance;
}
LockedClientMap(KeyToUsePrivateCtor)
{
}
private:
ClientMapType clientMap;
std::mutex queueMutex;
friend class MessageQueueClientPool;
};
// 300 seconds idle until cleanup
#define MAX_IDLE_TIME 300
@ -70,7 +87,7 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts)
MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrIp, uint64_t port)
{
auto lock = std::scoped_lock(lockedMap.queueMutex);
auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex);
std::ostringstream oss;
oss << dnOrIp << "_" << port;
@ -93,14 +110,13 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrI
newClientObject->client.reset(new MessageQueueClient(dnOrIp, port));
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
lockedMap.clientMap.emplace(std::move(searchString), std::move(newClientObject));
LockedClientMap::getInstance().clientMap.emplace(std::move(searchString), std::move(newClientObject));
return newClientObject->client.get();
}
MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& module)
{
auto lock = std::scoped_lock(lockedMap.queueMutex);
auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex);
MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module);
@ -116,13 +132,11 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& modul
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client.reset(new MessageQueueClient(module));
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
auto result = newClientObject->client.get();
lockedMap.clientMap.emplace(std::move(module), std::move(newClientObject));
LockedClientMap::getInstance().clientMap.emplace(std::move(module), std::move(newClientObject));
return result;
}
@ -133,11 +147,10 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search
uint64_t nowSeconds = TimeSpecToSeconds(&now);
MessageQueueClient* returnClient = NULL;
auto it = lockedMap.clientMap.begin();
auto it = LockedClientMap::getInstance().clientMap.begin();
// Scan pool
while (it != lockedMap.clientMap.end())
while (it != LockedClientMap::getInstance().clientMap.end())
{
ClientObject* clientObject = it->second.get();
uint64_t elapsedTime = nowSeconds - clientObject->lastUsed;
@ -149,7 +162,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search
// Do this so we don't invalidate current interator
auto toDelete = it;
it++;
lockedMap.clientMap.erase(toDelete);
LockedClientMap::getInstance().clientMap.erase(toDelete);
continue;
}
@ -163,7 +176,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search
// Do this so we don't invalidate current interator
auto toDelete = it;
it++;
lockedMap.clientMap.erase(toDelete);
LockedClientMap::getInstance().clientMap.erase(toDelete);
continue;
}
}
@ -193,10 +206,10 @@ void MessageQueueClientPool::releaseInstance(MessageQueueClient* client)
if (client == NULL)
return;
auto lock = std::scoped_lock(lockedMap.queueMutex);
auto it = lockedMap.clientMap.begin();
auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex);
auto it = LockedClientMap::getInstance().clientMap.begin();
while (it != lockedMap.clientMap.end())
while (it != LockedClientMap::getInstance().clientMap.end())
{
if (it->second->client.get() == client)
{
@ -221,15 +234,14 @@ void MessageQueueClientPool::deleteInstance(MessageQueueClient* client)
if (client == NULL)
return;
auto lock = std::scoped_lock(LockedClientMap::getInstance().queueMutex);
auto it = LockedClientMap::getInstance().clientMap.begin();
auto lock = std::scoped_lock(lockedMap.queueMutex);
auto it = lockedMap.clientMap.begin();
while (it != lockedMap.clientMap.end())
while (it != LockedClientMap::getInstance().clientMap.end())
{
if (it->second->client.get() == client)
{
lockedMap.clientMap.erase(it);
LockedClientMap::getInstance().clientMap.erase(it);
return;
}

10
utils/messageqcpp/messagequeuepool.h
View File

@ -26,12 +26,6 @@
namespace messageqcpp
{
static struct LockedClientMapInitilizer {
LockedClientMapInitilizer ();
~LockedClientMapInitilizer ();
} clientMapInitilizer; // static initializer for every translation unit
struct ClientObject
{
std::unique_ptr<MessageQueueClient> client;
@ -49,8 +43,8 @@ class MessageQueueClientPool
static MessageQueueClient* findInPool(const std::string& search);
private:
MessageQueueClientPool(){};
~MessageQueueClientPool(){};
MessageQueueClientPool() {};
~MessageQueueClientPool() {};
};
} // namespace messageqcpp

2
utils/querystats/CMakeLists.txt
View File

@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(querystats_LIB_SRCS querystats.cpp)
columnstore_library(querystats ${querystats_LIB_SRCS})
columnstore_link(querystats PRIVATE loggingcpp)
columnstore_link(querystats PRIVATE loggingcpp messageqcpp)

4
utils/querytele/CMakeLists.txt
View File

@ -5,7 +5,7 @@ set(querytele_LIB_SRCS querytele.cpp queryteleclient.cpp querytele_constants.cpp
)
columnstore_library(querytele ${querytele_LIB_SRCS})
columnstore_link(querytele ${THRIFT_LIBRARY})
columnstore_link(querytele ${THRIFT_LIBRARY} boost_thread)
target_include_directories(querytele PRIVATE ${THRIFT_INCLUDE_DIRS})
add_dependencies(querytele external_boost external_thrift)
add_dependencies(querytele external_thrift)

5
utils/regr/CMakeLists.txt
View File

@ -21,11 +21,12 @@ set(regr_LIB_SRCS
add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
columnstore_library(regr ${regr_LIB_SRCS})
columnstore_link(regr PRIVATE loggingcpp)
columnstore_link(regr PRIVATE loggingcpp messageqcpp)
set(regr_mysql_LIB_SRCS regrmysql.cpp modamysql.cpp)
# Do anyone use it?
columnstore_mysql_plugin_library(regr_mysql SHARED ${regr_mysql_LIB_SRCS})
add_dependencies(regr_mysql external_boost)
add_dependencies(regr_mysql external_boost GenError) # for "idb_mysql.h" that uses generated mysqld_error.h
set_target_properties(regr_mysql PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../)

6
utils/regr/moda.h
View File

@ -45,7 +45,7 @@
#include "calpontsystemcatalog.h"
#include "windowfunctioncolumn.h"
#include "hasher.h"
#include "collation.h"
#include "mariadb_charset/collation.h"
#define EXPORT
@ -277,7 +277,7 @@ class Moda_impl_T<string> : public mcsv1_UDAF
{
public:
// Defaults OK
Moda_impl_T() : cs(8){};
Moda_impl_T() : cs(8) {};
~Moda_impl_T() override = default;
mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override;
@ -305,7 +305,7 @@ class moda : public mcsv1_UDAF
{
public:
// Defaults OK
moda() : mcsv1_UDAF(){};
moda() : mcsv1_UDAF() {};
~moda() override = default;
mcsv1_UDAF::ReturnCode init(mcsv1Context* context, ColumnDatum* colTypes) override;

2
utils/rowgroup/CMakeLists.txt
View File

@ -5,5 +5,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(rowgroup_LIB_SRCS rowaggregation.cpp rowgroup.cpp rowstorage.cpp)
columnstore_library(rowgroup ${rowgroup_LIB_SRCS})
columnstore_link(rowgroup PRIVATE ${NETSNMP_LIBRARIES} funcexp loggingcpp)
columnstore_link(rowgroup PRIVATE funcexp loggingcpp compress)
add_dependencies(rowgroup external_boost)

2
utils/rowgroup/rowgroup.h
View File

@ -53,7 +53,7 @@
#include "branchpred.h"
#include "datatypes/mcs_int128.h"
#include "collation.h"
#include "mariadb_charset/collation.h"
#include "common/hashfamily.h"
#include "buffertypes.h"

6
utils/statistics_manager/CMakeLists.txt Normal file
View File

@ -0,0 +1,6 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
columnstore_static_library(statistics_manager statistics.cpp)
columnstore_link(statistics_manager PRIVATE rowgroup)

5
utils/common/statistics.cpp → utils/statistics_manager/statistics.cpp
View File

@ -116,9 +116,8 @@ void StatisticsManager::analyzeSample(bool traceOn)
// MCV statistics.
std::vector<pair<uint64_t, uint32_t>> mcvList(columnMCV.begin(), columnMCV.end());
std::sort(mcvList.begin(), mcvList.end(),
[](const std::pair<uint64_t, uint32_t>& a, const std::pair<uint64_t, uint32_t>& b) {
return a.second > b.second;
});
[](const std::pair<uint64_t, uint32_t>& a, const std::pair<uint64_t, uint32_t>& b)
{ return a.second > b.second; });
// 200 buckets as Microsoft does.
const auto mcvSize = std::min(columnMCV.size(), static_cast<uint64_t>(200));

0
utils/common/statistics.h → utils/statistics_manager/statistics.h
View File

4
utils/threadpool/CMakeLists.txt
View File

@ -2,6 +2,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp fair_threadpool.cpp)
columnstore_library(threadpool ${threadpool_LIB_SRCS})
columnstore_link(threadpool PRIVATE boost_chrono loggingcpp)
add_dependencies(threadpool external_boost)
columnstore_link(threadpool PRIVATE boost_chrono boost_thread loggingcpp messageqcpp)

4
utils/udfsdk/CMakeLists.txt
View File

@ -11,9 +11,11 @@ set(udfsdk_LIB_SRCS
)
columnstore_library(udfsdk ${udfsdk_LIB_SRCS})
columnstore_link(udfsdk PRIVATE loggingcpp)
columnstore_link(udfsdk PRIVATE loggingcpp messageqcpp)
# Do anyone use it?
add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
set(udf_mysql_LIB_SRCS udfmysql.cpp)
columnstore_mysql_plugin_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS})
add_dependencies(udf_mysql GenError) # for "idb_mysql.h" that uses generated mysqld_error.h

28
versioning/BRM/CMakeLists.txt
View File

@ -34,7 +34,7 @@ set(brm_LIB_SRCS
)
columnstore_library(brm ${brm_LIB_SRCS})
columnstore_link(brm loggingcpp datatypes)
columnstore_link(brm loggingcpp datatypes oamcpp boost_thread messageqcpp)
# ########## next target ###############
@ -62,53 +62,41 @@ columnstore_link(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS
set(reset_locks_SRCS reset_locks.cpp)
columnstore_executable(reset_locks ${reset_locks_SRCS})
columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES})
columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
# ########## next target ###############
set(rollback_SRCS rollback.cpp)
columnstore_executable(rollback ${rollback_SRCS})
columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES})
columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
# ########## next target ###############
set(save_brm_SRCS save_brm.cpp)
columnstore_executable(save_brm ${save_brm_SRCS})
columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES})
columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
# ########## next target ###############
set(load_brm_SRCS load_brm.cpp)
columnstore_executable(load_brm ${load_brm_SRCS})
columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES})
columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
columnstore_executable(mcs-load-em load_em.cpp)
columnstore_link(
mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}
)
columnstore_link(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS})
columnstore_executable(mcs-load-brm-from-file load_brm_from_file.cpp)
columnstore_link(
mcs-load-brm-from-file
${ENGINE_LDFLAGS}
${MARIADB_CLIENT_LIBS}
${ENGINE_OAM_LIBS}
${ENGINE_EXEC_LIBS}
${NETSNMP_LIBRARIES}
mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}
boost_program_options
)
columnstore_executable(mcs-shmem-locks shmem_locks.cpp)
columnstore_link(
mcs-shmem-locks
${ENGINE_LDFLAGS}
${MARIADB_CLIENT_LIBS}
${ENGINE_OAM_LIBS}
${ENGINE_EXEC_LIBS}
${NETSNMP_LIBRARIES}
mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}
boost_program_options
)

6
writeengine/bulk/CMakeLists.txt
View File

@ -29,7 +29,7 @@ set(we_bulk_STAT_SRCS
add_definitions(-D_FILE_OFFSET_BITS=64)
columnstore_static_library(we_bulk ${we_bulk_STAT_SRCS})
columnstore_link(we_bulk ${NETSNMP_LIBRARIES} loggingcpp boost_program_options)
columnstore_link(we_bulk loggingcpp boost_program_options)
remove_definitions(-D_FILE_OFFSET_BITS=64)
@ -38,13 +38,13 @@ remove_definitions(-D_FILE_OFFSET_BITS=64)
set(cpimport.bin_SRCS cpimport.cpp)
columnstore_executable(cpimport.bin ${cpimport.bin_SRCS})
add_dependencies(cpimport.bin marias3)
columnstore_link(
cpimport.bin
${ENGINE_LDFLAGS}
${NETSNMP_LIBRARIES}
${ENGINE_WRITE_LIBS}
${S3API_DEPS}
we_bulk
we_xml
marias3
)

Some files were not shown because too many files have changed in this diff Show More