database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

Merge pull request #545 from mariadb-corporation/1.1-merge-up-20180817

Browse Source 
Merge develop-1.1 into develop
This commit is contained in:
David.Hall
2018-08-20 14:07:35 -05:00
committed by GitHub
parent a7810f0d3c 03da6df473
commit 23e99db0ea
47 changed files with 1526 additions and 756 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -105,4 +105,4 @@ install_manifest_platform.txt
install_manifest_storage-engine.txt
_CPack_Packages
columnstoreversion.h
.idea/

1
README
View File

@@ -9,3 +9,4 @@ Additional features will be pushed in future releases.
A few things to notice:
- Do not use pre-releases on production systems.
- The building of the ColumnStore engine needs a special build environment. We're working on making it available for everyone to build.

4
cpackEngineRPM.cmake
View File

@@ -180,7 +180,6 @@ SET(CPACK_RPM_platform_USER_FILELIST
"/usr/local/mariadb/columnstore/bin/resourceReport.sh"
"/usr/local/mariadb/columnstore/bin/hadoopReport.sh"
"/usr/local/mariadb/columnstore/bin/alarmReport.sh"
"/usr/local/mariadb/columnstore/bin/amazonInstaller"
"/usr/local/mariadb/columnstore/bin/remote_command_verify.sh"
"/usr/local/mariadb/columnstore/bin/disable-rep-columnstore.sh"
"/usr/local/mariadb/columnstore/bin/columnstore.service"
@@ -216,6 +215,9 @@ SET(CPACK_RPM_platform_USER_FILELIST
"/usr/local/mariadb/columnstore/bin/os_detect.sh"
"/usr/local/mariadb/columnstore/bin/columnstoreClusterTester.sh"
"/usr/local/mariadb/columnstore/bin/mariadb-command-line.sh"
"/usr/local/mariadb/columnstore/bin/quick_installer_single_server.sh"
"/usr/local/mariadb/columnstore/bin/quick_installer_multi_server.sh"
"/usr/local/mariadb/columnstore/bin/quick_installer_amazon.sh"
${ignored})
SET(CPACK_RPM_libs_USER_FILELIST

3
dbcon/ddlpackage/CMakeLists.txt
View File

@@ -1,4 +1,3 @@
INCLUDE_DIRECTORIES( ${ENGINE_COMMON_INCLUDES} )
ADD_CUSTOM_COMMAND(
@@ -13,7 +12,7 @@ ADD_CUSTOM_TARGET(ddl-lexer DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ddl-scan.cpp)
ADD_CUSTOM_TARGET(ddl-parser DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ddl-gram.cpp)
# Parser puts extra info to stderr.
INCLUDE(../../check_compiler_flag.cmake)
MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG" DEBUG)
MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG=1" DEBUG)
########### next target ###############

3
dbcon/ddlpackage/ddl.l
View File

@@ -33,6 +33,9 @@
using namespace ddlpackage;
typedef enum { NOOP, STRIP_QUOTES } copy_action_t;
#if YYDEBUG == 0
int ddldebug = 0;
#endif
int lineno = 1;
void ddlerror(struct pass_to_bison* x, char const *s);

1
dbcon/ddlpackage/ddl.y
View File

@@ -63,7 +63,6 @@ char* copy_string(const char *str);
%pure-parser
%lex-param {void * scanner}
%parse-param {struct ddlpackage::pass_to_bison * x}
%debug
/* Bison uses this to generate a C union definition. This is used to
store the application created values associated with syntactic

4
dbcon/execplan/arithmeticcolumn.cpp
View File

@@ -368,11 +368,7 @@ bool ArithmeticColumn::operator==(const ArithmeticColumn& t) const
else if (fExpression != NULL || t.fExpression != NULL)
return false;
if (fAlias != t.fAlias)
return false;
if (fTableAlias != t.fTableAlias)
return false;
if (fData != t.fData)
return false;

31
dbcon/execplan/clientrotator.cpp
View File

@@ -49,6 +49,15 @@ using namespace logging;
#include "clientrotator.h"
//#include "idb_mysql.h"
/** Debug macro */
#ifdef INFINIDB_DEBUG
#define IDEBUG(x) {x;}
#else
#define IDEBUG(x) {}
#endif
#define LOG_TO_CERR
namespace execplan
@@ -60,14 +69,36 @@ const uint64_t LOCAL_EXEMGR_PORT = 8601;
string ClientRotator::getModule()
{
string installDir = startup::StartUp::installDir();
//Log to debug.log
LoggingID logid( 24, 0, 0);
string fileName = installDir + "/local/module";
string module;
ifstream moduleFile (fileName.c_str());
if (moduleFile.is_open())
{
getline (moduleFile, module);
}
else
{
{
logging::Message::Args args1;
logging::Message msg(1);
std::ostringstream oss;
oss << "ClientRotator::getModule open status2 =" << strerror(errno);
args1.add(oss.str());
args1.add(fileName);
msg.format( args1 );
Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_DEBUG, msg, logid);
}
}
moduleFile.close();
return module;
}

2
dbcon/execplan/constantcolumn.cpp
View File

@@ -328,8 +328,6 @@ bool ConstantColumn::operator==(const ConstantColumn& t) const
if (fType != t.fType)
return false;
if (fAlias != t.fAlias)
return false;
if (fData != t.fData)
return false;

2
dbcon/execplan/functioncolumn.cpp
View File

@@ -332,8 +332,6 @@ bool FunctionColumn::operator==(const FunctionColumn& t) const
if (**it != **it2)
return false;
// if (fAlias != t.fAlias)
// return false;
if (fTableAlias != t.fTableAlias)
return false;

9
dbcon/execplan/returnedcolumn.cpp
View File

@@ -173,14 +173,13 @@ void ReturnedColumn::unserialize(messageqcpp::ByteStream& b)
bool ReturnedColumn::operator==(const ReturnedColumn& t) const
{
// Not all fields are considered for a positive equality.
if (fData != t.fData)
return false;
if (fCardinality != t.fCardinality)
return false;
//if (fAlias != t.fAlias)
// return false;
if (fDistinct != t.fDistinct)
return false;
@@ -193,24 +192,18 @@ bool ReturnedColumn::operator==(const ReturnedColumn& t) const
if (fNullsFirst != t.fNullsFirst)
return false;
//if (fOrderPos != t.fOrderPos)
// return false;
if (fInputIndex != t.fInputIndex)
return false;
if (fOutputIndex != t.fOutputIndex)
return false;
//if (fSequence != t.fSequence)
// return false;
if (fResultType != t.fResultType)
return false;
if (fOperationType != t.fOperationType)
return false;
//if (fExpressionId != t.fExpressionId)
// return false;
return true;
}

6
dbcon/execplan/simplecolumn.cpp
View File

@@ -346,7 +346,6 @@ void SimpleColumn::serialize(messageqcpp::ByteStream& b) const
b << fViewName;
b << (uint32_t) fOid;
b << fData;
//b << fAlias;
b << fTableAlias;
b << (uint32_t) fSequence;
b << static_cast<const ByteStream::doublebyte>(fIsInfiniDB);
@@ -363,7 +362,6 @@ void SimpleColumn::unserialize(messageqcpp::ByteStream& b)
b >> fViewName;
b >> (uint32_t&) fOid;
b >> fData;
//b >> fAlias;
b >> fTableAlias;
b >> (uint32_t&) fSequence;
b >> reinterpret_cast< ByteStream::doublebyte&>(fIsInfiniDB);
@@ -388,8 +386,6 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const
if (fColumnName != t.fColumnName)
return false;
// if (fIndexName != t.fIndexName)
// return false;
if (fViewName != t.fViewName)
return false;
@@ -399,8 +395,6 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const
if (data() != t.data())
return false;
// if (fAlias != t.fAlias)
// return false;
if (fTableAlias != t.fTableAlias)
return false;

11
dbcon/joblist/fifo.h
View File

@@ -398,20 +398,29 @@ void FIFO<element_t>::signalPs()
template<typename element_t>
inline bool FIFO<element_t>::next(uint64_t id, element_t* out)
{
base::mutex.lock();
fConsumptionStarted = true;
if (cpos[id] >= fMaxElements)
{
base::mutex.unlock();
if (!waitForSwap(id))
return false;
base::mutex.lock();
}
*out = cBuffer[cpos[id]++];
#ifndef ONE_CS
if (cpos[id] == fMaxElements)
{
base::mutex.unlock();
signalPs();
return true;
}
#endif
base::mutex.unlock();
return true;
}

7
dbcon/mysql/ha_calpont_dml.cpp
View File

@@ -1820,8 +1820,11 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_
}
else if (ci.columnTypes[colpos].colWidth < 16777216)
{
dataLength = *(uint32_t*) buf;
buf = buf + 3 ;
dataLength = *(uint16_t*) buf;
buf = buf + 2 ;
if (*(uint8_t*)buf)
dataLength += 256*256*(*(uint8_t*)buf) ;
buf++;
}
else
{

59
dbcon/mysql/ha_calpont_execplan.cpp
View File

@@ -1548,8 +1548,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip)
ifp->functype() == Item_func::ISNOTNULL_FUNC)
{
ReturnedColumn* rhs = NULL;
if (!gwip->rcWorkStack.empty())
if (!gwip->rcWorkStack.empty() && !gwip->inCaseStmt)
{
rhs = gwip->rcWorkStack.top();
gwip->rcWorkStack.pop();
@@ -1650,8 +1649,49 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip)
idbassert(ifp->argument_count() == 1);
ParseTree* ptp = 0;
if (((Item_func*)(ifp->arguments()[0]))->functype() == Item_func::EQUAL_FUNC)
{
// negate it in place
// Note that an EQUAL_FUNC ( a <=> b) was converted to
// ( a = b OR ( a is null AND b is null) )
// NOT of the above expression is: ( a != b AND (a is not null OR b is not null )
if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM)
if (!gwip->ptWorkStack.empty())
ptp = gwip->ptWorkStack.top();
if (ptp)
{
ParseTree* or_ptp = ptp;
ParseTree* and_ptp = or_ptp->right();
ParseTree* equal_ptp = or_ptp->left();
ParseTree* nullck_left_ptp = and_ptp->left();
ParseTree* nullck_right_ptp = and_ptp->right();
SimpleFilter *sf_left_nullck = dynamic_cast<SimpleFilter*>(nullck_left_ptp->data());
SimpleFilter *sf_right_nullck = dynamic_cast<SimpleFilter*>(nullck_right_ptp->data());
SimpleFilter *sf_equal = dynamic_cast<SimpleFilter*>(equal_ptp->data());
if (sf_left_nullck && sf_right_nullck && sf_equal) {
// Negate the null checks
sf_left_nullck->op()->reverseOp();
sf_right_nullck->op()->reverseOp();
sf_equal->op()->reverseOp();
// Rehook the nodes
ptp = and_ptp;
ptp->left(equal_ptp);
ptp->right(or_ptp);
or_ptp->left(nullck_left_ptp);
or_ptp->right(nullck_right_ptp);
gwip->ptWorkStack.pop();
gwip->ptWorkStack.push(ptp);
}
else {
gwip->fatalParseError = true;
gwip->parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ASSERTION_FAILURE);
return false;
}
}
}
else if (isPredicateFunction(ifp->arguments()[0], gwip) || ifp->arguments()[0]->type() == Item::COND_ITEM)
{
// negate it in place
if (!gwip->ptWorkStack.empty())
@@ -1725,7 +1765,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip)
}
else if (ifp->functype() == Item_func::EQUAL_FUNC)
{
// a = b OR (a IS NULL AND b IS NULL)
// Convert "a <=> b" to (a = b OR (a IS NULL AND b IS NULL))"
idbassert (gwip->rcWorkStack.size() >= 2);
ReturnedColumn* rhs = gwip->rcWorkStack.top();
gwip->rcWorkStack.pop();
@@ -1737,7 +1777,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip)
// b IS NULL
ConstantColumn* nlhs1 = new ConstantColumn("", ConstantColumn::NULLDATA);
sop.reset(new PredicateOperator("isnull"));
sop->setOpType(lhs->resultType(), rhs->resultType());
sop->setOpType(lhs->resultType(), rhs->resultType());
sfn1 = new SimpleFilter(sop, rhs, nlhs1);
ParseTree* ptpl = new ParseTree(sfn1);
// a IS NULL
@@ -1752,7 +1792,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip)
ptpn->right(ptpr);
// a = b
sop.reset(new PredicateOperator("="));
sop->setOpType(lhs->resultType(), lhs->resultType());
sop->setOpType(lhs->resultType(), rhs->resultType());
sfo = new SimpleFilter(sop, lhs->clone(), rhs->clone());
// OR with the NULL comparison tree
ParseTree* ptp = new ParseTree(new LogicOperator("or"));
@@ -3772,8 +3812,12 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS
if (funcName == "case_searched" &&
(i < arg_offset))
{
// MCOL-1472 Nested CASE with an ISNULL predicate. We don't want the predicate
// to pull off of rcWorkStack, so we set this inCaseStmt flag to tell it
// not to.
gwi.inCaseStmt = true;
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
gwi.inCaseStmt = false;
if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data())
{
gwi.ptWorkStack.pop();
@@ -10233,3 +10277,4 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro
}
// vim:ts=4 sw=4:

4
dbcon/mysql/ha_calpont_impl.cpp
View File

@@ -1964,7 +1964,7 @@ uint32_t doUpdateDelete(THD* thd)
}
else
{
thd->set_row_count_func(dmlRowCount);
thd->set_row_count_func(dmlRowCount+thd->get_row_count_func());
}
push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, errorMsg.c_str());
@@ -1972,7 +1972,7 @@ uint32_t doUpdateDelete(THD* thd)
else
{
// if (dmlRowCount != 0) //Bug 5117. Handling self join.
thd->set_row_count_func(dmlRowCount);
thd->set_row_count_func(dmlRowCount+thd->get_row_count_func());
//cout << " error status " << ci->rc << " and rowcount = " << dmlRowCount << endl;

6
dbcon/mysql/ha_calpont_impl_if.h
View File

@@ -149,6 +149,9 @@ struct gp_walk_info
int32_t recursionHWM;
std::stack<int32_t> rcBookMarkStack;
// Kludge for MCOL-1472
bool inCaseStmt;
gp_walk_info() : sessionid(0),
fatalParseError(false),
condPush(false),
@@ -164,7 +167,8 @@ struct gp_walk_info
lastSub(0),
derivedTbCnt(0),
recursionLevel(-1),
recursionHWM(0)
recursionHWM(0),
inCaseStmt(false)
{}
~gp_walk_info() {}

4
exemgr/main.cpp
View File

@@ -1443,7 +1443,9 @@ int main(int argc, char* argv[])
#endif
setupSignalHandlers();
int err = setupResources();
int err = 0;
if (!gDebug)
err = setupResources();
string errMsg;
switch (err)

2
oam/etc/Columnstore.xml
View File

@@ -438,7 +438,7 @@
<SystemStartupOffline>n</SystemStartupOffline>
<InitialInstallFlag>n</InitialInstallFlag>
<SingleServerInstall>n</SingleServerInstall>
<ServerTypeInstall>2</ServerTypeInstall>
<ServerTypeInstall>1</ServerTypeInstall>
<PMwithUM>n</PMwithUM>
<MySQLRep>n</MySQLRep>
<DBRootStorageType>internal</DBRootStorageType>

4
oam/etc/ProcessConfig.xml
View File

@@ -97,7 +97,7 @@
<DepProcessName1>WriteEngineServer</DepProcessName1>
<DepModuleName1>pm*</DepModuleName1>
<DepProcessName2>DBRMWorkerNode</DepProcessName2>
<DepModuleName2>*</DepModuleName2>
<DepModuleName2>@</DepModuleName2>
<DepProcessName3>ExeMgr</DepProcessName3>
<DepModuleName3>*</DepModuleName3>
<RunType>SIMPLEX</RunType>
@@ -112,7 +112,7 @@
<DepProcessName1>WriteEngineServer</DepProcessName1>
<DepModuleName1>pm*</DepModuleName1>
<DepProcessName2>DBRMWorkerNode</DepProcessName2>
<DepModuleName2>*</DepModuleName2>
<DepModuleName2>@</DepModuleName2>
<DepProcessName3>DDLProc</DepProcessName3>
<DepModuleName3>@</DepModuleName3>
<RunType>SIMPLEX</RunType>

6
oam/install_scripts/columnstore
View File

@@ -94,13 +94,9 @@ start() {
CoreFileFlag=`$InstallDir/bin/getConfig -c $InstallDir/etc/Columnstore.xml Installation CoreFileFlag`
if [ $CoreFileFlag = "y" ]; then
SUDO=
if [ "$user" != "root" ]; then
SUDO="$SUDO"
fi
#columnstore core files
$SUDO ulimit -c unlimited > /dev/null 2>&1
ulimit -c unlimited > /dev/null 2>&1
$SUDO sysctl -q -w kernel.core_uses_pid=1 > /dev/null 2>&1
$SUDO sysctl -q -w kernel.core_pattern=/var/log/mariadb/columnstore/corefiles/core.%e.%p > /dev/null 2>&1
fi

4
oam/install_scripts/module_installer.sh
View File

@@ -186,7 +186,6 @@ if [ $module = "um" ]; then
echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE
echo "done" >> $RCFILE
else
sudo chmod 666 $RCFILE
sudo echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE
sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE
sudo echo "done" >> $RCFILE
@@ -201,7 +200,6 @@ else
echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE
echo "done" >> $RCFILE
else
sudo chmod 666 $RCFILE
sudo echo "for scsi_dev in \`mount | awk '/mnt\\/tmp/ {print $1}' | awk -F/ '{print $3}' | sed 's/[0-9]*$//'\`; do" >> $RCFILE
sudo echo "echo deadline > /sys/block/$scsi_dev/queue/scheduler" >> $RCFILE
sudo echo "done" >> $RCFILE
@@ -214,7 +212,7 @@ fi
if [ $user != "root" ]; then
echo "uncomment runuser in rc.local"
sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1
sudo sed -i -e 's/#sudo runuser/sudo runuser/g' $RCFILE >/dev/null 2>&1
fi
echo "!!!Module Installation Successfully Completed!!!"

2
oam/install_scripts/post-install
View File

@@ -164,7 +164,7 @@ mkdir -p /tmp/columnstore_tmp_files >/dev/null 2>&1
#setup core file directory and link
mkdir /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1
chmod 755 /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1
chmod 777 /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1
#create mount directories
mkdir /mnt/tmp > /dev/null 2>&1

102
oam/oamcpp/liboamcpp.cpp
View File

@@ -5615,6 +5615,7 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s
dbrootList dbroot1;
dbroot1.push_back(*pt1);
bool returnDbRoot = false;
//send msg to unmount dbroot if module is not offline
int opState;
@@ -5629,7 +5630,6 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s
if (opState != oam::AUTO_OFFLINE || opState != oam::AUTO_DISABLED)
{
// bool unmountPass = true;
try
{
mountDBRoot(dbroot1, false);
@@ -5639,13 +5639,8 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s
writeLog("ERROR: dbroot failed to unmount", LOG_TYPE_ERROR );
cout << endl << "ERROR: umountDBRoot api failure" << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
// unmountPass = false;
}
// if ( !unmountPass) {
// dbrootlist.erase(pt1);
// break;
// }
}
//check for amazon moving required
@@ -5663,40 +5658,79 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s
//if Gluster, do the assign command
if ( DataRedundancyConfig == "y")
{
try
{
try
{
string errmsg;
int ret = glusterctl(oam::GLUSTER_ASSIGN, *pt1, toPM, errmsg);
if ( ret != 0 )
if ( ret == 0 )
{
todbrootConfigList.push_back(*pt2);
residedbrootConfigList.erase(pt2);
}
else
{
cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID + ", error: " + errmsg << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID + ", error: " + errmsg, LOG_TYPE_ERROR );
returnDbRoot = true;
}
}
catch (exception& e)
{
cout << endl << "**** glusterctl API exception: " << e.what() << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID, LOG_TYPE_ERROR );
returnDbRoot = true;
}
catch (...)
{
cout << endl << "**** glusterctl API exception: UNKNOWN" << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID, LOG_TYPE_ERROR );
returnDbRoot = true;
}
}
todbrootConfigList.push_back(*pt2);
residedbrootConfigList.erase(pt2);
if (returnDbRoot)
{
// something went wrong return it back to original owner
try
{
string errmsg;
writeLog("reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR );
int ret = glusterctl(oam::GLUSTER_ASSIGN, *pt1, residePM, errmsg);
if ( ret != 0 )
{
cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID + ", error: " + errmsg << endl;
writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID + ", error: " + errmsg, LOG_TYPE_ERROR );
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
mountDBRoot(dbroot1);
//get updated Columnstore.xml distributed
distributeConfigFile("system");
return;
}
catch (exception& e)
{
cout << endl << "**** glusterctl API exception: " << e.what() << endl;
cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID << endl;
writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR );
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
catch (...)
{
cout << endl << "**** glusterctl API exception: UNKNOWN" << endl;
cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID << endl;
writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR );
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
}
break;
}
}
}
//set the 2 pms dbroot config
try
{
@@ -5706,7 +5740,7 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s
{
writeLog("ERROR: setPmDbrootConfig api failure for pm" + residePMID, LOG_TYPE_ERROR );
cout << endl << "ERROR: setPmDbrootConfig api failure for pm" + residePMID << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
try
@@ -5717,7 +5751,7 @@ void Oam::manualMovePmDbroot(std::string residePM, std::string dbrootIDs, std::s
{
writeLog("ERROR: setPmDbrootConfig api failure for pm" + toPMID, LOG_TYPE_ERROR );
cout << endl << "ERROR: setPmDbrootConfig api failure for pm" + toPMID << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
//send msg to mount dbroot
@@ -6361,7 +6395,7 @@ bool Oam::autoUnMovePmDbroot(std::string toPM)
if (!found)
{
writeLog("No dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_DEBUG );
writeLog("No dbroots found in " + InstallDir + "/moveDbrootTransactionLog", LOG_TYPE_DEBUG );
cout << "No dbroots found in " << fileName << endl;
}
@@ -6950,32 +6984,6 @@ void Oam::assignDbroot(std::string toPM, DBRootConfigList& dbrootlist)
for ( ; pt3 != dbrootlist.end() ; pt3++)
{
todbrootConfigList.push_back(*pt3);
/* if ( DataRedundancyConfig == "y")
{
try {
string errmsg;
int ret = glusterctl(oam::GLUSTER_ASSIGN, itoa(*pt3), toPM, errmsg);
if ( ret != 0 )
{
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID + ", error: " + errmsg << endl;
exceptionControl("assignPmDbrootConfig", API_FAILURE);
}
}
catch (exception& e)
{
cout << endl << "**** glusterctl API exception: " << e.what() << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID << endl;
exceptionControl("assignPmDbrootConfig", API_FAILURE);
}
catch (...)
{
cout << endl << "**** glusterctl API exception: UNKNOWN" << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID << endl;
exceptionControl("assignPmDbrootConfig", API_FAILURE);
}
}
*/
}
try
@@ -7435,12 +7443,14 @@ void Oam::removeDbroot(DBRootConfigList& dbrootlist)
{
cout << endl << "**** glusterctl API exception: " << e.what() << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl;
writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR );
exceptionControl("removeDbroot", API_FAILURE);
}
catch (...)
{
cout << endl << "**** glusterctl API exception: UNKNOWN" << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl;
writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR );
exceptionControl("removeDbroot", API_FAILURE);
}
}

8
oamapps/alarmmanager/alarmmanager.cpp
View File

@@ -463,7 +463,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
int pid = getpid();
int tid = gettid();
// get reporting Pprocess Name
// get reporting Process Name
string processName;
if ( repProcessName.empty())
@@ -514,7 +514,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
args.add("sendAlarmReport error:");
args.add(e.what());
msg.format(args);
ml.logErrorMessage(msg);
ml.logDebugMessage(msg);
}
catch (std::exception& e)
{
@@ -525,7 +525,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
args.add("sendAlarmReport error:");
args.add(e.what());
msg.format(args);
ml.logErrorMessage(msg);
ml.logDebugMessage(msg);
}
catch (...)
{
@@ -536,7 +536,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
args.add("sendAlarmReport error:");
args.add("general failure");
msg.format(args);
ml.logErrorMessage(msg);
ml.logDebugMessage(msg);
}
return;

13
oamapps/postConfigure/CMakeLists.txt
View File

@@ -37,13 +37,13 @@ install(TARGETS getMySQLpw DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
########### next target ###############
set(amazonInstaller_SRCS amazonInstaller.cpp helpers.cpp)
#set(amazonInstaller_SRCS amazonInstaller.cpp helpers.cpp)
add_executable(amazonInstaller ${amazonInstaller_SRCS})
#add_executable(amazonInstaller ${amazonInstaller_SRCS})
target_link_libraries(amazonInstaller ${ENGINE_LDFLAGS} readline ncurses ${SNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
#target_link_libraries(amazonInstaller ${ENGINE_LDFLAGS} readline ncurses ${SNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
install(TARGETS amazonInstaller DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
#install(TARGETS amazonInstaller DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
########### next target ###############
@@ -56,3 +56,8 @@ target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_
install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
########### next target ###############
install(PROGRAMS quick_installer_single_server.sh quick_installer_multi_server.sh quick_installer_amazon.sh
DESTINATION ${ENGINE_BINDIR} COMPONENT platform)

16
oamapps/postConfigure/installer.cpp
View File

@@ -170,6 +170,12 @@ int main(int argc, char* argv[])
if (p && *p)
USER = p;
// setup to start on reboot, for non-root amazon installs
if ( !rootUser )
{
system("sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1");
}
//copy Columnstore.xml.rpmsave if upgrade option is selected
if ( installType == "upgrade" )
{
@@ -896,7 +902,10 @@ int main(int argc, char* argv[])
cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl;
cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl;
if ( !rootUser )
cout << ". /etc/profile.d/columnstoreEnv.sh" << endl;
cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl;
cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl;
cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl;
@@ -912,7 +921,10 @@ int main(int argc, char* argv[])
cout << endl << "ERROR: MariaDB ColumnStore Process failed to start, check log files in /var/log/mariadb/columnstore" << endl;
cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl;
cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl;
if ( !rootUser )
cout << ". /etc/profile.d/columnstoreEnv.sh" << endl;
cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl;
cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl;
cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl;

1158
oamapps/postConfigure/postConfigure.cpp
View File

File diff suppressed because it is too large Load Diff

81
oamapps/postConfigure/quick_installer_amazon.sh Executable file
View File

@@ -0,0 +1,81 @@
#!/bin/bash
#
# $Id: quick_installer_amazon.sh 3705 2018-07-07 19:47:20Z dhill $
#
# Poddst- Quick Installer for Amazon MariaDB Columnstore
pmCount=""
umCount=""
systemName=""
for arg in "$@"; do
if [ `expr -- "$arg" : '--pm-count='` -eq 11 ]; then
pmCount="`echo $arg | awk -F= '{print $2}'`"
elif [ `expr -- "$arg" : '--um-count='` -eq 11 ]; then
umCount="`echo $arg | awk -F= '{print $2}'`"
elif [ `expr -- "$arg" : '--system-name='` -eq 14 ]; then
systemName="`echo $arg | awk -F= '{print $2}'`"
systemName="-sn "$systemName
elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then
nonDistrubutedInstall=" "
elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then
echo "Usage ./quick_installer_amazon.sh [OPTION]"
echo ""
echo "Quick Installer for an Amazon MariaDB ColumnStore Install"
echo "This requires to be run on a MariaDB ColumnStore AMI"
echo ""
echo "Performace Module (pm) number is required"
echo "User Module (um) number is option"
echo "When only pm counts provided, system is combined setup"
echo "When both pm/um counts provided, system is seperate setup"
echo
echo "--pm-count=x Number of pm instances to create"
echo "--um-count=x Number of um instances to create, optional"
echo "--system-name=nnnn System Name, optional"
echo ""
exit 1
else
echo "./quick_installer_amazon.sh: unknown argument: $arg, enter --help for help" 1>&2
exit 1
fi
done
if [[ $pmCount = "" ]]; then
echo ""
echo "Performace Module (pm) count is required, exiting"
exit 1
else
if [[ $umCount = "" ]]; then
echo ""
echo "NOTE: Performing a Multi-Server Combined install with um/pm running on some server"
echo""
else
echo ""
echo "NOTE: Performing a Multi-Server Seperate install with um and pm running on seperate servers"
echo""
fi
fi
if [[ $HOME = "/root" ]]; then
echo "${bold}Run post-install script${normal}"
echo ""
/usr/local/mariadb/columnstore/bin/post-install
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umCount = "" ]]; then
/usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount $systemName
else
/usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount -um-count $umCount $systemName
fi
else
echo "${bold}Run post-install script${normal}"
echo ""
$HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umCount = "" ]]; then
. /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName
else
. /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName
fi
fi

85
oamapps/postConfigure/quick_installer_multi_server.sh Executable file
View File

@@ -0,0 +1,85 @@
#!/bin/bash
#
# $Id: quick_installer_multi_server.sh 3705 2018-07-07 19:47:20Z dhill $
#
# Poddst- Quick Installer for Multi Server MariaDB Columnstore
pmIpAddrs=""
umIpAddrs=""
nonDistrubutedInstall="-n"
systemName=""
for arg in "$@"; do
if [ `expr -- "$arg" : '--pm-ip-addresses='` -eq 18 ]; then
pmIpAddrs="`echo $arg | awk -F= '{print $2}'`"
elif [ `expr -- "$arg" : '--um-ip-addresses='` -eq 18 ]; then
umIpAddrs="`echo $arg | awk -F= '{print $2}'`"
elif [ `expr -- "$arg" : '--system-name='` -eq 14 ]; then
systemName="`echo $arg | awk -F= '{print $2}'`"
systemName="-sn "$systemName
elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then
nonDistrubutedInstall=" "
elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then
echo "Usage ./quick_installer_multi_server.sh [OPTION]"
echo ""
echo "Quick Installer for a Multi Server MariaDB ColumnStore Install"
echo ""
echo "Defaults to non-distrubuted install, meaning MariaDB Columnstore"
echo "needs to be preinstalled on all nodes in the system"
echo ""
echo "Performace Module (pm) IP addresses are required"
echo "User Module (um) IP addresses are option"
echo "When only pm IP addresses provided, system is combined setup"
echo "When both pm/um IP addresses provided, system is seperate setup"
echo
echo "--pm-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx"
echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx, optional"
echo "--dist-install Use Distributed Install, optional"
echo "--system-name=nnnn System Name, optional"
echo ""
exit 1
else
echo "quick_installer_multi_server.sh: unknown argument: $arg, enter --help for help" 1>&2
exit 1
fi
done
if [[ $pmIpAddrs = "" ]]; then
echo ""
echo "Performace Module (pm) IP addresses required, exiting"
exit 1
else
if [[ $umIpAddrs = "" ]]; then
echo ""
echo "NOTE: Performing a Multi-Server Combined install with um/pm running on some server"
echo""
else
echo ""
echo "NOTE: Performing a Multi-Server Seperate install with um and pm running on seperate servers"
echo""
fi
fi
if [[ $HOME = "/root" ]]; then
echo "${bold}Run post-install script${normal}"
echo ""
/usr/local/mariadb/columnstore/bin/post-install
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umIpAddrs = "" ]]; then
/usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName
else
/usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName
fi
else
echo "${bold}Run post-install script${normal}"
echo ""
$HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umIpAddrs = "" ]]; then
. /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName
else
. /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName
fi
fi

34
oamapps/postConfigure/quick_installer_single_server.sh Executable file
View File

@@ -0,0 +1,34 @@
#!/bin/bash
#
# $Id: quick_installer_single_server.sh 3705 2018-07-07 19:47:20Z dhill $
#
# Poddst- Quick Installer for Single Server MariaDB Columnstore
for arg in "$@"; do
if [ `expr -- "$arg" : '--help'` -eq 6 ]; then
echo "Usage ./quick_installer_multi_server.sh"
echo ""
echo "Quick Installer for a Single Server MariaDB ColumnStore Install"
echo ""
exit 1
else
echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2
fi
done
if [ $HOME == "/root" ]; then
echo "Run post-install script"
echo ""
/usr/local/mariadb/columnstore/bin/post-install
echo "Run postConfigure script"
echo ""
/usr/local/mariadb/columnstore/bin/postConfigure -qs
else
echo "Run post-install script"
echo ""
$HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore
echo "Run postConfigure script"
echo ""
. /etc/profile.d/columnstoreEnv.sh; $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs
fi

3
primitives/linux-port/column.cpp
View File

@@ -65,8 +65,6 @@ inline uint64_t order_swap(uint64_t x)
template <int W>
inline string fixChar(int64_t intval);
idb_regex_t placeholderRegex;
template <class T>
inline int compareBlock( const void* a, const void* b )
{
@@ -1095,6 +1093,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
uint16_t* ridArray = 0;
uint8_t* in8 = reinterpret_cast<uint8_t*>(in);
const uint8_t filterSize = sizeof(uint8_t) + sizeof(uint8_t) + W;
idb_regex_t placeholderRegex;
placeholderRegex.used = false;

20
primitives/primproc/primproc.cpp
View File

@@ -321,6 +321,22 @@ int main(int argc, char* argv[])
// This is unset due to the way we start it
program_invocation_short_name = const_cast<char*>("PrimProc");
int gDebug = 0;
int c;
while ((c = getopt(argc, argv, "d")) != EOF)
{
switch(c)
{
case 'd':
gDebug++;
break;
case '?':
default:
break;
}
}
Config* cf = Config::makeConfig();
setupSignalHandlers();
@@ -329,7 +345,9 @@ int main(int argc, char* argv[])
mlp = new primitiveprocessor::Logger();
int err = setupResources();
int err = 0;
if (!gDebug)
err = setupResources();
string errMsg;
switch (err)

42
procmgr/main.cpp
View File

@@ -1657,9 +1657,6 @@ void pingDeviceThread()
break;
//set query system state not ready
BRM::DBRM dbrm;
dbrm.setSystemQueryReady(false);
processManager.setQuerySystemState(false);
processManager.setSystemState(oam::BUSY_INIT);
@@ -1734,7 +1731,7 @@ void pingDeviceThread()
//set query system state ready
processManager.setQuerySystemState(true);
break;
goto break_case;
}
}
catch (...)
@@ -1756,25 +1753,24 @@ void pingDeviceThread()
if ( retry == 5 )
{
log.writeLog(__LINE__, "autoUnMovePmDbroot: Failed. Fail Module", LOG_TYPE_WARNING);
log.writeLog(__LINE__, "System DBRM READ ONLY - Verify dbroot mounts.", LOG_TYPE_WARNING);
//Issue an alarm
aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, SET);
//set module to disable state
processManager.disableModule(moduleName, true);
//call dbrm control
oam.dbrmctl("reload");
log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG);
// Need to do something here to verify data mounts before resuming
// Best to assume if we reach this you need to put into readonly and verify all dbroots are mounted
// resume the dbrm
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
//call dbrm control
oam.dbrmctl("readonly");
log.writeLog(__LINE__, "'dbrmctl readonly' done", LOG_TYPE_DEBUG);
//clear count
moduleInfoList[moduleName] = 0;
processManager.setSystemState(oam::ACTIVE);
processManager.setSystemState(oam::DEGRADED);
//set query system state ready
processManager.setQuerySystemState(true);
@@ -1972,9 +1968,6 @@ void pingDeviceThread()
}
}
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
@@ -2034,9 +2027,6 @@ void pingDeviceThread()
else
processManager.setSystemState(oam::ACTIVE);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
@@ -2102,9 +2092,6 @@ void pingDeviceThread()
log.writeLog(__LINE__, "module is down: " + moduleName, LOG_TYPE_CRITICAL);
//set query system state not ready
BRM::DBRM dbrm;
dbrm.setSystemQueryReady(false);
processManager.setQuerySystemState(false);
processManager.setSystemState(oam::BUSY_INIT);
@@ -2169,9 +2156,6 @@ void pingDeviceThread()
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
@@ -2393,9 +2377,6 @@ void pingDeviceThread()
//set recycle process
processManager.recycleProcess(moduleName);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
@@ -2412,9 +2393,6 @@ void pingDeviceThread()
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
}
@@ -2429,9 +2407,6 @@ void pingDeviceThread()
//set recycle process
processManager.recycleProcess(moduleName);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
}
@@ -2564,6 +2539,7 @@ void pingDeviceThread()
}
} //end of for loop
}
break_case:
// check and take action if LAN outage is flagged
if (LANOUTAGESUPPORT && !LANOUTAGEACTIVE && LOCALNICDOWN)

128
procmgr/processmanager.cpp
View File

@@ -405,7 +405,7 @@ void processMSG(messageqcpp::IOSocket* cfIos)
msg >> target;
msg >> graceful;
msg >> ackIndicator;
msg >> manualFlag;
msg >> manualFlag;
switch (actionType)
{
@@ -902,29 +902,31 @@ void processMSG(messageqcpp::IOSocket* cfIos)
}
if (opState == oam::MAN_OFFLINE || opState == oam::MAN_DISABLED
|| opState == oam::AUTO_DISABLED )
|| opState == oam::AUTO_DISABLED || opState == oam::AUTO_OFFLINE)
{
oam.dbrmctl("halt");
log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG);
processManager.setSystemState(oam::BUSY_INIT);
//set query system state not ready
processManager.setQuerySystemState(false);
status = processManager.disableModule(moduleName, true);
log.writeLog(__LINE__, "Disable Module Completed on " + moduleName, LOG_TYPE_INFO);
//call dbrm control
oam.dbrmctl("reload");
log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG);
// resume the dbrm
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
processManager.recycleProcess(moduleName);
//check for SIMPLEX Processes on mate might need to be started
processManager.checkSimplexModule(moduleName);
processManager.setSystemState(oam::ACTIVE);
//set query system state ready
processManager.setQuerySystemState(true);
}
else
{
log.writeLog(__LINE__, "ERROR: module not stopped", LOG_TYPE_ERROR);
log.writeLog(__LINE__, "ERROR: module not stopped, state = " + oam.itoa(opState), LOG_TYPE_ERROR);
status = API_FAILURE;
break;
}
@@ -987,7 +989,7 @@ void processMSG(messageqcpp::IOSocket* cfIos)
DeviceNetworkList::iterator listPT = devicenetworklist.begin();
//stopModules being removed with the REMOVE option, which will stop process
// do stopmodule then enable
for ( ; listPT != devicenetworklist.end() ; listPT++)
{
string moduleName = (*listPT).DeviceName;
@@ -1013,6 +1015,9 @@ void processMSG(messageqcpp::IOSocket* cfIos)
if (opState == oam::MAN_DISABLED)
{
processManager.stopModule(moduleName, graceful, manualFlag);
log.writeLog(__LINE__, "stop Module Completed on " + moduleName, LOG_TYPE_INFO);
status = processManager.enableModule(moduleName, oam::MAN_OFFLINE);
log.writeLog(__LINE__, "Enable Module Completed on " + moduleName, LOG_TYPE_INFO);
}
@@ -1357,6 +1362,9 @@ void processMSG(messageqcpp::IOSocket* cfIos)
log.writeLog(__LINE__, "STOPSYSTEM: ACK back to sender");
}
//set query system state ready
processManager.setQuerySystemState(true);
startsystemthreadStop = false;
break;
@@ -3049,9 +3057,6 @@ void processMSG(messageqcpp::IOSocket* cfIos)
log.writeLog(__LINE__, "MSG RECEIVED: Process Restarted on " + moduleName + "/" + processName);
//set query system states not ready
BRM::DBRM dbrm;
dbrm.setSystemQueryReady(false);
processManager.setQuerySystemState(false);
processManager.setSystemState(oam::BUSY_INIT);
@@ -3150,14 +3155,15 @@ void processMSG(messageqcpp::IOSocket* cfIos)
sleep(1);
}
processManager.setQuerySystemState(true);
dbrm.setSystemQueryReady(true);
}
// if a DDLProc was restarted, reinit DMLProc
if ( processName == "DDLProc")
{
processManager.reinitProcessType("DMLProc");
processManager.setQuerySystemState(true);
}
//only run on auto process restart
@@ -3211,9 +3217,7 @@ void processMSG(messageqcpp::IOSocket* cfIos)
}
}
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system states ready
processManager.setQuerySystemState(true);
processManager.setSystemState(oam::ACTIVE);
@@ -3639,6 +3643,8 @@ int ProcessManager::disableModule(string target, bool manualFlag)
if (opState == oam::AUTO_DISABLED && newState == oam::MAN_DISABLED)
{
//removemodule to get proess in MAN_OFFLINE
stopModule(target, REMOVE, true);
try
{
@@ -3691,7 +3697,7 @@ int ProcessManager::disableModule(string target, bool manualFlag)
setModuleState(target, newState);
//set Columnstore.xml enbale state
//set Columnstore.xml enable state
setEnableState( target, SnewState);
log.writeLog(__LINE__, "disableModule - setEnableState", LOG_TYPE_DEBUG);
@@ -3777,18 +3783,18 @@ void ProcessManager::recycleProcess(string module, bool enableModule)
restartProcessType("PrimProc");
sleep(1);
restartProcessType("ExeMgr");
sleep(1);
restartProcessType("mysqld");
restartProcessType("WriteEngineServer");
sleep(1);
restartProcessType("DDLProc", module);
startProcessType("ExeMgr");
sleep(1);
restartProcessType("DMLProc", module);
startProcessType("DDLProc");
sleep(1);
startProcessType("DMLProc");
return;
}
@@ -4128,6 +4134,7 @@ void ProcessManager::setSystemState(uint16_t state)
Oam oam;
ALARMManager aManager;
Configuration config;
ProcessManager processManager(config, log);
log.writeLog(__LINE__, "Set System State = " + oamState[state], LOG_TYPE_DEBUG);
@@ -4149,9 +4156,10 @@ void ProcessManager::setSystemState(uint16_t state)
// Process Alarms
string system = "System";
if ( state == oam::ACTIVE )
{
if( state == oam::ACTIVE ) {
//set query system states ready
processManager.setQuerySystemState(true);
//clear alarms if set
aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_AUTO, CLEAR);
aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_MANUAL, CLEAR);
@@ -4542,7 +4550,8 @@ int ProcessManager::stopProcessType( std::string processName, bool manualFlag )
if ( systemprocessstatus.processstatus[i].ProcessName == processName)
{
//skip if in a COLD_STANDBY state
if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY )
// if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY )
if ( systemprocessstatus.processstatus[i].ProcessOpState != oam::ACTIVE )
continue;
// found one, request restart of it
@@ -4682,12 +4691,17 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski
if ( systemprocessstatus.processstatus[i].ProcessName == processName )
{
//skip if in a BUSY_INIT state
if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::BUSY_INIT ||
systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_INIT ||
systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_INIT ||
( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) )
continue;
// if ( systemprocessstatus.processstatus[i].ProcessOpState == oam::BUSY_INIT ||
// systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_OFFLINE ||
// systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_OFFLINE ||
// systemprocessstatus.processstatus[i].ProcessOpState == oam::AUTO_INIT ||
// systemprocessstatus.processstatus[i].ProcessOpState == oam::MAN_INIT ||
// ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) )
// continue;
if ( systemprocessstatus.processstatus[i].ProcessOpState != oam::ACTIVE )
continue;
if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) )
{
string procModuleType = systemprocessstatus.processstatus[i].Module.substr(0, MAX_MODULE_TYPE_SIZE);
@@ -6825,7 +6839,7 @@ int ProcessManager::sendMsgProcMon( std::string module, ByteStream msg, int requ
if ( IPAddr == oam::UnassignedIpAddr )
{
log.writeLog(__LINE__, "sendMsgProcMon ping failure", LOG_TYPE_ERROR);
log.writeLog(__LINE__, "sendMsgProcMon ping failure " + module + " " + IPAddr, LOG_TYPE_ERROR);
return oam::API_SUCCESS;
}
@@ -6836,7 +6850,7 @@ int ProcessManager::sendMsgProcMon( std::string module, ByteStream msg, int requ
if ( system(cmd.c_str()) != 0)
{
//ping failure
log.writeLog(__LINE__, "sendMsgProcMon ping failure", LOG_TYPE_ERROR);
log.writeLog(__LINE__, "sendMsgProcMon ping failure " + module + " " + IPAddr, LOG_TYPE_ERROR);
return oam::API_SUCCESS;
}
}
@@ -7649,7 +7663,7 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist)
}
//set query system state not ready
processManager.setQuerySystemState(true);
processManager.setQuerySystemState(false);
// Bug 4554: Wait until DMLProc is finished with rollback
if (status == oam::API_SUCCESS)
@@ -7726,6 +7740,9 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist)
processManager.setSystemState(rtn);
}
//set query system state ready
processManager.setQuerySystemState(true);
// exit thread
log.writeLog(__LINE__, "startSystemThread Exit", LOG_TYPE_DEBUG);
startsystemthreadStatus = status;
@@ -8254,19 +8271,18 @@ void ProcessManager::checkSimplexModule(std::string moduleName)
if ( state == oam::COLD_STANDBY )
{
//set Primary UM Module
if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc" )
{
//process DDL/DMLProc
if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc")
{
setPMProcIPs((*pt).DeviceName);
log.writeLog(__LINE__, "Set Primary UM Module = " + (*pt).DeviceName, LOG_TYPE_DEBUG);
oam.setSystemConfig("PrimaryUMModuleName", (*pt).DeviceName);
//distribute config file
distributeConfigFile("system");
sleep(2);
//add MySQL Replication setup, if needed
log.writeLog(__LINE__, "Setup MySQL Replication for COLD_STANDBY DMLProc going ACTIVE", LOG_TYPE_DEBUG);
oam::DeviceNetworkList devicenetworklist;
processManager.setMySQLReplication(devicenetworklist, (*pt).DeviceName);
}
int status = processManager.startProcess((*pt).DeviceName,
@@ -8277,12 +8293,24 @@ void ProcessManager::checkSimplexModule(std::string moduleName)
{
log.writeLog(__LINE__, "checkSimplexModule: mate process started: " + (*pt).DeviceName + "/" + systemprocessconfig.processconfig[j].ProcessName, LOG_TYPE_DEBUG);
//check to see if DDL/DML IPs need to be updated
if ( systemprocessconfig.processconfig[j].ProcessName == "DDLProc" )
setPMProcIPs((*pt).DeviceName);
status = processManager.startProcess((*pt).DeviceName,
"DMLProc",
FORCEFUL);
if ( status == API_SUCCESS ) {
log.writeLog(__LINE__, "checkSimplexModule: mate process started: " + (*pt).DeviceName + "/DMLProc", LOG_TYPE_DEBUG);
}
else
log.writeLog(__LINE__, "checkSimplexModule: mate process failed to start: " + (*pt).DeviceName + "/DMLProc", LOG_TYPE_DEBUG);
}
else
log.writeLog(__LINE__, "checkSimplexModule: mate process failed to start: " + (*pt).DeviceName + "/" + systemprocessconfig.processconfig[j].ProcessName, LOG_TYPE_DEBUG);
//setup new MariaDB Replication Master
if ( systemprocessconfig.processconfig[j].ProcessName == "DMLProc" ) {
log.writeLog(__LINE__, "Setup MySQL Replication for COLD_STANDBY DMLProc going ACTIVE", LOG_TYPE_DEBUG);
oam::DeviceNetworkList devicenetworklist;
processManager.setMySQLReplication(devicenetworklist, (*pt).DeviceName);
}
}
else
{
@@ -10344,7 +10372,7 @@ int ProcessManager::OAMParentModuleChange()
if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) &&
( moduleNameList.size() <= 0 && config.moduleType() == "pm") )
{
int status = 0;
status = 0;
}
else
{

23
procmon/main.cpp
View File

@@ -340,8 +340,9 @@ int main(int argc, char** argv)
{
log.writeLog(__LINE__, "Standby PM not responding, infinidb shutting down", LOG_TYPE_CRITICAL);
//Set the alarm
aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET);
sleep (1);
// aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET);
// sleep (1);
string cmd = startup::StartUp::installDir() + "/bin/infinidb stop > /dev/null 2>&1";
system(cmd.c_str());
}
@@ -369,8 +370,7 @@ int main(int argc, char** argv)
sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", IPaddr);
log.writeLog(__LINE__, "set ProcMgr IPaddr to Old Standby Module: " + IPaddr, LOG_TYPE_DEBUG);
//update Calpont Config table
//update MariaDB ColumnStore Config table
try
{
sysConfig->write();
@@ -554,8 +554,8 @@ int main(int argc, char** argv)
{
log.writeLog(__LINE__, "Check DB mounts failed, shutting down", LOG_TYPE_CRITICAL);
//Set the alarm
aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET);
sleep (1);
// aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET);
// sleep (1);
string cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /dev/null 2>&1";
system(cmd.c_str());
}
@@ -1463,7 +1463,7 @@ static void chldHandleThread(MonitorConfig config)
(*listPtr).processID != 0 ) ||
( (*listPtr).state == oam::ACTIVE && (*listPtr).processID == 0 ) )
{
log.writeLog(__LINE__, "*****Calpont Process Restarting: " + (*listPtr).ProcessName + ", old PID = " + oam.itoa((*listPtr).processID), LOG_TYPE_CRITICAL);
log.writeLog(__LINE__, "*****MariaDB ColumnStore Process Restarting: " + (*listPtr).ProcessName + ", old PID = " + oam.itoa((*listPtr).processID), LOG_TYPE_CRITICAL);
if ( (*listPtr).dieCounter >= processRestartCount ||
processRestartCount == 0)
@@ -1523,7 +1523,7 @@ static void chldHandleThread(MonitorConfig config)
{}
// check if process failover is needed due to process outage
aMonitor.checkProcessFailover((*listPtr).ProcessName);
aMonitor.checkModuleFailover((*listPtr).ProcessName);
//check the db health
if (DBFunctionalMonitorFlag == "y" )
@@ -1605,7 +1605,7 @@ static void chldHandleThread(MonitorConfig config)
(*listPtr).processID = 0;
// check if process failover is needed due to process outage
aMonitor.checkProcessFailover((*listPtr).ProcessName);
aMonitor.checkModuleFailover((*listPtr).ProcessName);
break;
}
else
@@ -1681,7 +1681,7 @@ static void chldHandleThread(MonitorConfig config)
}
//Log this event
log.writeLog(__LINE__, "Calpont Process " + (*listPtr).ProcessName + restartStatus, LOG_TYPE_INFO);
log.writeLog(__LINE__, "MariaDB ColumnStore Process " + (*listPtr).ProcessName + restartStatus, LOG_TYPE_INFO);
}
}
}
@@ -2707,6 +2707,9 @@ void processStatusMSG(messageqcpp::IOSocket* cfIos)
memcpy(fShmSystemStatus[0].StateChangeDate, oam.getCurrentTime().c_str(), DATESIZE);
log.writeLog(__LINE__, "statusControl: REQUEST RECEIVED: Set System State = " + oamState[state], LOG_TYPE_DEBUG);
}
BRM::DBRM dbrm;
dbrm.setSystemQueryReady(true);
}
}
break;

61
procmon/processmonitor.cpp
View File

@@ -1261,7 +1261,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
// error in launching a process
if ( requestStatus == oam::API_FAILURE &&
(*listPtr).RunType == SIMPLEX)
checkProcessFailover((*listPtr).ProcessName);
checkModuleFailover((*listPtr).ProcessName);
else
break;
}
@@ -4963,20 +4963,19 @@ std::string ProcessMonitor::sendMsgProcMon1( std::string module, ByteStream msg,
}
/******************************************************************************************
* @brief checkProcessFailover
* @brief checkModuleFailover
*
* purpose: check if process failover is needed due to a process outage
* purpose: check if module failover is needed due to a process outage
*
******************************************************************************************/
void ProcessMonitor::checkProcessFailover( std::string processName)
void ProcessMonitor::checkModuleFailover( std::string processName)
{
Oam oam;
//force failover on certain processes
if ( processName == "DDLProc" ||
processName == "DMLProc" )
{
log.writeLog(__LINE__, "checkProcessFailover: process failover, process outage of " + processName, LOG_TYPE_CRITICAL);
processName == "DMLProc" ) {
log.writeLog(__LINE__, "checkModuleFailover: process failover, process outage of " + processName, LOG_TYPE_CRITICAL);
try
{
@@ -4999,27 +4998,36 @@ void ProcessMonitor::checkProcessFailover( std::string processName)
systemprocessstatus.processstatus[i].ProcessOpState == oam::FAILED )
{
// found a AVAILABLE mate, start it
log.writeLog(__LINE__, "start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG);
log.writeLog(__LINE__, "Change UM Master to module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG);
log.writeLog(__LINE__, "Disable local UM module " + config.moduleName(), LOG_TYPE_DEBUG);
log.writeLog(__LINE__, "Stop local UM module " + config.moduleName(), LOG_TYPE_DEBUG);
log.writeLog(__LINE__, "Disable Local will Enable UM module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG);
oam::DeviceNetworkConfig devicenetworkconfig;
oam::DeviceNetworkList devicenetworklist;
devicenetworkconfig.DeviceName = config.moduleName();
devicenetworklist.push_back(devicenetworkconfig);
try
{
oam.setSystemConfig("PrimaryUMModuleName", systemprocessstatus.processstatus[i].Module);
oam.stopModule(devicenetworklist, oam::FORCEFUL, oam::ACK_YES);
log.writeLog(__LINE__, "success stopModule on module " + config.moduleName(), LOG_TYPE_DEBUG);
//distribute config file
oam.distributeConfigFile("system");
sleep(1);
}
catch (...) {}
try
{
oam.startProcess(systemprocessstatus.processstatus[i].Module, processName, FORCEFUL, ACK_YES);
log.writeLog(__LINE__, "success start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG);
}
catch (exception& e)
{
log.writeLog(__LINE__, "failed start process on module " + systemprocessstatus.processstatus[i].Module, LOG_TYPE_ERROR);
try
{
oam.disableModule(devicenetworklist);
log.writeLog(__LINE__, "success disableModule on module " + config.moduleName(), LOG_TYPE_DEBUG);
}
catch (exception& e)
{
log.writeLog(__LINE__, "failed disableModule on module " + config.moduleName(), LOG_TYPE_ERROR);
}
}
catch (exception& e)
{
log.writeLog(__LINE__, "failed stopModule on module " + config.moduleName(), LOG_TYPE_ERROR);
}
break;
}
@@ -5036,9 +5044,6 @@ void ProcessMonitor::checkProcessFailover( std::string processName)
// log.writeLog(__LINE__, "EXCEPTION ERROR on getProcessStatus: Caught unknown exception!", LOG_TYPE_ERROR);
}
}
return;
}
/******************************************************************************************
@@ -6583,6 +6588,8 @@ int ProcessMonitor::glusterAssign(std::string dbrootID)
if ( WEXITSTATUS(ret) != 0 )
{
log.writeLog(__LINE__, "glusterAssign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR);
ifstream in("/tmp/glusterAssign.txt");
in.seekg(0, std::ios::end);
int size = in.tellg();
@@ -6630,6 +6637,8 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID)
if ( WEXITSTATUS(ret) != 0 )
{
log.writeLog(__LINE__, "glusterUnassign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR);
ifstream in("/tmp/glusterUnassign.txt");
in.seekg(0, std::ios::end);
int size = in.tellg();

2
procmon/processmonitor.h
View File

@@ -488,7 +488,7 @@ public:
/**
*@brief check if module failover is needed due to a process outage
*/
void checkProcessFailover( std::string processName);
void checkModuleFailover(std::string processName);
/**
*@brief run upgrade script

156
utils/clusterTester/columnstoreClusterTester.sh
View File

@@ -10,7 +10,7 @@ CHECK=true
REPORTPASS=true
LOGFILE=""
OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16")
OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16" "ubuntu18")
NODE_IPADDRESS=""
@@ -37,7 +37,7 @@ checkContinue() {
}
###
# Print Fucntions
# Print Functions
###
helpPrint () {
@@ -57,7 +57,7 @@ helpPrint () {
echo ""
echo "Additional information on Tool is documented at:"
echo ""
echo "https://mariadb.com/kb/en/mariadb/*****/"
echo "https://mariadb.com/kb/en/library/mariadb-columnstore-cluster-test-tool/"
echo ""
echo "Items that are checked:"
echo " Node Ping test"
@@ -65,6 +65,7 @@ helpPrint () {
echo " ColumnStore Port test"
echo " OS version"
echo " Locale settings"
echo " Umask settings"
echo " Firewall settings"
echo " Date/time settings"
echo " Dependent packages installed"
@@ -326,16 +327,18 @@ checkSSH()
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
if [ $PASSWORD == "ssh" ] ; then
echo $ipadd " Node Passed SSH login test using ssh-keys"
echo $ipadd " Node Passed SSH login test using ssh-keys"
else
echo $ipadd " Node Passed SSH login test using user password"
echo $ipadd " Node Passed SSH login test using user password"
fi
else
if [ $PASSWORD == "ssh" ] ; then
echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys"
echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys"
else
echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password"
echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password"
fi
echo "Error - Fix the SSH login issue and rerun test"
exit 1
fi
done
@@ -489,12 +492,47 @@ checkLocale()
fi
}
checkSELINUX()
checkLocalUMASK()
{
# UMASK check
#
echo ""
echo "** Run Local UMASK check"
echo ""
pass=true
filename=UMASKtest
rm -f $filename
touch $filename
permission=$(stat -c "%A" "$filename")
result=${permission:4:1}
if [ ${result} == "r" ] ; then
result=${permission:7:1}
if [ ${result} == "r" ] ; then
echo "UMASK local setting test passed"
else
echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022"
pass=false
fi
if ! $pass; then
checkContinue
fi
rm -f $filename
}
checkLocalSELINUX()
{
# SELINUX check
#
echo ""
echo "** Run SELINUX check"
echo "** Run Local SELINUX check"
echo ""
pass=true
@@ -511,21 +549,86 @@ checkSELINUX()
echo "Local Node SELINUX setting is Not Enabled"
fi
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "$ipadd Node SELINUX setting is Not Enabled"
else
`cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results"
pass=false
else
echo "$ipadd Node SELINUX setting is Not Enabled"
fi
`rm -f config`
fi
done
if ! $pass; then
checkContinue
fi
}
checkUMASK()
{
# UMASK check
#
echo ""
echo "** Run UMASK check"
echo ""
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > /tmp/remote_command_check 2>&1`
if [ "$?" -eq 0 ]; then
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> /tmp/remote_scp_get 2>&1`
if [ "$?" -eq 0 ]; then
permission=`cat test.log`
result=${permission:4:1}
if [ ${result} == "r" ] ; then
result=${permission:7:1}
if [ ${result} == "r" ] ; then
echo "$ipadd Node UMASK setting test passed"
else
echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check /tmp/remote_scp_get"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check /tmp/remote_command_check"
pass=false
fi
`rm -f test.log`
done
if ! $pass; then
checkContinue
fi
rm -f $filename
}
checkSELINUX()
{
# SELINUX check
#
echo ""
echo "** Run SELINUX check"
echo ""
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "$ipadd Node SELINUX setting is Not Enabled"
else
`cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results"
pass=false
else
echo "$ipadd Node SELINUX setting is Not Enabled"
fi
`rm -f config`
fi
done
if ! $pass; then
checkContinue
fi
}
checkFirewalls()
@@ -949,7 +1052,7 @@ checkPackages()
declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" )
declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18")
if [ "$OS" == "ubuntu16" ] ; then
if [ "$OS" == "ubuntu16" ] || [ "$OS" == "ubuntu18" ]; then
if [ ! `which dpkg 2>/dev/null` ] ; then
echo "${bold}Failed${normal}, Local Node ${bold}rpm${normal} package not installed"
pass=false
@@ -1307,12 +1410,15 @@ echo ""
checkLocalOS
checkLocalDir
checkLocalUMASK
checkLocalSELINUX
if [ "$IPADDRESSES" != "" ]; then
checkPing
checkSSH
checkRemoteDir
checkOS
checkLocale
checkUMASK
checkSELINUX
checkFirewalls
checkPorts

2
utils/clusterTester/os_detect.sh
View File

@@ -29,7 +29,7 @@ detectOS () {
echo Operating System name: $osPrettyName
echo Operating System tag: $osTag
case "$osTag" in
centos6|centos7|ubuntu16|debian8|suse12|debian9)
centos6|centos7|ubuntu16|debian8|suse12|debian9|ubuntu18)
;;
*)
echo OS not supported

70
utils/funcexp/func_case.cpp
View File

@@ -315,20 +315,13 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType,
bool simpleCase)
{
FunctionParm::size_type n = fp.size();
uint64_t simple = simpleCase ? 1 : 0;
bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist
if (simpleCase) // simple case has an expression
n -= 1; // remove expression from count of expression_i + result_i
bool hasElse = ((n % 2) != 0); // if 1, then ELSE exist
if (hasElse)
--n; // n now is an even number
uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1);
uint64_t whereCount = hasElse ? (fp.size() - 2 + simpleCase) / 2 : (fp.size() - 1) / 2 + simpleCase;
idbassert((n % 2) == 0);
uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple;
bool allStringO = true;
bool allStringR = true;
@@ -341,33 +334,24 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp,
for (uint64_t i = 0; i <= parmCount; i++)
{
// operation or result type
operation = ((i > 0) && (i <= whereCount));
// the result type of ELSE, if exists.
if (i == n)
{
if (!hasElse)
break;
if (simpleCase)
// for SimpleCase, we return the type of the case expression,
// which will always be in position 0.
if (i == 0 && simpleCase)
{
if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR)
{
// the case expression
if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR)
{
PredicateOperator op;
op.setOpType(oct, fp[i]->data()->resultType());
allStringO = false;
oct = op.operationType();
}
i += 1;
PredicateOperator op;
op.setOpType(oct, fp[i]->data()->resultType());
allStringO = false;
oct = op.operationType();
}
i += 1;
}
operation = false;
}
// operation or result type
operation = ((i > 0+simple) && (i <= whereCount));
if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT &&
@@ -378,10 +362,13 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp,
if (operation)
{
op.setOpType(oct, fp[i]->data()->resultType());
allStringO = false;
oct = op.operationType();
}
if (!simpleCase)
{
op.setOpType(oct, fp[i]->data()->resultType());
allStringO = false;
oct = op.operationType();
}
}
// If any parm is of string type, the result type should be string. (same as if)
else if (rct.colDataType != CalpontSystemCatalog::CHAR &&
@@ -457,6 +444,13 @@ bool Func_simple_case::getBoolVal(Row& row,
if (isNull)
return joblist::BIGINTNULL;
ParseTree* lop = parm[i]->left();
ParseTree* rop = parm[i]->right();
if (lop && rop)
{
return (reinterpret_cast<Operator*>(parm[i]->data()))->getBoolVal(row, isNull, lop, rop);
}
return parm[i]->data()->getBoolVal(row, isNull);
}

6
utils/joiner/tuplejoiner.cpp
View File

@@ -149,8 +149,7 @@ TupleJoiner::TupleJoiner(
for (uint32_t i = 0; i < smallKeyColumns.size(); i++)
{
discreteValues[i] = false;
if (isUnsigned(smallRG.getColType(i)))
if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]]))
{
cpValues[i].push_back(static_cast<int64_t>(numeric_limits<uint64_t>::max()));
cpValues[i].push_back(0);
@@ -1033,8 +1032,7 @@ boost::shared_ptr<TupleJoiner> TupleJoiner::copyForDiskJoin()
for (uint32_t i = 0; i < smallKeyColumns.size(); i++)
{
ret->discreteValues[i] = false;
if (isUnsigned(smallRG.getColType(i)))
if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]]))
{
ret->cpValues[i].push_back(static_cast<int64_t>(numeric_limits<uint64_t>::max()));
ret->cpValues[i].push_back(0);

65
utils/messageqcpp/messagequeue.cpp
View File

@@ -157,24 +157,42 @@ void MessageQueueClient::setup(bool syncProto)
{
string otherEndIPStr;
string otherEndPortStr;
uint16_t port;
struct addrinfo hints, *servinfo;
int rc = 0;
otherEndIPStr = fConfig->getConfig(fOtherEnd, "IPAddr");
otherEndPortStr = fConfig->getConfig(fOtherEnd, "Port");
if (otherEndIPStr.length() == 0) otherEndIPStr = "127.0.0.1";
if (otherEndPortStr.length() == 0 || (port = static_cast<uint16_t>(strtol(otherEndPortStr.c_str(), 0, 0))) == 0)
if (otherEndPortStr.length() == 0 || static_cast<uint16_t>(strtol(otherEndPortStr.c_str(), 0, 0)) == 0)
{
string msg = "MessageQueueClient::MessageQueueClient: config error: Invalid/Missing Port attribute";
string msg = "MessageQueueClient::setup(): config error: Invalid/Missing Port attribute";
throw runtime_error(msg);
}
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
sinp->sin_family = AF_INET;
sinp->sin_port = htons(port);
sinp->sin_addr.s_addr = inet_addr(otherEndIPStr.c_str());
memset(&hints, 0, sizeof hints);
// ATM We support IPv4 only.
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if( !(rc = getaddrinfo(otherEndIPStr.c_str(), otherEndPortStr.c_str(), &hints, &servinfo)) )
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
*sinp = *reinterpret_cast<sockaddr_in*>(servinfo->ai_addr);
freeaddrinfo(servinfo);
}
else
{
string msg = "MessageQueueClient::setup(): ";
msg.append(gai_strerror(rc));
logging::Message::Args args;
logging::LoggingID li(31);
args.add(msg);
fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li);
}
#ifdef SKIP_IDB_COMPRESSION
fClientSock.setSocketImpl(new InetStreamSocket());
@@ -200,15 +218,34 @@ MessageQueueClient::MessageQueueClient(const string& otherEnd, Config* config, b
setup(syncProto);
}
MessageQueueClient::MessageQueueClient(const string& ip, uint16_t port, bool syncProto) :
MessageQueueClient::MessageQueueClient(const string& dnOrIp, uint16_t port, bool syncProto) :
fLogger(31), fIsAvailable(true)
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
sinp->sin_family = AF_INET;
sinp->sin_port = htons(port);
sinp->sin_addr.s_addr = inet_addr(ip.c_str());
struct addrinfo hints, *servinfo;
int rc = 0;
memset(&hints, 0, sizeof hints);
// ATM We support IPv4 only.
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if( !(rc = getaddrinfo(dnOrIp.c_str(), NULL, &hints, &servinfo)) )
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
*sinp = *reinterpret_cast<sockaddr_in*>(servinfo->ai_addr);
sinp->sin_port = htons(port);
freeaddrinfo(servinfo);
}
else
{
string msg = "MessageQueueClient::MessageQueueClient(): ";
msg.append(gai_strerror(rc));
logging::Message::Args args;
logging::LoggingID li(31);
args.add(msg);
fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li);
}
#ifdef SKIP_IDB_COMPRESSION
fClientSock.setSocketImpl(new InetStreamSocket());
#else

3
utils/messageqcpp/messagequeue.h
View File

@@ -33,6 +33,7 @@
#include <stdio.h>
#else
#include <netinet/in.h>
#include <netdb.h>
#endif
#include "serversocket.h"
@@ -183,7 +184,7 @@ public:
*
* construct a queue from this process to otherEnd on the given IP and Port.
*/
EXPORT explicit MessageQueueClient(const std::string& ip, uint16_t port, bool syncProto = true);
EXPORT explicit MessageQueueClient(const std::string& dnOrIp, uint16_t port, bool syncProto=true);
/**

6
utils/messageqcpp/messagequeuepool.cpp
View File

@@ -37,12 +37,12 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts)
return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000;
}
MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& ip, uint64_t port)
MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &dnOrIp, uint64_t port)
{
boost::mutex::scoped_lock lock(queueMutex);
std::ostringstream oss;
oss << ip << "_" << port;
oss << dnOrIp << "_" << port;
std::string searchString = oss.str();
MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(searchString);
@@ -59,7 +59,7 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& ip, u
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client = new MessageQueueClient(ip, port);
newClientObject->client = new MessageQueueClient(dnOrIp, port);
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
clientMap.insert(std::pair<std::string, ClientObject*>(searchString, newClientObject));

2
utils/messageqcpp/messagequeuepool.h
View File

@@ -42,7 +42,7 @@ class MessageQueueClientPool
{
public:
static MessageQueueClient* getInstance(const std::string& module);
static MessageQueueClient* getInstance(const std::string& ip, uint64_t port);
static MessageQueueClient *getInstance(const std::string &dnOrIp, uint64_t port);
static void releaseInstance(MessageQueueClient* client);
static void deleteInstance(MessageQueueClient* client);
static MessageQueueClient* findInPool(const std::string& search);

19
writeengine/server/we_server.cpp
View File

@@ -110,6 +110,21 @@ int main(int argc, char** argv)
printf ("Locale is : %s\n", systemLang.c_str() );
int gDebug = 0;
int c;
while ((c = getopt(argc, argv, "d")) != EOF)
{
switch (c)
{
case 'd':
gDebug++;
break;
case '?':
default:
break;
}
}
//set BUSY_INIT state
{
// Is there a reason to have a seperate Oam instance for this?
@@ -210,7 +225,9 @@ int main(int argc, char** argv)
}
}
int err = setupResources();
int err = 0;
if (!gDebug)
err = setupResources();
string errMsg;
switch (err)