database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Merge branch 'develop-1.1' into MCOL-1523

Browse Source
This commit is contained in:
david hill
2018-09-10 14:15:32 -05:00
committed by GitHub
parent 8b0507b987 a3cc11e003
commit 43b97e7fa3
30 changed files with 542 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dbcon/ddlpackage/CMakeLists.txt
View File

@ -1,4 +1,3 @@
include_directories( ${ENGINE_COMMON_INCLUDES} )
ADD_CUSTOM_COMMAND(
@ -9,9 +8,10 @@ ADD_CUSTOM_COMMAND(
DEPENDS ddl.y ddl.l
)
# Parser puts extra info to stderr.
INCLUDE(../../check_compiler_flag.cmake)
MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG" DEBUG)
MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG=1" DEBUG)
########### next target ###############

9
dbcon/ddlpackage/ddl.l
View File

@ -33,6 +33,9 @@
using namespace ddlpackage;
typedef enum { NOOP, STRIP_QUOTES } copy_action_t;
#if YYDEBUG == 0
int ddldebug = 0;
#endif
int lineno = 1;
void ddlerror(struct pass_to_bison* x, char const *s);
@ -69,9 +72,9 @@ ident_start [A-Za-z\200-\377_]
ident_cont [A-Za-z\200-\377_0-9\$]
identifier {ident_start}{ident_cont}*
/* fully qualified names regexes */
fq_identifier {identifier}\.{identifier}
identifier_quoted {grave_accent}{identifier}{grave_accent}
identifier_double_quoted {double_quote}{identifier}{double_quote}
ident_w_spaces {identifier}\x20*
identifier_quoted {grave_accent}{ident_w_spaces}+{grave_accent}
identifier_double_quoted {double_quote}{ident_w_spaces}+{double_quote}
integer [-+]?{digit}+
decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*))

3
dbcon/ddlpackage/ddl.y
View File

@ -63,7 +63,6 @@ char* copy_string(const char *str);
%pure-parser
%lex-param {void * scanner}
%parse-param {struct ddlpackage::pass_to_bison * x}
%debug
/* Bison uses this to generate a C union definition. This is used to
store the application created values associated with syntactic
@ -605,7 +604,7 @@ table_name:
;
qualified_name:
| ident {
ident {
if (x->fDBSchema.size())
$$ = new QualifiedName((char*)x->fDBSchema.c_str(), $1);
else

7
dbcon/mysql/ha_calpont_dml.cpp
View File

@ -1615,8 +1615,11 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_
}
else if (ci.columnTypes[colpos].colWidth < 16777216)
{
dataLength = *(uint32_t*) buf;
buf = buf + 3 ;
dataLength = *(uint16_t*) buf;
buf = buf + 2 ;
if (*(uint8_t*)buf)
dataLength += 256*256*(*(uint8_t*)buf) ;
buf++;
}
else
{

7
dbcon/mysql/ha_calpont_execplan.cpp
View File

@ -1268,7 +1268,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip)
ifp->functype() == Item_func::ISNOTNULL_FUNC)
{
ReturnedColumn* rhs = NULL;
if (!gwip->rcWorkStack.empty())
if (!gwip->rcWorkStack.empty() && !gwip->inCaseStmt)
{
rhs = gwip->rcWorkStack.top();
gwip->rcWorkStack.pop();
@ -3267,7 +3267,12 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS
if (funcName == "case_searched" &&
(i < arg_offset))
{
// MCOL-1472 Nested CASE with an ISNULL predicate. We don't want the predicate
// to pull off of rcWorkStack, so we set this inCaseStmt flag to tell it
// not to.
gwi.inCaseStmt = true;
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
gwi.inCaseStmt = false;
if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data())
{
gwi.ptWorkStack.pop();

6
dbcon/mysql/ha_calpont_impl_if.h
View File

@ -147,6 +147,9 @@ struct gp_walk_info
int32_t recursionHWM;
std::stack<int32_t> rcBookMarkStack;
// Kludge for MCOL-1472
bool inCaseStmt;
gp_walk_info() : sessionid(0),
fatalParseError(false),
condPush(false),
@ -162,7 +165,8 @@ struct gp_walk_info
lastSub(0),
derivedTbCnt(0),
recursionLevel(-1),
recursionHWM(0)
recursionHWM(0),
inCaseStmt(false)
{}
~gp_walk_info() {}

33
ddlproc/ddlproc.cpp
View File

@ -135,8 +135,30 @@ int main(int argc, char* argv[])
{
oam.processInitComplete("DDLProc", ACTIVE);
}
catch (std::exception& ex)
{
cerr << ex.what() << endl;
LoggingID logid(23, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("DDLProc init caught exception: ");
args1.add(ex.what());
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
return 1;
}
catch (...)
{
cerr << "Caught unknown exception in init!" << endl;
LoggingID logid(23, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("DDLProc init caught unknown exception");
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
return 1;
}
}
@ -147,21 +169,28 @@ int main(int argc, char* argv[])
catch (std::exception& ex)
{
cerr << ex.what() << endl;
LoggingID logid(23, 0, 0);
Message::Args args;
Message message(8);
args.add("DDLProc failed on: ");
args.add(ex.what());
message.format( args );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, message, logid);
return 1;
}
catch (...)
{
cerr << "Caught unknown exception!" << endl;
LoggingID logid(23, 0, 0);
Message::Args args;
Message message(8);
args.add("DDLProc failed on: ");
args.add("receiving DDLPackage");
args.add("receiving DDLPackage (unknown exception)");
message.format( args );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, message, logid);
return 1;
}
return 0;
}

44
dmlproc/dmlproc.cpp
View File

@ -494,8 +494,30 @@ int main(int argc, char* argv[])
// At first we set to BUSY_INIT
oam.processInitComplete("DMLProc", oam::BUSY_INIT);
}
catch (std::exception& ex)
{
cerr << ex.what() << endl;
LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("DMLProc init caught exception: ");
args1.add(ex.what());
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
return 1;
}
catch (...)
{
cerr << "Caught unknown exception in init!" << endl;
LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("DMLProc init caught unknown exception");
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
return 1;
}
//@Bug 1627
@ -584,8 +606,30 @@ int main(int argc, char* argv[])
{
oam.processInitComplete("DMLProc", ACTIVE);
}
catch (std::exception& ex)
{
cerr << ex.what() << endl;
LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("DMLProc init caught exception: ");
args1.add(ex.what());
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
return 1;
}
catch (...)
{
cerr << "Caught unknown exception in init!" << endl;
LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("DMLProc init caught unknown exception");
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
return 1;
}
Dec = DistributedEngineComm::instance(rm);

20
dmlproc/dmlprocessor.cpp
View File

@ -1155,8 +1155,28 @@ void DMLServer::start()
}
cancelThread.join();
}
catch (std::exception& ex)
{
cerr << ex.what() << endl;
logging::LoggingID lid(21);
Message::Args args;
Message message(8);
args.add("DMLProc init caught exception: ");
args.add(ex.what());
message.format(args);
logging::Logger logger(lid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_CRITICAL, message, lid);
}
catch (...)
{
cerr << "Caught unknown exception!" << endl;
logging::LoggingID lid(21);
Message::Args args;
Message message(8);
args.add("DMLProc init caught unknown exception");
message.format(args);
logging::Logger logger(lid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_CRITICAL, message, lid);
}
}

30
exemgr/main.cpp
View File

@ -1300,8 +1300,34 @@ void cleanTempDir()
assert(tmpPrefix != "/");
/* This is quite scary as ExeMgr usually runs as root */
boost::filesystem::remove_all(tmpPrefix);
boost::filesystem::create_directories(tmpPrefix);
try
{
boost::filesystem::remove_all(tmpPrefix);
boost::filesystem::create_directories(tmpPrefix);
}
catch (std::exception& ex)
{
cerr << ex.what() << endl;
LoggingID logid(16, 0, 0);
Message::Args args;
Message message(8);
args.add("Execption whilst cleaning tmpdir: ");
args.add(ex.what());
message.format( args );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_WARNING, message, logid);
}
catch (...)
{
cerr << "Caught unknown exception during tmpdir cleanup" << endl;
LoggingID logid(16, 0, 0);
Message::Args args;
Message message(8);
args.add("Unknown execption whilst cleaning tmpdir");
message.format( args );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_WARNING, message, logid);
}
}

4
oam/etc/ProcessConfig.xml
View File

@ -107,7 +107,7 @@
<DepProcessName1>WriteEngineServer</DepProcessName1>
<DepModuleName1>pm*</DepModuleName1>
<DepProcessName2>DBRMWorkerNode</DepProcessName2>
<DepModuleName2>*</DepModuleName2>
<DepModuleName2>@</DepModuleName2>
<DepProcessName3>ExeMgr</DepProcessName3>
<DepModuleName3>*</DepModuleName3>
<RunType>SIMPLEX</RunType>
@ -122,7 +122,7 @@
<DepProcessName1>WriteEngineServer</DepProcessName1>
<DepModuleName1>pm*</DepModuleName1>
<DepProcessName2>DBRMWorkerNode</DepProcessName2>
<DepModuleName2>*</DepModuleName2>
<DepModuleName2>@</DepModuleName2>
<DepProcessName3>DDLProc</DepProcessName3>
<DepModuleName3>@</DepModuleName3>
<RunType>SIMPLEX</RunType>

1
oam/install_scripts/post-install
View File

@ -247,7 +247,6 @@ else
$SUDO chmod 777 /tmp
$installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
$SUDO chown $user:$user $installdir/etc/Columnstore.xml
$SUDO chmod -R 777 /dev/shm
$SUDO mkdir /var/lock/subsys > /dev/null 2>&1
$SUDO chmod 777 /var/lock/subsys > /dev/null 2>&1
$SUDO rm -f /var/lock/subsys/mysql-Columnstore

101
oam/oamcpp/liboamcpp.cpp
View File

@ -5295,6 +5295,7 @@ namespace oam
dbrootList dbroot1;
dbroot1.push_back(*pt1);
bool returnDbRoot = false;
//send msg to unmount dbroot if module is not offline
int opState;
@ -5306,7 +5307,6 @@ namespace oam
{}
if (opState != oam::AUTO_OFFLINE || opState != oam::AUTO_DISABLED) {
// bool unmountPass = true;
try
{
mountDBRoot(dbroot1, false);
@ -5316,13 +5316,8 @@ namespace oam
writeLog("ERROR: dbroot failed to unmount", LOG_TYPE_ERROR );
cout << endl << "ERROR: umountDBRoot api failure" << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
// unmountPass = false;
}
// if ( !unmountPass) {
// dbrootlist.erase(pt1);
// break;
// }
}
//check for amazon moving required
@ -5340,38 +5335,79 @@ namespace oam
//if Gluster, do the assign command
if ( DataRedundancyConfig == "y")
{
try {
try
{
string errmsg;
int ret = glusterctl(oam::GLUSTER_ASSIGN, *pt1, toPM, errmsg);
if ( ret != 0 )
if ( ret == 0 )
{
todbrootConfigList.push_back(*pt2);
residedbrootConfigList.erase(pt2);
}
else
{
cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID + ", error: " + errmsg << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID + ", error: " + errmsg, LOG_TYPE_ERROR );
returnDbRoot = true;
}
}
catch (exception& e)
{
cout << endl << "**** glusterctl API exception: " << e.what() << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID, LOG_TYPE_ERROR );
returnDbRoot = true;
}
catch (...)
{
cout << endl << "**** glusterctl API exception: UNKNOWN" << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
writeLog("FAILURE: Error assigning gluster dbroot# " + *pt1 + " to pm" + toPMID, LOG_TYPE_ERROR );
returnDbRoot = true;
}
}
todbrootConfigList.push_back(*pt2);
residedbrootConfigList.erase(pt2);
if (returnDbRoot)
{
// something went wrong return it back to original owner
try
{
string errmsg;
writeLog("reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR );
int ret = glusterctl(oam::GLUSTER_ASSIGN, *pt1, residePM, errmsg);
if ( ret != 0 )
{
cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID + ", error: " + errmsg << endl;
writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID + ", error: " + errmsg, LOG_TYPE_ERROR );
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
mountDBRoot(dbroot1);
//get updated Columnstore.xml distributed
distributeConfigFile("system");
return;
}
catch (exception& e)
{
cout << endl << "**** glusterctl API exception: " << e.what() << endl;
cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID << endl;
writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR );
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
catch (...)
{
cout << endl << "**** glusterctl API exception: UNKNOWN" << endl;
cerr << "FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID << endl;
writeLog("FAILURE: Error reassigning gluster dbroot# " + *pt1 + " to pm" + residePMID, LOG_TYPE_ERROR );
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
}
break;
}
}
}
//set the 2 pms dbroot config
try
{
@ -5381,7 +5417,7 @@ namespace oam
{
writeLog("ERROR: setPmDbrootConfig api failure for pm" + residePMID , LOG_TYPE_ERROR );
cout << endl << "ERROR: setPmDbrootConfig api failure for pm" + residePMID << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
try
@ -5392,7 +5428,7 @@ namespace oam
{
writeLog("ERROR: setPmDbrootConfig api failure for pm" + toPMID , LOG_TYPE_ERROR );
cout << endl << "ERROR: setPmDbrootConfig api failure for pm" + toPMID << endl;
exceptionControl("manualMovePmDbroot", API_FAILURE);
exceptionControl("manualMovePmDbroot", API_INVALID_STATE);
}
//send msg to mount dbroot
@ -5980,7 +6016,7 @@ namespace oam
}
if (!found) {
writeLog("No dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_DEBUG );
writeLog("No dbroots found in " + InstallDir + "/moveDbrootTransactionLog", LOG_TYPE_DEBUG );
cout << "No dbroots found in " << fileName << endl;
}
@ -6518,32 +6554,7 @@ namespace oam
for( ; pt3 != dbrootlist.end() ; pt3++)
{
todbrootConfigList.push_back(*pt3);
/* if ( DataRedundancyConfig == "y")
{
try {
string errmsg;
int ret = glusterctl(oam::GLUSTER_ASSIGN, itoa(*pt3), toPM, errmsg);
if ( ret != 0 )
{
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID + ", error: " + errmsg << endl;
exceptionControl("assignPmDbrootConfig", API_FAILURE);
}
}
catch (exception& e)
{
cout << endl << "**** glusterctl API exception: " << e.what() << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID << endl;
exceptionControl("assignPmDbrootConfig", API_FAILURE);
}
catch (...)
{
cout << endl << "**** glusterctl API exception: UNKNOWN" << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(*pt3) + " to pm" + toPMID << endl;
exceptionControl("assignPmDbrootConfig", API_FAILURE);
}
}
*/ }
}
try
{
@ -6961,12 +6972,14 @@ namespace oam
{
cout << endl << "**** glusterctl API exception: " << e.what() << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl;
writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR );
exceptionControl("removeDbroot", API_FAILURE);
}
catch (...)
{
cout << endl << "**** glusterctl API exception: UNKNOWN" << endl;
cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl;
writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR );
exceptionControl("removeDbroot", API_FAILURE);
}
}

8
oamapps/alarmmanager/alarmmanager.cpp
View File

@ -422,7 +422,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
int pid = getpid();
int tid = gettid();
// get reporting Pprocess Name
// get reporting Process Name
string processName;
if ( repProcessName.empty()) {
// get current process name
@ -468,7 +468,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
args.add("sendAlarmReport error:");
args.add(e.what());
msg.format(args);
ml.logErrorMessage(msg);
ml.logDebugMessage(msg);
}
catch (std::exception& e)
{
@ -479,7 +479,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
args.add("sendAlarmReport error:");
args.add(e.what());
msg.format(args);
ml.logErrorMessage(msg);
ml.logDebugMessage(msg);
}
catch (...)
{
@ -490,7 +490,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
args.add("sendAlarmReport error:");
args.add("general failure");
msg.format(args);
ml.logErrorMessage(msg);
ml.logDebugMessage(msg);
}
return;

10
oamapps/postConfigure/installer.cpp
View File

@ -813,7 +813,10 @@ int main(int argc, char *argv[])
cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl;
cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl;
if ( !rootUser )
cout << ". /etc/profile.d/columnstoreEnv.sh" << endl;
cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl;
cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl;
cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl;
@ -829,7 +832,10 @@ int main(int argc, char *argv[])
cout << endl << "ERROR: MariaDB ColumnStore Process failed to start, check log files in /var/log/mariadb/columnstore" << endl;
cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl;
cout << ". " + installDir + "/bin/columnstoreAlias" << endl << endl;
if ( !rootUser )
cout << ". /etc/profile.d/columnstoreEnv.sh" << endl;
cout << ". /etc/profile.d/columnstoreAlias.sh" << endl << endl;
cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl;
cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl;

4
oamapps/postConfigure/quick_installer_amazon.sh
View File

@ -74,8 +74,8 @@ else
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umCount = "" ]]; then
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName
. /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName
else
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName
. /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName
fi
fi

4
oamapps/postConfigure/quick_installer_multi_server.sh
View File

@ -78,8 +78,8 @@ else
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umIpAddrs = "" ]]; then
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName
. /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName
else
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName
. /etc/profile.d/columnstoreEnv.sh;$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName
fi
fi

2
oamapps/postConfigure/quick_installer_single_server.sh
View File

@ -30,5 +30,5 @@ else
$HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore
echo "Run postConfigure script"
echo ""
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs
. /etc/profile.d/columnstoreEnv.sh; $HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs
fi

45
procmgr/main.cpp
View File

@ -1523,9 +1523,6 @@ void pingDeviceThread()
break;
//set query system state not ready
BRM::DBRM dbrm;
dbrm.setSystemQueryReady(false);
processManager.setQuerySystemState(false);
processManager.setSystemState(oam::BUSY_INIT);
@ -1574,7 +1571,7 @@ void pingDeviceThread()
{
// no dbroots, fail module
log.writeLog(__LINE__, "autoUnMovePmDbroot left no dbroots mounted, failing module restart: " + moduleName, LOG_TYPE_WARNING);
//Issue an alarm
aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, SET);
@ -1597,7 +1594,7 @@ void pingDeviceThread()
//set query system state ready
processManager.setQuerySystemState(true);
break;
goto break_case;
}
}
catch(...)
@ -1619,25 +1616,24 @@ void pingDeviceThread()
if ( retry == 5 )
{
log.writeLog(__LINE__, "autoUnMovePmDbroot: Failed. Fail Module", LOG_TYPE_WARNING);
log.writeLog(__LINE__, "System DBRM READ ONLY - Verify dbroot mounts.", LOG_TYPE_WARNING);
//Issue an alarm
aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, SET);
//set module to disable state
processManager.disableModule(moduleName, true);
// Need to do something here to verify data mounts before resuming
// Best to assume if we reach this you need to put into readonly and verify all dbroots are mounted
//call dbrm control
oam.dbrmctl("reload");
log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG);
// resume the dbrm
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
oam.dbrmctl("readonly");
log.writeLog(__LINE__, "'dbrmctl readonly' done", LOG_TYPE_DEBUG);
//clear count
moduleInfoList[moduleName] = 0;
processManager.setSystemState(oam::ACTIVE);
processManager.setSystemState(oam::DEGRADED);
//set query system state ready
processManager.setQuerySystemState(true);
@ -1807,9 +1803,6 @@ void pingDeviceThread()
}
}
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
@ -1866,9 +1859,6 @@ void pingDeviceThread()
else
processManager.setSystemState(oam::ACTIVE);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
@ -1930,9 +1920,6 @@ void pingDeviceThread()
log.writeLog(__LINE__, "*** module is down: " + moduleName, LOG_TYPE_CRITICAL);
//set query system state not ready
BRM::DBRM dbrm;
dbrm.setSystemQueryReady(false);
processManager.setQuerySystemState(false);
processManager.setSystemState(oam::BUSY_INIT);
@ -1994,9 +1981,6 @@ void pingDeviceThread()
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
@ -2199,9 +2183,6 @@ void pingDeviceThread()
//set recycle process
processManager.recycleProcess(moduleName);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
@ -2218,9 +2199,6 @@ void pingDeviceThread()
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
}
@ -2231,8 +2209,8 @@ void pingDeviceThread()
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
//enable query stats
dbrm.setSystemQueryReady(true);
//set recycle process
processManager.recycleProcess(moduleName);
//set query system state ready
processManager.setQuerySystemState(true);
@ -2352,6 +2330,7 @@ void pingDeviceThread()
}
} //end of for loop
}
break_case:
// check and take action if LAN outage is flagged
if (LANOUTAGESUPPORT && !LANOUTAGEACTIVE && LOCALNICDOWN)

10
procmgr/processmanager.cpp
View File

@ -836,10 +836,15 @@ void processMSG(messageqcpp::IOSocket* cfIos)
status = processManager.disableModule(moduleName, true);
log.writeLog(__LINE__, "Disable Module Completed on " + moduleName, LOG_TYPE_INFO);
processManager.recycleProcess(moduleName);
//check for SIMPLEX Processes on mate might need to be started
processManager.checkSimplexModule(moduleName);
processManager.setSystemState(oam::ACTIVE);
//set query system state ready
processManager.setQuerySystemState(true);
processManager.setSystemState(oam::ACTIVE);
}
else
{
@ -7068,6 +7073,7 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist)
else
processManager.setSystemState(oam::FAILED);
// exit thread
log.writeLog(__LINE__, "startSystemThread Exit", LOG_TYPE_DEBUG);
startsystemthreadStatus = status;

39
procmon/main.cpp
View File

@ -131,13 +131,6 @@ int main(int argc, char **argv)
if (p && *p)
USER = p;
// change permissions on /dev/shm
if ( !rootUser)
{
string cmd = "sudo chmod 777 /dev/shm >/dev/null 2>&1";
system(cmd.c_str());
}
// get and set locale language
string systemLang = "C";
@ -315,8 +308,9 @@ int main(int argc, char **argv)
if ( count >= 120 ) {
log.writeLog(__LINE__, "Standby PM not responding, infinidb shutting down", LOG_TYPE_CRITICAL);
//Set the alarm
aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET);
sleep (1);
// aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET);
// sleep (1);
string cmd = startup::StartUp::installDir() + "/bin/infinidb stop > /dev/null 2>&1";
system(cmd.c_str());
}
@ -342,7 +336,7 @@ int main(int argc, char **argv)
sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", IPaddr);
log.writeLog(__LINE__, "set ProcMgr IPaddr to Old Standby Module: " + IPaddr, LOG_TYPE_DEBUG);
//update Calpont Config table
//update MariaDB ColumnStore Config table
try {
sysConfig->write();
sleep(1);
@ -500,8 +494,8 @@ int main(int argc, char **argv)
{
log.writeLog(__LINE__, "Check DB mounts failed, shutting down", LOG_TYPE_CRITICAL);
//Set the alarm
aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET);
sleep (1);
// aMonitor.sendAlarm(config.moduleName().c_str(), STARTUP_DIAGNOTICS_FAILURE, SET);
// sleep (1);
string cmd = startup::StartUp::installDir() + "/bin/columnstore stop > /dev/null 2>&1";
system(cmd.c_str());
}
@ -1339,7 +1333,7 @@ static void chldHandleThread(MonitorConfig config)
(*listPtr).processID != 0 ) ||
( (*listPtr).state == oam::ACTIVE && (*listPtr).processID == 0 ) )
{
log.writeLog(__LINE__, "*****Calpont Process Restarting: " + (*listPtr).ProcessName + ", old PID = " + oam.itoa((*listPtr).processID), LOG_TYPE_CRITICAL);
log.writeLog(__LINE__, "*****MariaDB ColumnStore Process Restarting: " + (*listPtr).ProcessName + ", old PID = " + oam.itoa((*listPtr).processID), LOG_TYPE_CRITICAL);
if ( (*listPtr).dieCounter >= processRestartCount ||
processRestartCount == 0) {
@ -1536,7 +1530,7 @@ static void chldHandleThread(MonitorConfig config)
}
//Log this event
log.writeLog(__LINE__, "Calpont Process " + (*listPtr).ProcessName + restartStatus, LOG_TYPE_INFO);
log.writeLog(__LINE__, "MariaDB ColumnStore Process " + (*listPtr).ProcessName + restartStatus, LOG_TYPE_INFO);
}
}
}
@ -2449,19 +2443,10 @@ void processStatusMSG(messageqcpp::IOSocket* cfIos)
memcpy(fShmSystemStatus[0].StateChangeDate, oam.getCurrentTime().c_str(), DATESIZE);
log.writeLog(__LINE__, "statusControl: REQUEST RECEIVED: Set System State = " + oamState[state], LOG_TYPE_DEBUG);
}
//if DMLProc set to ACTIVE, set system state to ACTIVE if in an INIT state
// if ( processName == "DMLProc" && state == oam::ACTIVE )
// {
// if ( fShmSystemStatus[0].OpState == oam::BUSY_INIT ||
// fShmSystemStatus[0].OpState == oam::MAN_INIT ||
// fShmSystemStatus[0].OpState == oam::AUTO_INIT )
// {
// fShmSystemStatus[0].OpState = state;
// memcpy(fShmSystemStatus[0].StateChangeDate, oam.getCurrentTime().c_str(), DATESIZE);
// log.writeLog(__LINE__, "statusControl: REQUEST RECEIVED: Set System State = " + oamState[state], LOG_TYPE_DEBUG);
// }
// }
BRM::DBRM dbrm;
dbrm.setSystemQueryReady(true);
}
}
break;

14
procmon/processmonitor.cpp
View File

@ -988,13 +988,6 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
int requestStatus = oam::API_SUCCESS;
log.writeLog(__LINE__, "MSG RECEIVED: Start All process request...");
// change permissions on /dev/shm
string cmd = "chmod 755 /dev/shm >/dev/null 2>&1";
if ( !rootUser)
cmd = "sudo chmod 777 /dev/shm >/dev/null 2>&1";
system(cmd.c_str());
//start the mysqld daemon
try {
oam.actionMysqlCalpont(MYSQL_START);
@ -6107,10 +6100,13 @@ int ProcessMonitor::glusterAssign(std::string dbrootID)
command = "sudo mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" +
dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterAssign.txt 2>&1";
}
int ret = system(command.c_str());
if ( WEXITSTATUS(ret) != 0 )
{
log.writeLog(__LINE__, "glusterAssign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR);
ifstream in("/tmp/glusterAssign.txt");
in.seekg(0, std::ios::end);
int size = in.tellg();
@ -6151,9 +6147,13 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID)
{
command = "sudo umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterUnassign.txt 2>&1";
}
int ret = system(command.c_str());
if ( WEXITSTATUS(ret) != 0 )
{
log.writeLog(__LINE__, "glusterUnassign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR);
ifstream in("/tmp/glusterUnassign.txt");
in.seekg(0, std::ios::end);
int size = in.tellg();

156
utils/clusterTester/columnstoreClusterTester.sh
View File

@ -10,7 +10,7 @@ CHECK=true
REPORTPASS=true
LOGFILE=""
OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16")
OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16" "ubuntu18")
NODE_IPADDRESS=""
@ -37,7 +37,7 @@ checkContinue() {
}
###
# Print Fucntions
# Print Functions
###
helpPrint () {
@ -57,7 +57,7 @@ helpPrint () {
echo ""
echo "Additional information on Tool is documented at:"
echo ""
echo "https://mariadb.com/kb/en/mariadb/*****/"
echo "https://mariadb.com/kb/en/library/mariadb-columnstore-cluster-test-tool/"
echo ""
echo "Items that are checked:"
echo " Node Ping test"
@ -65,6 +65,7 @@ helpPrint () {
echo " ColumnStore Port test"
echo " OS version"
echo " Locale settings"
echo " Umask settings"
echo " Firewall settings"
echo " Date/time settings"
echo " Dependent packages installed"
@ -326,16 +327,18 @@ checkSSH()
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
if [ $PASSWORD == "ssh" ] ; then
echo $ipadd " Node Passed SSH login test using ssh-keys"
echo $ipadd " Node Passed SSH login test using ssh-keys"
else
echo $ipadd " Node Passed SSH login test using user password"
echo $ipadd " Node Passed SSH login test using user password"
fi
else
if [ $PASSWORD == "ssh" ] ; then
echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys"
echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys"
else
echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password"
echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password"
fi
echo "Error - Fix the SSH login issue and rerun test"
exit 1
fi
done
@ -489,12 +492,47 @@ checkLocale()
fi
}
checkSELINUX()
checkLocalUMASK()
{
# UMASK check
#
echo ""
echo "** Run Local UMASK check"
echo ""
pass=true
filename=UMASKtest
rm -f $filename
touch $filename
permission=$(stat -c "%A" "$filename")
result=${permission:4:1}
if [ ${result} == "r" ] ; then
result=${permission:7:1}
if [ ${result} == "r" ] ; then
echo "UMASK local setting test passed"
else
echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022"
pass=false
fi
if ! $pass; then
checkContinue
fi
rm -f $filename
}
checkLocalSELINUX()
{
# SELINUX check
#
echo ""
echo "** Run SELINUX check"
echo "** Run Local SELINUX check"
echo ""
pass=true
@ -511,21 +549,86 @@ checkSELINUX()
echo "Local Node SELINUX setting is Not Enabled"
fi
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "$ipadd Node SELINUX setting is Not Enabled"
else
`cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results"
pass=false
else
echo "$ipadd Node SELINUX setting is Not Enabled"
fi
`rm -f config`
fi
done
if ! $pass; then
checkContinue
fi
}
checkUMASK()
{
# UMASK check
#
echo ""
echo "** Run UMASK check"
echo ""
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > /tmp/remote_command_check 2>&1`
if [ "$?" -eq 0 ]; then
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> /tmp/remote_scp_get 2>&1`
if [ "$?" -eq 0 ]; then
permission=`cat test.log`
result=${permission:4:1}
if [ ${result} == "r" ] ; then
result=${permission:7:1}
if [ ${result} == "r" ] ; then
echo "$ipadd Node UMASK setting test passed"
else
echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check /tmp/remote_scp_get"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check /tmp/remote_command_check"
pass=false
fi
`rm -f test.log`
done
if ! $pass; then
checkContinue
fi
rm -f $filename
}
checkSELINUX()
{
# SELINUX check
#
echo ""
echo "** Run SELINUX check"
echo ""
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "$ipadd Node SELINUX setting is Not Enabled"
else
`cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results"
pass=false
else
echo "$ipadd Node SELINUX setting is Not Enabled"
fi
`rm -f config`
fi
done
if ! $pass; then
checkContinue
fi
}
checkFirewalls()
@ -949,7 +1052,7 @@ checkPackages()
declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" )
declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18")
if [ "$OS" == "ubuntu16" ] ; then
if [ "$OS" == "ubuntu16" ] || [ "$OS" == "ubuntu18" ]; then
if [ ! `which dpkg 2>/dev/null` ] ; then
echo "${bold}Failed${normal}, Local Node ${bold}rpm${normal} package not installed"
pass=false
@ -1307,12 +1410,15 @@ echo ""
checkLocalOS
checkLocalDir
checkLocalUMASK
checkLocalSELINUX
if [ "$IPADDRESSES" != "" ]; then
checkPing
checkSSH
checkRemoteDir
checkOS
checkLocale
checkUMASK
checkSELINUX
checkFirewalls
checkPorts

2
utils/clusterTester/os_detect.sh
View File

@ -29,7 +29,7 @@ detectOS () {
echo Operating System name: $osPrettyName
echo Operating System tag: $osTag
case "$osTag" in
centos6|centos7|ubuntu16|debian8|suse12|debian9)
centos6|centos7|ubuntu16|debian8|suse12|debian9|ubuntu18)
;;
*)
echo OS not supported

81
utils/messageqcpp/messagequeue.cpp
View File

@ -152,26 +152,44 @@ void MessageQueueClient::shutdown()
void MessageQueueClient::setup(bool syncProto)
{
string otherEndIPStr;
string otherEndPortStr;
uint16_t port;
string otherEndIPStr;
string otherEndPortStr;
struct addrinfo hints, *servinfo;
int rc = 0;
otherEndIPStr = fConfig->getConfig(fOtherEnd, "IPAddr");
otherEndPortStr = fConfig->getConfig(fOtherEnd, "Port");
otherEndIPStr = fConfig->getConfig(fOtherEnd, "IPAddr");
otherEndPortStr = fConfig->getConfig(fOtherEnd, "Port");
if (otherEndIPStr.length() == 0) otherEndIPStr = "127.0.0.1";
if (otherEndIPStr.length() == 0) otherEndIPStr = "127.0.0.1";
if (otherEndPortStr.length() == 0 || (port = static_cast<uint16_t>(strtol(otherEndPortStr.c_str(), 0, 0))) == 0)
{
string msg = "MessageQueueClient::MessageQueueClient: config error: Invalid/Missing Port attribute";
throw runtime_error(msg);
}
if (otherEndPortStr.length() == 0 || static_cast<uint16_t>(strtol(otherEndPortStr.c_str(), 0, 0)) == 0)
{
string msg = "MessageQueueClient::setup(): config error: Invalid/Missing Port attribute";
throw runtime_error(msg);
}
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
sinp->sin_family = AF_INET;
sinp->sin_port = htons(port);
sinp->sin_addr.s_addr = inet_addr(otherEndIPStr.c_str());
memset(&hints, 0, sizeof hints);
// ATM We support IPv4 only.
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if( !(rc = getaddrinfo(otherEndIPStr.c_str(), otherEndPortStr.c_str(), &hints, &servinfo)) )
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
*sinp = *reinterpret_cast<sockaddr_in*>(servinfo->ai_addr);
freeaddrinfo(servinfo);
}
else
{
string msg = "MessageQueueClient::setup(): ";
msg.append(gai_strerror(rc));
logging::Message::Args args;
logging::LoggingID li(31);
args.add(msg);
fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li);
}
#ifdef SKIP_IDB_COMPRESSION
fClientSock.setSocketImpl(new InetStreamSocket());
@ -197,15 +215,34 @@ MessageQueueClient::MessageQueueClient(const string& otherEnd, Config* config, b
setup(syncProto);
}
MessageQueueClient::MessageQueueClient(const string& ip, uint16_t port, bool syncProto) :
MessageQueueClient::MessageQueueClient(const string& dnOrIp, uint16_t port, bool syncProto) :
fLogger(31), fIsAvailable(true)
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
sinp->sin_family = AF_INET;
sinp->sin_port = htons(port);
sinp->sin_addr.s_addr = inet_addr(ip.c_str());
struct addrinfo hints, *servinfo;
int rc = 0;
memset(&hints, 0, sizeof hints);
// ATM We support IPv4 only.
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if( !(rc = getaddrinfo(dnOrIp.c_str(), NULL, &hints, &servinfo)) )
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
*sinp = *reinterpret_cast<sockaddr_in*>(servinfo->ai_addr);
sinp->sin_port = htons(port);
freeaddrinfo(servinfo);
}
else
{
string msg = "MessageQueueClient::MessageQueueClient(): ";
msg.append(gai_strerror(rc));
logging::Message::Args args;
logging::LoggingID li(31);
args.add(msg);
fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li);
}
#ifdef SKIP_IDB_COMPRESSION
fClientSock.setSocketImpl(new InetStreamSocket());
#else

3
utils/messageqcpp/messagequeue.h
View File

@ -33,6 +33,7 @@
#include <stdio.h>
#else
#include <netinet/in.h>
#include <netdb.h>
#endif
#include "serversocket.h"
@ -182,7 +183,7 @@ public:
*
* construct a queue from this process to otherEnd on the given IP and Port.
*/
EXPORT explicit MessageQueueClient(const std::string& ip, uint16_t port, bool syncProto=true);
EXPORT explicit MessageQueueClient(const std::string& dnOrIp, uint16_t port, bool syncProto=true);
/**

6
utils/messageqcpp/messagequeuepool.cpp
View File

@ -36,12 +36,12 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts)
return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000;
}
MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &ip, uint64_t port)
MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &dnOrIp, uint64_t port)
{
boost::mutex::scoped_lock lock(queueMutex);
std::ostringstream oss;
oss << ip << "_" << port;
oss << dnOrIp << "_" << port;
std::string searchString = oss.str();
MessageQueueClient *returnClient = MessageQueueClientPool::findInPool(searchString);
@ -58,7 +58,7 @@ MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &ip, u
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client = new MessageQueueClient(ip, port);
newClientObject->client = new MessageQueueClient(dnOrIp, port);
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
clientMap.insert(std::pair<std::string, ClientObject*>(searchString, newClientObject));

2
utils/messageqcpp/messagequeuepool.h
View File

@ -41,7 +41,7 @@ class MessageQueueClientPool
{
public:
static MessageQueueClient *getInstance(const std::string &module);
static MessageQueueClient *getInstance(const std::string &ip, uint64_t port);
static MessageQueueClient *getInstance(const std::string &dnOrIp, uint64_t port);
static void releaseInstance(MessageQueueClient * client);
static void deleteInstance(MessageQueueClient * client);
static MessageQueueClient *findInPool(const std::string &search);

111
writeengine/wrapper/writeengine.cpp
View File

@ -182,6 +182,37 @@ int WriteEngineWrapper::checkValid(const TxnID& txnid, const ColStructList& colS
return NO_ERROR;
}
/*@brief findSmallestColumn --Find the smallest column for this table
*/
/***********************************************************
* DESCRIPTION:
* Find the smallest column for this table
* PARAMETERS:
* lowColLen - returns smallest column width
* colId - returns smallest column id
* colStructList - column struct list
* RETURN:
* void
***********************************************************/
void WriteEngineWrapper::findSmallestColumn(uint32_t& colId, ColStructList colStructList)
// MCOL-1675: find the smallest column width to calculate the RowID from so
// that all HWMs will be incremented by this operation
{
int32_t lowColLen = 8192;
for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++)
{
if (colStructList[colIt].colWidth < lowColLen)
{
colId = colIt;
lowColLen = colStructList[colId].colWidth;
if ( lowColLen == 1 )
{
break;
}
}
}
}
/*@convertValArray - Convert interface values to internal values
*/
/***********************************************************
@ -847,6 +878,11 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
for (i = 0; i < colStructList.size(); i++)
Convertor::convertColType(&colStructList[i]);
uint32_t colId = 0;
// MCOL-1675: find the smallest column width to calculate the RowID from so
// that all HWMs will be incremented by this operation
findSmallestColumn(colId, colStructList);
// rc = checkValid(txnid, colStructList, colValueList, ridList);
// if (rc != NO_ERROR)
// return rc;
@ -873,8 +909,8 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
//--------------------------------------------------------------------------
if (isFirstBatchPm)
{
currentDBrootIdx = dbRootExtentTrackers[0]->getCurrentDBRootIdx();
extentInfo = dbRootExtentTrackers[0]->getDBRootExtentList();
currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx();
extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList();
dbRoot = extentInfo[currentDBrootIdx].fDbRoot;
partitionNum = extentInfo[currentDBrootIdx].fPartition;
@ -914,7 +950,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
{
colOp = m_colOp[op(colStructList[i].fCompressionType)];
colOp->initColumn(curCol);
colOp->setColParam(curCol, 0, colStructList[i].colWidth, colStructList[i].colDataType,
colOp->setColParam(curCol, colId, colStructList[i].colWidth, colStructList[i].colDataType,
colStructList[i].colType, colStructList[i].dataOid, colStructList[i].fCompressionType,
dbRoot, partitionNum, segmentNum);
rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, extents[i].allocSize, dbRoot,
@ -1040,7 +1076,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
} // if (isFirstBatchPm)
else //get the extent info from tableMetaData
{
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid);
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid);
ColExtsInfo::iterator it = aColExtsInfo.begin();
while (it != aColExtsInfo.end())
{
@ -1073,7 +1109,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
//--------------------------------------------------------------------------
// allocate row id(s)
//--------------------------------------------------------------------------
curColStruct = colStructList[0];
curColStruct = colStructList[colId];
colOp = m_colOp[op(curColStruct.fCompressionType)];
colOp->initColumn(curCol);
@ -1084,23 +1120,27 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
vector<ExtentInfo> fileInfo;
dbRoot = curColStruct.fColDbRoot;
//use the first column to calculate row id
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid);
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid);
ColExtsInfo::iterator it = aColExtsInfo.begin();
while (it != aColExtsInfo.end())
{
if ((it->dbRoot == colStructList[0].fColDbRoot) && (it->partNum == colStructList[0].fColPartition) && (it->segNum == colStructList[0].fColSegment) && it->current )
if ((it->dbRoot == colStructList[colId].fColDbRoot) &&
(it->partNum == colStructList[colId].fColPartition) &&
(it->segNum == colStructList[colId].fColSegment) && it->current )
{
break;
}
it++;
}
if (it != aColExtsInfo.end())
{
hwm = it->hwm;
//cout << "Got from colextinfo hwm for oid " << colStructList[0].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl;
//cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl;
}
oldHwm = hwm; //Save this info for rollback
//need to pass real dbRoot, partition, and segment to setColParam
colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType,
colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType,
curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType,
curColStruct.fColDbRoot, curColStruct.fColPartition, curColStruct.fColSegment);
rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix); // @bug 5572 HDFS tmp file
@ -1123,13 +1163,13 @@ timer.start("allocRowId");
if (idbdatafile::IDBPolicy::useHdfs())
insertSelect = true;
rc = colOp->allocRowId(txnid, bUseStartExtent,
rc = colOp->allocRowId(txnid, bUseStartExtent,
curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile,
newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm);
//cout << "after allocrowid, total row = " <<totalRow << " newExtent is " << newExtent << endl;
// cout << "column oid " << curColStruct.dataOid << " has hwm:newHwm = " << hwm <<":" << newHwm<< endl;
if (rc != NO_ERROR) //Clean up is already done
//cout << "after allocrowid, total row = " <<totalRow << " newExtent is " << newExtent << endl;
//cout << "column oid " << curColStruct.dataOid << " has hwm:newHwm = " << hwm <<":" << newHwm<< endl;
if (rc != NO_ERROR) //Clean up is already done
return rc;
#ifdef PROFILE
@ -1139,14 +1179,17 @@ timer.stop("allocRowId");
// Expand initial abbreviated extent if any RID in 1st extent is > 256K.
// if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it.
//--------------------------------------------------------------------------
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
if ((curCol.dataFile.fPartition == 0) &&
(curCol.dataFile.fSegment == 0) &&
((totalRow-rowsLeft) > 0) &&
(rowIdArray[totalRow-rowsLeft-1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK))
{
for (unsigned k=1; k<colStructList.size(); k++)
for (unsigned k=0; k<colStructList.size(); k++)
{
// Skip the selected column
if (k == colId)
continue;
Column expandCol;
colOp = m_colOp[op(colStructList[k].fCompressionType)];
colOp->setColParam(expandCol, 0,
@ -1505,18 +1548,10 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
for (i = 0; i < colStructList.size(); i++)
Convertor::convertColType(&colStructList[i]);
// MCOL-984: find the smallest column width to calculate the RowID from so
// that all HWMs will be incremented by this operation
int32_t lowColLen = 8192;
int32_t colId = 0;
for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++)
{
if (colStructList[colIt].colWidth < lowColLen)
{
colId = colIt;
lowColLen = colStructList[colId].colWidth;
}
}
uint32_t colId = 0;
// MCOL-1675: find the smallest column width to calculate the RowID from so
// that all HWMs will be incremented by this operation
findSmallestColumn(colId, colStructList);
// rc = checkValid(txnid, colStructList, colValueList, ridList);
// if (rc != NO_ERROR)
@ -1809,7 +1844,7 @@ timer.stop("allocRowId");
// Expand initial abbreviated extent if any RID in 1st extent is > 256K.
// if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it.
//--------------------------------------------------------------------------
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
if ((curCol.dataFile.fPartition == 0) &&
(curCol.dataFile.fSegment == 0) &&
((totalRow-rowsLeft) > 0) &&
@ -1821,7 +1856,8 @@ timer.stop("allocRowId");
if (k == colId)
continue;
Column expandCol;
colOp = m_colOp[op(colStructList[k].fCompressionType)];
colOp = m_colOp[op(colStructList[k].fCompressionType)];
// Shouldn't we change 0 to colId here?
colOp->setColParam(expandCol, 0,
colStructList[k].colWidth,
colStructList[k].colDataType,
@ -2782,6 +2818,11 @@ StopWatch timer;
for (i = 0; i < colStructList.size(); i++)
Convertor::convertColType(&colStructList[i]);
uint32_t colId = 0;
// MCOL-1675: find the smallest column width to calculate the RowID from so
// that all HWMs will be incremented by this operation
findSmallestColumn(colId, colStructList);
rc = checkValid(txnid, colStructList, colValueList, ridList);
if (rc != NO_ERROR)
return rc;
@ -2799,7 +2840,7 @@ StopWatch timer;
//--------------------------------------------------------------------------
// allocate row id(s)
//--------------------------------------------------------------------------
curColStruct = colStructList[0];
curColStruct = colStructList[colId];
colOp = m_colOp[op(curColStruct.fCompressionType)];
colOp->initColumn(curCol);
@ -2834,7 +2875,7 @@ StopWatch timer;
oldHwm = hwm; //Save this info for rollback
//need to pass real dbRoot, partition, and segment to setColParam
colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType,
colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType,
curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType,
dbRoot, partitionNum, segmentNum);
@ -2944,13 +2985,15 @@ timer.stop("allocRowId");
// if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it.
//--------------------------------------------------------------------------
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
if ((colStructList[0].fColPartition == 0) &&
(colStructList[0].fColSegment == 0) &&
if ((colStructList[colId].fColPartition == 0) &&
(colStructList[colId].fColSegment == 0) &&
((totalRow-rowsLeft) > 0) &&
(rowIdArray[totalRow-rowsLeft-1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK))
{
for (unsigned k=1; k<colStructList.size(); k++)
for (unsigned k=0; k<colStructList.size(); k++)
{
if (k == colId)
continue;
Column expandCol;
colOp = m_colOp[op(colStructList[k].fCompressionType)];
colOp->setColParam(expandCol, 0,

5
writeengine/wrapper/writeengine.h
View File

@ -607,6 +607,11 @@ private:
*/
int checkValid(const TxnID& txnid, const ColStructList& colStructList, const ColValueList& colValueList, const RIDList& ridList) const;
/**
* @brief Find the smallest column for this table
*/
void findSmallestColumn(uint32_t &colId, ColStructList colStructList);
/**
* @brief Convert interface column type to a internal column type
*/