database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-10 01:22:48 +03:00
Code Activity

Merge pull request #439 from mariadb-corporation/dev-merge-up-20180409

Browse Source 
Dev merge up 20180409
This commit is contained in:
David.Hall
2018-04-09 13:23:21 -05:00
committed by GitHub
parent 7075b9571d 280dcb44bf
commit c9be90740c
31 changed files with 304 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cpackEngineDEB.cmake
View File

@@ -69,7 +69,7 @@ if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8")
elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5")
else()
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, snappy, net-tools")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools")
endif ()
SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs")

6
dbcon/ddlpackageproc/altertableprocessor.cpp
View File

@@ -223,7 +223,13 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType
break;
case (CalpontSystemCatalog::BLOB):
if (newType.fType == DDL_BLOB && colType.colWidth == newType.fLength) return true;
break;
case (CalpontSystemCatalog::TEXT):
if (newType.fType == DDL_TEXT && colType.colWidth == newType.fLength) return true;
break;
default:

2
dbcon/execplan/arithmeticoperator.cpp
View File

@@ -111,7 +111,7 @@ void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b)
bool ArithmeticOperator::operator==(const ArithmeticOperator& t) const
{
if (fData == t.fData)
if (data() == t.data())
return true;
return false;

2
dbcon/execplan/filter.cpp
View File

@@ -73,7 +73,7 @@ const string Filter::toString() const
bool Filter::operator==(const Filter& t) const
{
if (fData == t.fData)
if (data() == t.data())
return true;
return false;

2
dbcon/execplan/logicoperator.cpp
View File

@@ -115,7 +115,7 @@ void LogicOperator::unserialize(messageqcpp::ByteStream& b)
bool LogicOperator::operator==(const LogicOperator& t) const
{
if (fData == t.fData)
if (data() == t.data())
return true;
return false;

2
dbcon/execplan/predicateoperator.cpp
View File

@@ -164,7 +164,7 @@ void PredicateOperator::unserialize(messageqcpp::ByteStream& b)
bool PredicateOperator::operator==(const PredicateOperator& t) const
{
if (fData == t.fData)
if (data() == t.data())
return true;
return false;

12
dbcon/execplan/simplecolumn.cpp
View File

@@ -388,21 +388,19 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const
if (fColumnName != t.fColumnName)
return false;
if (fIndexName != t.fIndexName)
return false;
// if (fIndexName != t.fIndexName)
// return false;
if (fViewName != t.fViewName)
return false;
if (fOid != t.fOid)
return false;
if (fData != t.fData)
return false;
if (fAlias != t.fAlias)
if (data() != t.data())
return false;
// if (fAlias != t.fAlias)
// return false;
if (fTableAlias != t.fTableAlias)
return false;

2
dbcon/execplan/treenodeimpl.cpp
View File

@@ -64,7 +64,7 @@ const string TreeNodeImpl::toString() const
bool TreeNodeImpl::operator==(const TreeNodeImpl& t) const
{
if (fData == t.fData)
if (data() == t.data())
return true;
return false;

10
dbcon/joblist/lbidlist.cpp
View File

@@ -27,6 +27,7 @@
#include "calpontsystemcatalog.h"
#include "brm.h"
#include "brmtypes.h"
#include "dataconvert.h"
#define IS_VERBOSE (fDebug >= 4)
#define IS_DETAIL (fDebug >= 3)
@@ -741,7 +742,14 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min,
if (bIsChar && 1 < ct.colWidth)
{
scan = compareVal(order_swap(Min), order_swap(Max), order_swap(value),
// MCOL-1246 Trim trailing whitespace for matching so that we have
// the same as InnoDB behaviour
int64_t tMin = Min;
int64_t tMax = Max;
dataconvert::DataConvert::trimWhitespace(tMin);
dataconvert::DataConvert::trimWhitespace(tMax);
scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value),
op, lcf);
// cout << "scan=" << (uint32_t) scan << endl;
}

6
dbcon/mysql/columnstore_info.sql
View File

@@ -52,21 +52,21 @@ CREATE PROCEDURE table_usage (IN t_schema char(64), IN t_name char(64))
CREATE TABLE columnstore_info.columnstore_files engine=myisam as (select * from information_schema.columnstore_files);
ALTER TABLE columnstore_info.columnstore_files ADD INDEX `object_id` (`object_id`);
IF t_name IS NOT NULL THEN
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics where table_name = t_name and (table_schema = t_schema or t_schema IS NULL)
group by table_schema, table_name
) q;
ELSEIF t_schema IS NOT NULL THEN
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics where table_schema = t_schema
group by table_schema, table_name
) q;
ELSE
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
FROM
columnstore_info.columnstore_columns ics

16
dbcon/mysql/ha_calpont_ddl.cpp
View File

@@ -2183,6 +2183,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
THD* thd = current_thd;
string emsg;
ostringstream stmt1;
pair<string, string> fromPair;
pair<string, string> toPair;
string stmt;
@@ -2210,16 +2211,15 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
return -1;
}
stmt = thd->query();
stmt += ';';
stmt1 << "alter table " << fromPair.second << " rename to " << toPair.second << ";";
stmt = stmt1.str();
string db;
if ( thd->db )
db = thd->db;
else if ( fromPair.first.length() != 0 )
if ( fromPair.first.length() != 0 )
db = fromPair.first;
else
db = toPair.first;
else if ( thd->db )
db = thd->db;
int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg);
@@ -2266,7 +2266,7 @@ extern "C"
int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype);
if (rc != 0)
push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str());
push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str());
return rc;
}

150
dbcon/mysql/ha_calpont_execplan.cpp
View File

@@ -2945,6 +2945,13 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp
break;
}
case Item::COND_ITEM:
{
// MCOL-1196: Allow COND_ITEM thru. They will be picked up
// by further logic. It may become desirable to add code here.
break;
}
default:
{
gwi.fatalParseError = true;
@@ -3629,123 +3636,92 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS
if (((Item_func_case*)item)->get_first_expr_num() == -1)
funcName = "case_searched";
if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause
{
// the first argument
if (funcName == "case_searched")
{
for (uint32_t i = 0; i < item->argument_count(); i++)
{
if (i % 2 == 0 && i != 1 && i != item->argument_count() - 1)
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
funcParms.push_back(sptp);
}
else
{
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
if (parm)
{
sptp.reset(new ParseTree(parm));
}
else
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
}
funcParms.push_back(sptp);
}
}
}
else
{
for (uint32_t i = 0; i < item->argument_count(); i++)
{
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
if (parm)
{
sptp.reset(new ParseTree(parm));
}
else
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
}
funcParms.push_back(sptp);
}
}
}
else // where clause
{
funcParms.reserve(item->argument_count());
// so buildXXXcolumn function will not pop stack.
ClauseType realClauseType = gwi.clauseType;
gwi.clauseType = SELECT;
if (funcName == "case_searched")
// We ought to be able to just build from the stack, and would
// be able to if there were any way to know which stack had the
// next case item. Unfortunately, parameters may have been pushed
// onto the ptWorkStack or rcWorkStack or neither, depending on type
// and position. We can't tell which at this point, so we
// rebuild the item from the arguments directly and then try to
// figure what to pop, if anything, in order to sync the stacks.
for (int32_t i = item->argument_count() - 1; i >= 0; i--)
{
for (uint32_t i = 0; i < item->argument_count(); i++)
// For case_searched, we know the items for the WHEN clause will
// not be ReturnedColumns. We do this separately just to save
// some cpu cycles trying to build a ReturnedColumn as below.
// Every even numbered arg is a WHEN. In between are the THEN.
// An odd number of args indicates an ELSE residing in the last spot.
if (funcName == "case_searched" &&
i % 2 == 0 && uint(i) != item->argument_count() - 1)
{
if (i % 2 == 0 && i != item->argument_count() - 1)
{
// build item from arguments to avoid parm sequence complexity
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
funcParms.push_back(sptp);
if (!gwi.ptWorkStack.empty())
if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data())
{
gwi.ptWorkStack.pop();
}
}
else
{
// First try building a ReturnedColumn. It may or may not succeed
// depending on the types involved. There's also little correlation
// between buildReturnedColumn and the existance of the item on
// rwWorkStack or ptWorkStack.
// For example, simple predicates, such as 1=1 or 1=0, land in the
// ptWorkStack but other stuff might land in the rwWorkStack
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
if (parm)
{
sptp.reset(new ParseTree(parm));
if (!gwi.rcWorkStack.empty())
// We need to pop whichever stack is holding it, if any.
if ((!gwi.rcWorkStack.empty()) &&
*gwi.rcWorkStack.top() == parm)
{
gwi.rcWorkStack.pop();
}
else if (!gwi.ptWorkStack.empty())
{
ReturnedColumn* ptrc = dynamic_cast<ReturnedColumn*>(gwi.ptWorkStack.top()->data());
if (ptrc && *ptrc == *parm)
gwi.ptWorkStack.pop();
}
}
else
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
if (!gwi.ptWorkStack.empty())
// We need to pop whichever stack is holding it, if any.
if ((!gwi.ptWorkStack.empty()) &&
*gwi.ptWorkStack.top()->data() == sptp->data())
{
gwi.ptWorkStack.pop();
}
funcParms.push_back(sptp);
}
}
}
else // simple_case
else if (!gwi.rcWorkStack.empty())
{
for (uint32_t i = 0; i < item->argument_count(); i++)
{
ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport);
// Probably won't happen, but it might have been on the
// rcWorkStack all along.
ReturnedColumn* ptrc = dynamic_cast<ReturnedColumn*>(sptp->data());
if (parm)
if (ptrc && *ptrc == *gwi.rcWorkStack.top())
{
sptp.reset(new ParseTree(parm));
if (!gwi.rcWorkStack.empty())
gwi.rcWorkStack.pop();
}
else
{
sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport));
if (!gwi.ptWorkStack.empty())
gwi.ptWorkStack.pop();
}
}
}
funcParms.push_back(sptp);
}
funcParms.insert(funcParms.begin(), sptp);
}
// recover clause type
gwi.clauseType = WHERE;
}
gwi.clauseType = realClauseType;
if (gwi.fatalParseError)
{
@@ -4818,9 +4794,9 @@ void gp_walk(const Item* item, void* arg)
// bug 3137. If filter constant like 1=0, put it to ptWorkStack
// MariaDB bug 750. Breaks if compare is an argument to a function.
if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top())
&& isPredicateFunction(ifp, gwip))
// if (isPredicateFunction(ifp, gwip))
// if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top())
// && isPredicateFunction(ifp, gwip))
if (isPredicateFunction(ifp, gwip))
gwip->ptWorkStack.push(new ParseTree(cc));
else
gwip->rcWorkStack.push(cc);
@@ -6223,7 +6199,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i
// @bug 1706
String funcStr;
ifp->print(&funcStr, QT_INFINIDB);
gwi.selectCols.push_back(string(funcStr.c_ptr()) + " `" + escapeBackTick(ifp->name) + "`");
string valStr;
valStr.assign(funcStr.ptr(), funcStr.length());
gwi.selectCols.push_back(valStr + " `" + escapeBackTick(ifp->name) + "`");
// clear the error set by buildFunctionColumn
gwi.fatalParseError = false;
gwi.parseErrorText = "";

2
oam/install_scripts/columnstore
View File

@@ -36,7 +36,7 @@ InstallDir=$COLUMNSTORE_INSTALL_DIR
if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then
export PATH=$InstallDir/bin:$InstallDir/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$InstallDir/lib:$InstallDir/mysql/lib/mysql
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$InstallDir/lib:$InstallDir/mysql/lib
fi
#hadoop

2
oam/install_scripts/module_installer.sh
View File

@@ -47,7 +47,7 @@ shift $shiftcnt
if [ $installdir != "/usr/local/mariadb/columnstore" ]; then
export COLUMNSTORE_INSTALL_DIR=$installdir
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
else
export COLUMNSTORE_INSTALL_DIR=$installdir
fi

11
oam/install_scripts/post-install
View File

@@ -104,6 +104,9 @@ if [ $installdir != "/usr/local/mariadb/columnstore" ]; then
fi
if [ $user != "root" ]; then
sudo rm -f $profileFileEnv
sudo rm -f $profileFileAlias
sudo touch $profileFileEnv
sudo chmod 666 $profileFileEnv
egrep -qs 'MariaDB Columnstore Non-Root' ${profileFileEnv}
@@ -112,7 +115,7 @@ if [ $user != "root" ]; then
sudo echo " " >> ${profileFileEnv}
sudo echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${profileFileEnv}
sudo echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${profileFileEnv}
sudo echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${profileFileEnv}
sudo echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib" >> ${profileFileEnv}
. ${profileFileEnv}
fi
@@ -183,13 +186,13 @@ else
RCFILE=/etc/rc.local
fi
touch $RCFILE
if [ $user = "root" ]; then
touch $RCFILE
chmod +x $RCFILE
else
$SUDO touch $RCFILE
$SUDO chmod 777 $RCFILE
printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILEl > /dev/null 2>&1
$SUDO printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILE > /dev/null 2>&1
if [ -n "$systemctl" ]; then
$SUDO systemctl start rc-local >/dev/null 2>&1

2
oam/install_scripts/post-mysql-install
View File

@@ -64,7 +64,7 @@ USER=`whoami 2>/dev/null`
if [ $USER != "root" ]; then
sudo ldconfig >/dev/null 2>&1
export COLUMNSTORE_INSTALL_DIR=$installdir
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
else
ldconfig
fi

99
oam/oamcpp/liboamcpp.cpp
View File

@@ -9486,6 +9486,105 @@ std::string Oam::updateFstab(std::string device, std::string dbrootID)
return entry;
}
/******************************************************************************************
* @brief waitForActive
*
* purpose: wait for system to be active
*
******************************************************************************************/
void Oam::waitForActive()
{
SystemStatus systemstatus;
SystemProcessStatus systemprocessstatus;
bool bfirst = true;
int dot = 0;
for (int i = 0 ; i < 120 ; i ++, dot ++)
{
sleep (3);
try
{
getSystemStatus(systemstatus);
if (systemstatus.SystemOpState == ACTIVE)
{
BRM::DBRM dbrm;
try
{
int rc = dbrm.getSystemQueryReady();
if (rc == -1 )
{
writeLog("waitForActive: getSystemQueryReady error return: startSystem failed", LOG_TYPE_ERROR);
exceptionControl("waitForActive", API_FAILURE);
}
if ( rc != 0 )
return;
writeLog("waitForActive: getSystemQueryReady not ready", LOG_TYPE_DEBUG);
}
catch (...)
{}
}
if (systemstatus.SystemOpState == FAILED)
{
exceptionControl("waitForActive", API_FAILURE);
}
if (systemstatus.SystemOpState == MAN_OFFLINE)
{
exceptionControl("waitForActive", API_FAILURE);
}
if (dot >= 3 )
{
cout << "." << flush;
dot = 0;
}
// Check DMLProc for a switch to BUSY_INIT.
// In such a case, we need to print a message that rollbacks
// are occurring and will take some time.
if (bfirst) // Once we've printed our message, no need to waste cpu looking
{
getProcessStatus(systemprocessstatus);
for (unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++)
{
if (systemprocessstatus.processstatus[i].ProcessName == "DMLProc")
{
if (systemprocessstatus.processstatus[i].ProcessOpState == oam::ROLLBACK_INIT)
{
cout << endl << endl << " System Not Ready, DMLProc is checking/processing rollback of abandoned transactions. Processing could take some time, please wait..." << flush;
bfirst = false;
}
// At this point, we've found our DMLProc, so there's no need to spin the for loop
// any further.
break;
}
}
}
}
catch (...)
{
// At some point, we need to give up, ProcMon just isn't going to respond.
if (i > 60) // 3 minutes
{
cout << endl << endl << "TIMEOUT: ProcMon not responding to getSystemStatus";
break;
}
}
}
exceptionControl("waitForActive", API_FAILURE);
}
/***************************************************************************
* PRIVATE FUNCTIONS
***************************************************************************/

4
oam/oamcpp/liboamcpp.h
View File

@@ -2481,6 +2481,10 @@ public:
bool checkSystemRunning();
/** @brief wait for system to be active
*/
EXPORT void waitForActive();
private:
int sendMsgToProcMgr3(messageqcpp::ByteStream::byte requestType, alarmmanager::AlarmList& alarmlist, const std::string date);

2
oamapps/calpontSupport/findStranded.sh
View File

@@ -18,7 +18,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
fi
cd $COLUMNSTORE_INSTALL_DIR

2
oamapps/calpontSupport/sqlLogs.sh
View File

@@ -14,7 +14,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
fi

2
oamapps/columnstoreSupport/findStranded.sh
View File

@@ -18,7 +18,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
fi
cd $COLUMNSTORE_INSTALL_DIR

2
oamapps/columnstoreSupport/sqlLogs.sh
View File

@@ -14,7 +14,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib
fi

59
oamapps/mcsadmin/mcsadmin.cpp
View File

@@ -77,69 +77,14 @@ bool SendToWES(Oam& oam, ByteStream bs);
bool waitForActive()
{
Oam oam;
SystemStatus systemstatus;
SystemProcessStatus systemprocessstatus;
bool bfirst = true;
for (int i = 0 ; i < 1200 ; i ++)
{
sleep (3);
try
{
oam.getSystemStatus(systemstatus);
if (systemstatus.SystemOpState == ACTIVE)
{
oam.waitForActive();
return true;
}
if (systemstatus.SystemOpState == FAILED)
{
return false;
}
if (systemstatus.SystemOpState == MAN_OFFLINE)
{
return false;
}
cout << "." << flush;
// Check DMLProc for a switch to BUSY_INIT.
// In such a case, we need to print a message that rollbacks
// are occurring and will take some time.
if (bfirst) // Once we've printed our message, no need to waste cpu looking
{
oam.getProcessStatus(systemprocessstatus);
for (unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++)
{
if (systemprocessstatus.processstatus[i].ProcessName == "DMLProc")
{
if (systemprocessstatus.processstatus[i].ProcessOpState == oam::ROLLBACK_INIT)
{
cout << endl << endl << " System Not Ready, DMLProc is checking/processing rollback of abandoned transactions. Processing could take some time, please wait..." << flush;
bfirst = false;
}
// At this point, we've found our DMLProc, so there's no need to spin the for loop
// any further.
break;
}
}
}
}
catch (...)
{
// At some point, we need to give up, ProcMgr just isn't going to respond.
if (i > 60) // 3 minutes
{
cout << "ProcMgr not responding while waiting for system to start";
break;
}
}
}
{}
return false;
}

18
oamapps/postConfigure/helpers.cpp
View File

@@ -82,26 +82,18 @@ bool waitForActive()
{
Oam oam;
const string cmd = installDir + "/bin/mcsadmin getsystemstatus > /tmp/wait.log";
system(cmd.c_str());
for ( int i = 0 ; i < 120 ; i ++ )
try
{
if (oam.checkLogStatus("/tmp/wait.log", "System ACTIVE") )
oam.waitForActive();
return true;
if ( oam.checkLogStatus("/tmp/wait.log", "System FAILED") )
return false;
cout << ".";
cout.flush();
sleep (10);
system(cmd.c_str());
}
catch (...)
{}
return false;
}
void dbrmDirCheck()
{

7
primitives/linux-port/column.cpp
View File

@@ -39,6 +39,7 @@ using namespace boost;
#include "we_type.h"
#include "stats.h"
#include "primproc.h"
#include "dataconvert.h"
using namespace logging;
using namespace dbbc;
using namespace primitives;
@@ -603,7 +604,13 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int
type == CalpontSystemCatalog::TEXT) && !isNull )
{
if (!regex.used && !rf)
{
// MCOL-1246 Trim trailing whitespace for matching, but not for
// regex
dataconvert::DataConvert::trimWhitespace(val1);
dataconvert::DataConvert::trimWhitespace(val2);
return colCompare_(order_swap(val1), order_swap(val2), COP);
}
else
return colStrCompare_(order_swap(val1), order_swap(val2), COP, rf, &regex);
}

12
primitives/linux-port/dictionary.cpp
View File

@@ -21,6 +21,7 @@
#include <iostream>
#include <boost/scoped_array.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <sys/types.h>
using namespace std;
@@ -181,7 +182,10 @@ void PrimitiveProcessor::p_TokenByScan(const TokenByScanRequestHeader* h,
if (eqFilter)
{
bool gotIt = eqFilter->find(string(sig, siglen)) != eqFilter->end();
// MCOL-1246 Trim whitespace before match
string strData(sig, siglen);
boost::trim_right_if(strData, boost::is_any_of(" "));
bool gotIt = eqFilter->find(strData) != eqFilter->end();
if ((h->COP1 == COMPARE_EQ && gotIt) || (h->COP1 == COMPARE_NE &&
!gotIt))
@@ -902,8 +906,10 @@ void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector<uint8_t>* out,
if (eqFilter)
{
bool gotIt = (eqFilter->find(string((char*) sigptr.data, sigptr.len))
!= eqFilter->end());
// MCOL-1246 Trim whitespace before match
string strData((char*)sigptr.data, sigptr.len);
boost::trim_right_if(strData, boost::is_any_of(" "));
bool gotIt = eqFilter->find(strData) != eqFilter->end();
if ((gotIt && eqOp == COMPARE_EQ) || (!gotIt && eqOp == COMPARE_NE))
goto store;

7
procmgr/main.cpp
View File

@@ -594,7 +594,10 @@ static void alarmMessageThread(Configuration config)
msg = fIos.read();
if (msg.length() <= 0)
{
fIos.close();
continue;
}
//log.writeLog(__LINE__, "MSG RECEIVED: Process Alarm Message");
@@ -626,16 +629,20 @@ static void alarmMessageThread(Configuration config)
ALARMManager aManager;
aManager.processAlarmReport(calAlarm);
fIos.close();
}
catch (exception& ex)
{
string error = ex.what();
log.writeLog(__LINE__, "EXCEPTION ERROR on read for ProcMgr_Alarm:" + error, LOG_TYPE_ERROR);
fIos.close();
continue;
}
catch (...)
{
log.writeLog(__LINE__, "EXCEPTION ERROR on read for ProcMgr_Alarm: Caught unknown exception!", LOG_TYPE_ERROR);
fIos.close();
continue;
}
}

2
tools/setConfig/configxml.sh
View File

@@ -18,7 +18,7 @@ InstallDir=$COLUMNSTORE_INSTALL_DIR
if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then
export PATH=$InstallDir/bin:$InstallDir/mysql/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$InstallDir/lib:$InstallDir/mysql/lib/mysql
export LD_LIBRARY_PATH=$InstallDir/lib:$InstallDir/mysql/lib
fi
case "$1" in

68
utils/clusterTester/columnstoreClusterTester.sh
View File

@@ -494,7 +494,7 @@ checkSELINUX()
# SELINUX check
#
echo ""
echo "** Run SELINUX check - Setting should to be disabled on all nodes"
echo "** Run SELINUX check"
echo ""
pass=true
@@ -502,9 +502,8 @@ checkSELINUX()
if [ -f /etc/selinux/config ]; then
`cat /etc/selinux/config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node SELINUX setting is Enabled, please disable"
echo "${bold}Warning${normal}, Local Node SELINUX setting is Enabled, check port test results"
pass=false
REPORTPASS=false
else
echo "Local Node SELINUX setting is Not Enabled"
fi
@@ -519,19 +518,14 @@ checkSELINUX()
else
`cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd SELINUX setting is Enabled, please disable"
echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results"
pass=false
REPORTPASS=false
else
echo "$ipadd Node SELINUX setting is Not Enabled"
fi
`rm -f config`
fi
done
if ! $pass; then
checkContinue
fi
}
checkFirewalls()
@@ -539,28 +533,23 @@ checkFirewalls()
# FIREWALL checks
#
echo ""
echo "** Run Firewall Services check - Firewall Services should to be Inactive on all nodes"
echo "** Run Firewall Services check"
echo ""
declare -a FIREWALL_LIST=("iptables" "ufw" "firewalld" "firewall")
fpass=true
#check local FIREWALLS
for firewall in "${FIREWALL_LIST[@]}"; do
pass=true
`service $firewall status > /tmp/firewall1_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node $firewall service is Active, please disable"
echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results"
pass=false
fpass=false
REPORTPASS=false
else
`systemctl status $firewall > /tmp/firewall1_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node $firewall service is Active, please disable"
echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results"
pass=false
fpass=false
REPORTPASS=false
fi
fi
@@ -569,10 +558,6 @@ checkFirewalls()
fi
done
if ! $fpass; then
checkContinue
fi
echo ""
fpass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
@@ -581,17 +566,13 @@ checkFirewalls()
pass=true
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable"
echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results"
pass=false
fpass=false
REPORTPASS=false
else
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable"
echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results"
pass=false
fpass=false
REPORTPASS=false
fi
fi
@@ -603,24 +584,19 @@ checkFirewalls()
echo ""
done
if ! $fpass; then
checkContinue
fi
if [ $OS == "suse12" ]; then
# rcSuSEfirewall2 check
#
echo ""
echo "** Run rcSuSEfirewall2 check - Service should to be disabled on all nodes"
echo "** Run rcSuSEfirewall2 check"
echo ""
pass=true
#check local IPTABLES
`/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, Local Node rcSuSEfirewall2 service is Enabled, please disable"
echo "${bold}Failed${normal}, Local Node rcSuSEfirewall2 service is Enabled, check port test results"
pass=false
REPORTPASS=false
else
echo "Local Node rcSuSEfirewall2 service is Not Enabled"
fi
@@ -629,17 +605,12 @@ checkFirewalls()
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check`
rc="$?"
if [ $rc -eq 0 ] ; then
echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, please disable"
echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, check port test results"
pass=false
REPORTPASS=false
else
echo "$ipadd Node rcSuSEfirewall2 service is Not Enabled"
fi
done
if ! $pass; then
checkContinue
fi
fi
}
@@ -648,17 +619,18 @@ checkPorts()
# port test
#
echo ""
echo "** Run MariaDB ColumnStore Port (8600-8620) availibility test"
echo "** Run MariaDB ColumnStore Port (8600-8630,8700,8800,3306) availability test"
echo ""
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`nmap $ipadd -p 8600-8620 | grep 'closed unknown' > /dev/null`
if [ "$?" -eq 0 ]; then
`sudo nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > /tmp/port_test`
if [ "$?" -ne 0 ]; then
echo $ipadd " Node Passed port test"
else
echo $ipadd " Node ${bold}Failed${normal} port test, check and disable any firwalls that were reported enabled"
echo $ipadd " Node ${bold}Failed${normal} port test, check and disable any firewalls or open ports in firewall"
cat /tmp/port_test
pass=false
REPORTPASS=false
fi
@@ -764,7 +736,7 @@ checkPackages()
echo "** Run MariaDB ColumnStore Dependent Package Check"
echo ""
declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "perl-DBD-MySQL")
declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools")
declare -a CENTOS_PKG_NOT=("mariadb-libs")
if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then
@@ -883,7 +855,7 @@ checkPackages()
fi
fi
declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "perl-DBD-mysql")
declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools")
declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18")
if [ "$OS" == "suse12" ]; then
@@ -974,7 +946,7 @@ checkPackages()
fi
fi
declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libdbd-mysql-perl")
declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools")
declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18")
if [ "$OS" == "ubuntu16" ] ; then
@@ -1091,7 +1063,7 @@ checkPackages()
fi
fi
declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libdbd-mysql-perl")
declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools")
declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server")
if [ "$OS" == "debian8" ]; then

14
utils/dataconvert/dataconvert.h
View File

@@ -424,6 +424,7 @@ public:
static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType);
static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType);
static inline std::string constructRegexp(const std::string& str);
static inline void trimWhitespace(int64_t& charData);
static inline bool isEscapedChar(char c)
{
return ('%' == c || '_' == c);
@@ -578,6 +579,19 @@ inline void DataConvert::decimalToString(int64_t int_val, uint8_t scale, char* b
*(ptr + l1) = '.';
}
inline void DataConvert::trimWhitespace(int64_t& charData)
{
// Trims whitespace characters off non-dict character data
char* ch_data = (char*) &charData;
for (int8_t i = 7; i > 0; i--)
{
if (ch_data[i] == ' ' || ch_data[i] == '\0')
ch_data[i] = '\0';
else
break;
}
}
//FIXME: copy/pasted from dictionary.cpp: refactor
inline std::string DataConvert::constructRegexp(const std::string& str)

8
utils/funcexp/func_case.cpp
View File

@@ -489,6 +489,14 @@ bool Func_searched_case::getBoolVal(Row& row,
if (isNull)
return joblist::BIGINTNULL;
ParseTree* lop = parm[i + 1]->left();
ParseTree* rop = parm[i + 1]->right();
if (lop && rop)
{
return (reinterpret_cast<Operator*>(parm[i + 1]->data()))->getBoolVal(row, isNull, lop, rop);
}
return parm[i + 1]->data()->getBoolVal(row, isNull);
}