database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-10 01:22:48 +03:00
Code Activity

Merge pull request #873 from mariadb-corporation/develop-merge-up-20190924-2

Browse Source 
Merge develop-1.2 into develop
This commit is contained in:
David.Hall
2019-09-24 09:58:16 -05:00
committed by GitHub
parent 66d59a760d 4e92e75274
commit d4e6a282ab
25 changed files with 289 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dbcon/ddlpackage/ddl.l
View File

@@ -65,7 +65,7 @@ double_quote \"
grave_accent `
comment ("--"{non_newline}*)
extended_ident_cont [A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\]
extended_ident_cont [ A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\]
self [,()\[\].;\:\+\-\*\/\%\^\<\>\=]
whitespace ({space}+|{comment})
@@ -75,10 +75,8 @@ ident_cont [A-Za-z\200-\377_0-9\$]
identifier {ident_start}{ident_cont}*
extended_identifier {ident_start}{extended_ident_cont}*
/* fully qualified names regexes */
ident_w_spaces {identifier}\x20*
identifier_quoted {grave_accent}{extended_identifier}{grave_accent}
identifier_double_quoted {double_quote}{extended_identifier}{double_quote}
column_ident_quoted {grave_accent}{ident_w_spaces}+{grave_accent}
integer [-+]?{digit}+
decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*))
@@ -192,8 +190,6 @@ MEDIUMINT {return MEDIUMINT;}
\n { lineno++;}
{column_ident_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES); return IDENT;}
{whitespace} {
/* ignore */
}

14
dbcon/execplan/constantcolumn.cpp
View File

@@ -206,11 +206,11 @@ ConstantColumn::ConstantColumn( const ConstantColumn& rhs):
if (fRegex.get() != NULL)
{
fRegex.reset(new CNX_Regex());
#ifdef _MSC_VER
*fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal);
#else
#ifdef POSIX_REGEX
string str = dataconvert::DataConvert::constructRegexp(fResult.strVal);
regcomp(fRegex.get(), str.c_str(), REG_NOSUB | REG_EXTENDED);
#else
*fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal);
#endif
}
}
@@ -257,7 +257,7 @@ ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) :
ConstantColumn::~ConstantColumn()
{
#ifndef _MSC_VER
#ifdef POSIX_REGEX
if (fRegex.get() != NULL)
regfree(fRegex.get());
@@ -400,11 +400,11 @@ void ConstantColumn::constructRegex()
{
//fRegex = new regex_t();
fRegex.reset(new CNX_Regex());
#ifdef _MSC_VER
*fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal);
#else
#ifdef POSIX_REGEX
string str = dataconvert::DataConvert::constructRegexp(fResult.strVal);
regcomp(fRegex.get(), str.c_str(), REG_NOSUB | REG_EXTENDED);
#else
*fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal);
#endif
}

63
dbcon/execplan/predicateoperator.h
View File

@@ -35,6 +35,7 @@
#include <alloca.h>
#endif
#include <cstring>
#include <cmath>
#include <boost/regex.hpp>
#include "expressionparser.h"
@@ -113,6 +114,8 @@ private:
template <typename result_t>
inline bool numericCompare(result_t op1, result_t op2);
inline bool strCompare(const std::string& op1, const std::string& op2);
// MCOL-1559
inline bool strTrimCompare(const std::string& op1, const std::string& op2);
};
inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop)
@@ -126,31 +129,31 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu
// considers these nulls significant, but they're not in the pattern, so we need to strip
// them off...
const std::string& v = lop->getStrVal(row, isNull);
char* c = (char*)alloca(v.length() + 1);
memcpy(c, v.c_str(), v.length());
c[v.length()] = 0;
std::string vv(c);
// char* c = (char*)alloca(v.length() + 1);
// memcpy(c, v.c_str(), v.length());
// c[v.length()] = 0;
// std::string vv(c);
if (regex)
{
#ifdef _MSC_VER
bool ret = boost::regex_match(vv, *regex);
#ifdef POSIX_REGEX
bool ret = regexec(regex.get(), v.c_str(), 0, NULL, 0) == 0;
#else
bool ret = regexec(regex.get(), vv.c_str(), 0, NULL, 0) == 0;
bool ret = boost::regex_match(v.c_str(), *regex);
#endif
return (((fOp == OP_LIKE) ? ret : !ret) && !isNull);
}
else
{
#ifdef _MSC_VER
boost::regex regex(dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull)));
bool ret = boost::regex_match(vv, regex);
#else
#ifdef POSIX_REGEX
regex_t regex;
std::string str = dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull));
regcomp(&regex, str.c_str(), REG_NOSUB | REG_EXTENDED);
bool ret = regexec(&regex, vv.c_str(), 0, NULL, 0) == 0;
bool ret = regexec(&regex, v.c_str(), 0, NULL, 0) == 0;
regfree(&regex);
#else
boost::regex regex(dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull)));
bool ret = boost::regex_match(v.c_str(), regex);
#endif
return (((fOp == OP_LIKE) ? ret : !ret) && !isNull);
}
@@ -489,11 +492,12 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu
return false;
const std::string& val1 = lop->getStrVal(row, isNull);
if (isNull)
return false;
return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull;
return strTrimCompare(val1, rop->getStrVal(row, isNull)) && !isNull;
// return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull;
}
//FIXME: ???
@@ -577,6 +581,37 @@ inline bool PredicateOperator::strCompare(const std::string& op1, const std::str
}
}
inline bool PredicateOperator::strTrimCompare(const std::string& op1, const std::string& op2)
{
switch (fOp)
{
case OP_EQ:
return funcexp::utf8::idb_strtrimcoll(op1, op2) == 0;
case OP_NE:
return funcexp::utf8::idb_strtrimcoll(op1, op2) != 0;
case OP_GT:
return funcexp::utf8::idb_strtrimcoll(op1, op2) > 0;
case OP_GE:
return funcexp::utf8::idb_strtrimcoll(op1, op2) >= 0;
case OP_LT:
return funcexp::utf8::idb_strtrimcoll(op1, op2) < 0;
case OP_LE:
return funcexp::utf8::idb_strtrimcoll(op1, op2) <= 0;
default:
{
std::ostringstream oss;
oss << "Non support predicate operation: " << fOp;
throw logging::InvalidOperationExcept(oss.str());
}
}
}
std::ostream& operator<<(std::ostream& os, const PredicateOperator& rhs);
}

6
dbcon/execplan/treenode.h
View File

@@ -164,10 +164,10 @@ typedef IDB_Decimal CNX_Decimal;
* @brief IDB_Regex struct
*
*/
#ifdef _MSC_VER
typedef boost::regex IDB_Regex;
#else
#ifdef POSIX_REGEX
typedef regex_t IDB_Regex;
#else
typedef boost::regex IDB_Regex;
#endif
typedef IDB_Regex CNX_Regex;

15
dbcon/joblist/jlf_execplantojoblist.cpp
View File

@@ -1644,7 +1644,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
string constval(cc->constval());
CalpontSystemCatalog::OID dictOid = 0;
CalpontSystemCatalog::ColType ct = sc->colType();
const PseudoColumn* pc = dynamic_cast<const PseudoColumn*>(sc);
@@ -1653,8 +1652,13 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
// type of pseudo column is set by connector
if (!sc->schemaName().empty() && sc->isColumnStore() && !pc)
ct = jobInfo.csc->colType(sc->oid());
//X
// Because, on a filter, we want to compare ignoring trailing spaces in many cases
if (sf->op()->op() != execplan::OP_LIKE)
{
boost::algorithm::trim_right_if(constval, boost::is_any_of(" "));
}
//@bug 339 nulls are not stored in dictionary
if ((dictOid = isDictCol(ct)) > 0 && ConstantColumn::NULLDATA != cc->type())
{
@@ -2778,7 +2782,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
cop = COMPARE_NIL;
string value = cc->constval();
// Because, on a filter, we want to compare ignoring trailing spaces
boost::algorithm::trim_right_if(value, boost::is_any_of(" "));
pds->addFilter(cop, value);
}
@@ -2861,7 +2866,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
cop = COMPARE_NIL;
string value = cc->constval();
// Because, on a filter, we want to compare ignoring trailing spaces
boost::algorithm::trim_right_if(value, boost::is_any_of(" "));
pds->addFilter(cop, value);
}
@@ -2968,7 +2974,6 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
int64_t value = 0;
string constval = cc->constval();
// @bug 1151 string longer than colwidth of char/varchar.
uint8_t rf = 0;
bool isNull = ConstantColumn::NULLDATA == cc->type();

20
dbcon/joblist/joblistfactory.cpp
View File

@@ -400,6 +400,7 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
void preProcessFunctionOnAggregation(const vector<SimpleColumn*>& scs,
const vector<AggregateColumn*>& aggs,
const vector<WindowFunctionColumn*>& wcs,
JobInfo& jobInfo)
{
// append the simple columns if not already projected
@@ -433,6 +434,10 @@ void preProcessFunctionOnAggregation(const vector<SimpleColumn*>& scs,
for (vector<AggregateColumn*>::const_iterator i = aggs.begin(); i != aggs.end(); i++)
{
addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo);
if (wcs.size() > 0)
{
jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone()));
}
}
}
@@ -484,12 +489,12 @@ void checkReturnedColumns(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
if (ac != NULL && ac->aggColumnList().size() > 0)
{
jobInfo.nonConstCols[i]->outputIndex(i);
preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), jobInfo);
preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), ac->windowfunctionColumnList(), jobInfo);
}
else if (fc != NULL && fc->aggColumnList().size() > 0)
{
jobInfo.nonConstCols[i]->outputIndex(i);
preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), jobInfo);
preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), fc->windowfunctionColumnList(), jobInfo);
}
}
}
@@ -600,7 +605,8 @@ void checkAggregation(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
jobInfo.hasDistinct = csep->distinct();
if (csep->distinct() == true)
// DISTINCT with window functions must be done in tupleannexstep
if (csep->distinct() == true && jobInfo.windowDels.size() == 0)
{
jobInfo.hasAggregation = true;
}
@@ -874,6 +880,10 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo
const SimpleColumn* sc = dynamic_cast<const SimpleColumn*>(srcp.get());
AggregateColumn* aggc = dynamic_cast<AggregateColumn*>(srcp.get());
bool doDistinct = (csep->distinct() && csep->groupByCols().empty());
// Use this instead of the above line to mimic MariaDB's sql_mode = 'ONLY_FULL_GROUP_BY'
// bool doDistinct = (csep->distinct() &&
// csep->groupByCols().empty() &&
// !jobInfo.hasAggregation);
uint32_t tupleKey = -1;
string alias;
string view;
@@ -1122,7 +1132,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo
// remember the columns to be returned
jobInfo.returnedColVec.push_back(make_pair(tupleKey, op));
// bug 1499 distinct processing, save unique distinct columns
// bug 1499 distinct processing, save unique distinct columns
if (doDistinct &&
(jobInfo.distinctColVec.end() ==
find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), tupleKey)))
@@ -1521,7 +1531,7 @@ void parseExecutionPlan(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo,
// bug4531, window function support
WindowFunctionStep::checkWindowFunction(csep, jobInfo);
// bug3391, move forward the aggregation check for no aggregte having clause.
// bug3391, move forward the aggregation check for no aggregate having clause.
checkAggregation(csep, jobInfo);
// include filters in having clause, if any.

179
dbcon/joblist/tupleaggregatestep.cpp
View File

@@ -840,6 +840,7 @@ const string TupleAggregateStep::toString() const
SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo)
{
SJSTEP spjs;
TupleDeliveryStep* tds = dynamic_cast<TupleDeliveryStep*>(step.get());
TupleBPS* tbps = dynamic_cast<TupleBPS*>(step.get());
TupleHashJoinStep* thjs = dynamic_cast<TupleHashJoinStep*>(step.get());
SubAdapterStep* sas = dynamic_cast<SubAdapterStep*>(step.get());
@@ -917,171 +918,83 @@ SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo)
// preprocess the columns used by group_concat
jobInfo.groupConcatInfo.prepGroupConcat(jobInfo);
bool doGroupConcat = false;
bool doUMOnly = jobInfo.groupConcatInfo.columns().size() > 0
// || jobInfo.windowSet.size() > 0
|| sas
|| ces;
rgs.push_back(tds->getDeliveredRowGroup());
// get rowgroup and aggregator
// For TupleHashJoin, we prepare for both PM and UM only aggregation
if (doUMOnly || thjs)
{
if (distinctAgg == true)
prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
else
prep1PhaseAggregate(jobInfo, rgs, aggs);
// TODO: fix this
if (doUMOnly)
rgs.push_back(rgs[0]);
}
if (!doUMOnly)
{
if (distinctAgg == true)
prep2PhasesDistinctAggregate(jobInfo, rgs, aggs);
else
prep2PhasesAggregate(jobInfo, rgs, aggs);
}
if (tbps != NULL)
{
// get rowgroup and aggregator
rgs.push_back(tbps->getDeliveredRowGroup());
if (jobInfo.groupConcatInfo.columns().size() == 0)
{
if (distinctAgg == true)
prep2PhasesDistinctAggregate(jobInfo, rgs, aggs);
else
prep2PhasesAggregate(jobInfo, rgs, aggs);
}
else
{
if (distinctAgg == true)
prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
else
prep1PhaseAggregate(jobInfo, rgs, aggs);
// TODO: fix this
rgs.push_back(rgs[0]);
doGroupConcat = true;
}
// make sure connected by a RowGroupDL
JobStepAssociation tbpsJsa;
AnyDataListSPtr spdl(new AnyDataList());
RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
dl->OID(execplan::CNX_VTABLE_ID);
spdl->rowGroupDL(dl);
tbpsJsa.outAdd(spdl);
// create delivery step
aggUM = dynamic_pointer_cast<RowAggregationUM>(aggs[0]);
spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[2], jobInfo));
spjs->inputAssociation(tbpsJsa);
// step id??
spjs->stepId(step->stepId() + 1);
// set the PM/UM side aggregate structs
tbps->outputAssociation(tbpsJsa);
if (doGroupConcat)
if (doUMOnly)
dynamic_cast<TupleAggregateStep*>(spjs.get())->umOnly(true);
else
tbps->setAggregateStep(aggs[1], rgs[2]);
}
else if (thjs != NULL)
{
// default to UM aggregation
rgs.push_back(thjs->getDeliveredRowGroup());
if (distinctAgg == true)
prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
else
prep1PhaseAggregate(jobInfo, rgs, aggs);
// also prepare for PM aggregation
// rowgroups -- 0-proj, 1-um, [2-phase case: 2-um, 3-pm]
// aggregators -- 0-um, [2-phase case: 1-um, 2-pm]
if (jobInfo.groupConcatInfo.columns().size() == 0)
{
if (distinctAgg == true)
prep2PhasesDistinctAggregate(jobInfo, rgs, aggs);
else
prep2PhasesAggregate(jobInfo, rgs, aggs);
}
else
{
// TODO: fix this
rgs.push_back(rgs[0]);
doGroupConcat = true;
}
// make sure connected by a RowGroupDL
JobStepAssociation thjsJsa;
AnyDataListSPtr spdl(new AnyDataList());
RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
dl->OID(execplan::CNX_VTABLE_ID);
spdl->rowGroupDL(dl);
thjsJsa.outAdd(spdl);
// create delivery step
aggUM = dynamic_pointer_cast<RowAggregationUM>(aggs[0]);
spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo));
spjs->inputAssociation(thjsJsa);
if (doGroupConcat)
if (doUMOnly)
dynamic_cast<TupleAggregateStep*>(spjs.get())->umOnly(true);
else
dynamic_cast<TupleAggregateStep*>(spjs.get())->savePmHJData(aggs[1], aggs[2], rgs[3]);
// step id??
spjs->stepId(step->stepId() + 1);
// set input side
thjs->outputAssociation(thjsJsa);
thjs->deliveryStep(spjs);
}
else if (sas != NULL)
else
{
// UM aggregation
// rowgroups -- 0-proj, 1-um
// aggregators -- 0-um
rgs.push_back(sas->getDeliveredRowGroup());
if (distinctAgg == true)
prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
else
prep1PhaseAggregate(jobInfo, rgs, aggs);
// make sure connected by a RowGroupDL
JobStepAssociation sasJsa;
AnyDataListSPtr spdl(new AnyDataList());
RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
dl->OID(execplan::CNX_VTABLE_ID);
spdl->rowGroupDL(dl);
sasJsa.outAdd(spdl);
// create delivery step
aggUM = dynamic_pointer_cast<RowAggregationUM>(aggs[0]);
spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo));
spjs->inputAssociation(sasJsa);
// step id??
spjs->stepId(step->stepId() + 1);
// set input side
sas->outputAssociation(sasJsa);
}
else if (ces != NULL)
{
// UM aggregation
// rowgroups -- 0-proj, 1-um
// aggregators -- 0-um
rgs.push_back(ces->getDeliveredRowGroup());
if (distinctAgg == true)
prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
else
prep1PhaseAggregate(jobInfo, rgs, aggs);
// Setup the input JobstepAssoctiation -- the mechanism
// whereby the previous step feeds data to this step.
// Otherwise, we need to create one and hook to the
// previous step as well as this aggregate step.
spjs->stepId(step->stepId() + 1);
// make sure connected by a RowGroupDL
JobStepAssociation cesJsa;
AnyDataListSPtr spdl(new AnyDataList());
RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
dl->OID(execplan::CNX_VTABLE_ID);
spdl->rowGroupDL(dl);
cesJsa.outAdd(spdl);
JobStepAssociation jsa;
AnyDataListSPtr spdl(new AnyDataList());
RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
dl->OID(execplan::CNX_VTABLE_ID);
spdl->rowGroupDL(dl);
jsa.outAdd(spdl);
// create delivery step
aggUM = dynamic_pointer_cast<RowAggregationUM>(aggs[0]);
spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo));
spjs->inputAssociation(cesJsa);
spjs->inputAssociation(jsa); // Aggregate input
// step id??
spjs->stepId(step->stepId() + 1);
// set input side
ces->outputAssociation(cesJsa);
}
//Previous step output
step->outputAssociation(jsa);
// add the aggregate on constants
if (constAggDataVec.size() > 0)

39
dbcon/joblist/windowfunctionstep.cpp
View File

@@ -477,6 +477,17 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J
colSet.insert(key);
}
// MCOL-3435 We haven't yet checked for aggregate, but we need to know
bool hasAggregation = false;
for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++)
{
if (dynamic_cast<AggregateColumn*>(jobInfo.deliveredCols[i].get()) != NULL)
{
hasAggregation = true;
break;
}
}
// add non-duplicate auxiliary columns
RetColsVector colsInAf;
@@ -499,10 +510,30 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J
if (colSet.find(key) == colSet.end())
{
jobInfo.deliveredCols.push_back(*j);
// MCOL-3343 Enable this if we decide to allow Window Functions to run with
// aggregates with no group by. MariaDB allows this. Nobody else in the world does.
// There will be more work to get it to function if we try this.
// jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true));
// MCOL-3435 Allow Window Functions to run with aggregates with
// no group by by inserting a group by for window parameters.
if (hasAggregation)
{
// If an argument is an AggregateColumn, don't group by it.
if (dynamic_cast<AggregateColumn*>(j->get()) == NULL)
{
bool bFound = false;
for (std::vector<SRCP>::iterator igpc = csep->groupByCols().begin();
igpc < csep->groupByCols().end();
++igpc)
{
if (*igpc->get() == *j->get())
{
bFound = true;
break;
}
}
if (!bFound)
{
csep->groupByCols().push_back(*j);
}
}
}
}
colSet.insert(key);

2
dbcon/mysql/ha_calpont_execplan.cpp
View File

@@ -3160,7 +3160,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp
String val, *str = item->val_str(&val);
string valStr;
valStr.assign(str->ptr(), str->length());
rc = new ConstantColumn(valStr);
rc = new ConstantColumn(valStr, ConstantColumn::NUM);
break;
}
case REAL_RESULT:

6
primitives/linux-port/primitiveprocessor.h
View File

@@ -33,9 +33,9 @@
#include <unordered_set>
#endif
#ifdef __linux__
#define POSIX_REGEX
#endif
//#ifdef __linux__
//#define POSIX_REGEX
//#endif
#ifdef POSIX_REGEX
#include <regex.h>

3
procmon/processmonitor.cpp
View File

@@ -2621,6 +2621,7 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName,
// DBRMroot = tempDBRMDir + "/BRM_saves";
}
//
// run the 'load_brm' script first if files exist
//
@@ -4419,7 +4420,6 @@ int ProcessMonitor::getDBRMdata(string *path)
bf::create_directories(pTmp);
*path = pTmp.string();
log.writeLog(__LINE__, "Downloading DBRM files to " + *path, LOG_TYPE_DEBUG);
for ( int i = 0 ; i < numFiles ; i ++ )
{
string fileName;
@@ -6359,7 +6359,6 @@ int ProcessMonitor::checkDataMount()
/* StorageManager isn't running yet. Can't check for writability here. */
return API_SUCCESS;
}
//go unmount disk NOT assigned to this pm
unmountExtraDBroots();

10
utils/funcexp/func_bitwise.cpp
View File

@@ -94,10 +94,6 @@ bool getUIntValFromParm(
{
isNull = true;
}
else
{
value = 0;
}
}
break;
@@ -111,9 +107,9 @@ bool getUIntValFromParm(
{
d.value = 0;
}
int64_t tmpval = d.value / helpers::power(d.scale);
int lefto = (d.value - tmpval * helpers::power(d.scale)) / helpers::power(d.scale - 1);
double dscale = d.scale;
int64_t tmpval = d.value / pow(10.0, dscale);
int lefto = (d.value - tmpval * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( tmpval >= 0 && lefto > 4 )
tmpval++;

10
utils/funcexp/func_cast.cpp
View File

@@ -180,8 +180,9 @@ int64_t Func_cast_signed::getIntVal(Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
int64_t value = d.value / helpers::power(d.scale);
int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1);
double dscale = d.scale;
int64_t value = d.value / pow(10.0, dscale);
int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( value >= 0 && lefto > 4 )
value++;
@@ -337,14 +338,15 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
double dscale = d.scale;
if (d.value < 0)
{
return 0;
}
uint64_t value = d.value / helpers::power(d.scale);
int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1);
uint64_t value = d.value / pow(10.0, dscale);
int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( value >= 0 && lefto > 4 )
value++;

5
utils/funcexp/func_char.cpp
View File

@@ -156,9 +156,10 @@ string Func_char::getStrVal(Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
double dscale = d.scale;
// get decimal and round up
int value = d.value / helpers::power(d.scale);
int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1);
int value = d.value / pow(10.0, dscale);
int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( lefto > 4 )
value++;

11
utils/funcexp/func_div.cpp
View File

@@ -51,6 +51,16 @@ int64_t Func_div::getIntVal(rowgroup::Row& row,
{
double val1 = parm[0]->data()->getDoubleVal(row, isNull);
double val2 = parm[1]->data()->getDoubleVal(row, isNull);
if (val2 == 0 || val2 == NAN)
{
isNull = true;
return 0;
}
// MCOL-179 InnoDB doesn't round or convert to int before dividing.
return static_cast<int64_t>(val1 / val2);
#if 0
int64_t int_val2 = (int64_t)(val2 > 0 ? val2 + 0.5 : val2 - 0.5);
if (int_val2 == 0)
@@ -69,6 +79,7 @@ int64_t Func_div::getIntVal(rowgroup::Row& row,
}
return int_val1 / int_val2;
#endif
}

5
utils/funcexp/func_elt.cpp
View File

@@ -71,8 +71,9 @@ string Func_elt::getStrVal(rowgroup::Row& row,
case CalpontSystemCatalog::DECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
number = d.value / helpers::power(d.scale);
int lefto = (d.value - number * helpers::power(d.scale)) / helpers::power(d.scale - 1);
double dscale = d.scale;
number = d.value / pow(10.0, dscale);
int lefto = (d.value - number * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( number >= 0 && lefto > 4 )
number++;

10
utils/funcexp/func_makedate.cpp
View File

@@ -68,8 +68,9 @@ uint64_t makedate(rowgroup::Row& row,
case CalpontSystemCatalog::DECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
year = d.value / helpers::power(d.scale);
int lefto = (d.value - year * helpers::power(d.scale)) / helpers::power(d.scale - 1);
double dscale = d.scale;
year = d.value / pow(10.0, dscale);
int lefto = (d.value - year * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( year >= 0 && lefto > 4 )
year++;
@@ -127,8 +128,9 @@ uint64_t makedate(rowgroup::Row& row,
case CalpontSystemCatalog::DECIMAL:
{
IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull);
int64_t tmp = d.value / helpers::power(d.scale);
int lefto = (d.value - tmp * helpers::power(d.scale)) / helpers::power(d.scale - 1);
double dscale = d.scale;
int64_t tmp = d.value / pow(10.0, dscale);
int lefto = (d.value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( tmp >= 0 && lefto > 4 )
tmp++;

15
utils/funcexp/func_maketime.cpp
View File

@@ -74,8 +74,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row,
case CalpontSystemCatalog::DECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
hour = d.value / helpers::power(d.scale);
int lefto = (d.value - hour * helpers::power(d.scale)) / helpers::power(d.scale - 1);
double dscale = d.scale;
hour = d.value / pow(10.0, dscale);
int lefto = (d.value - hour * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( hour >= 0 && lefto > 4 )
hour++;
@@ -113,8 +114,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row,
case CalpontSystemCatalog::DECIMAL:
{
IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull);
min = d.value / helpers::power(d.scale);
int lefto = (d.value - min * helpers::power(d.scale)) / helpers::power(d.scale - 1);
double dscale = d.scale;
min = d.value / pow(10.0, dscale);
int lefto = (d.value - min * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( min >= 0 && lefto > 4 )
min++;
@@ -158,8 +160,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row,
case CalpontSystemCatalog::DECIMAL:
{
IDB_Decimal d = parm[2]->data()->getDecimalVal(row, isNull);
sec = d.value / helpers::power(d.scale);
int lefto = (d.value - sec * helpers::power(d.scale)) / helpers::power(d.scale - 1);
double dscale = d.scale;
sec = d.value / pow(10.0, dscale);
int lefto = (d.value - sec * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( sec >= 0 && lefto > 4 )
sec++;

14
utils/funcexp/func_mod.cpp
View File

@@ -74,10 +74,10 @@ IDB_Decimal Func_mod::getDecimalVal(Row& row,
}
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
int64_t value = d.value / helpers::power(d.scale);
int lefto = d.value % helpers::power(d.scale);
int64_t value = d.value / pow(10.0, d.scale);
int lefto = d.value % (int)pow(10.0, d.scale);
int64_t mod = (value % div) * helpers::power(d.scale) + lefto;
int64_t mod = (value % div) * pow(10.0, d.scale) + lefto;
retValue.value = mod;
retValue.scale = d.scale;
@@ -164,7 +164,7 @@ double Func_mod::getDoubleVal(Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
int64_t value = d.value / helpers::power(d.scale);
int64_t value = d.value / pow(10.0, d.scale);
mod = value % div;
}
@@ -268,7 +268,7 @@ long double Func_mod::getLongDoubleVal(Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
int64_t value = d.value / helpers::power(d.scale);
int64_t value = d.value / pow(10.0, d.scale);
mod = value % div;
}
@@ -375,7 +375,7 @@ int64_t Func_mod::getIntVal(Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
int64_t value = d.value / helpers::power(d.scale);
int64_t value = d.value / pow(10.0, d.scale);
mod = value % div;
}
@@ -473,7 +473,7 @@ uint64_t Func_mod::getUIntVal(Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
int64_t value = d.value / helpers::power(d.scale);
int64_t value = d.value / pow(10.0, d.scale);
mod = value % div;
}

4
utils/funcexp/func_period_diff.cpp
View File

@@ -86,7 +86,7 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
period1 = d.value / helpers::power(d.scale);
period1 = d.value / pow(10.0, d.scale);
break;
}
@@ -135,7 +135,7 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull);
period2 = d.value / helpers::power(d.scale);
period2 = d.value / pow(10.0, d.scale);
break;
}

3
utils/funcexp/funcexp.cpp
View File

@@ -46,6 +46,9 @@ namespace funcexp
{
namespace utf8
{
// A global loc object so we don't construct one at every compare
std::locale loc;
bool JPcodePoint = false; // extern-ed in utils_utf8.h
}

54
utils/funcexp/utils_utf8.h
View File

@@ -36,7 +36,6 @@
#include <cstdlib>
#include <clocale>
#include "liboamcpp.h"
/** @file */
@@ -49,6 +48,10 @@ extern bool JPcodePoint; // code point ordering (Japanese UTF) flag, used in id
const int MAX_UTF8_BYTES_PER_CHAR = 4;
// A global loc object so we don't construct one at every compare
extern std::locale loc;
// Is there a way to construct a global reference to a facet?
// const std::collate<char>& coll = std::use_facet<std::collate<char> >(loc);
//Infinidb version of strlocale BUG 5362
//set System Locale "C" by default
@@ -110,6 +113,10 @@ std::string idb_setlocale()
if (systemLang.find("ja_JP") != std::string::npos)
JPcodePoint = true;
// MCOL-1559 Save off the locale to save runtime cpus
std::locale localloc(systemLang.c_str());
loc = localloc;
return systemLang;
}
@@ -125,6 +132,51 @@ int idb_strcoll(const char* str1, const char* str2)
return strcoll(str1, str2);
}
// MCOL-1559 Add a trimmed version of strcoll
// The intent here is to make no copy of the original strings and
// not modify them, so we can't use trim to deal with the spaces.
inline
int idb_strtrimcoll(const std::string& str1, const std::string& str2)
{
static const std::string whitespaces (" ");
const char* s1 = str1.c_str();
const char* s2 = str2.c_str();
// Set found1 to the last non-whitespace char in str1
std::size_t found1 = str1.find_last_not_of(whitespaces);
// Set found2 to the first whitespace char in str2
std::size_t found2 = str2.find_last_not_of(whitespaces);
// Are both strings empty or all whitespace?
if (found1 == std::string::npos && found2 == std::string::npos)
{
return 0; // they match
}
// If str1 is empty or all spaces
if (found1 == std::string::npos)
{
return -1;
}
// If str2 is empty or all spaces
if (found2 == std::string::npos)
{
return 1;
}
// found1 and found2 point to the character that is not a space.
// compare wants it to point to one past.
found1 += 1;
found2 += 1;
// If no trimming needs doing, then strcoll is faster
if (found1 == str1.size() && found2 == str2.size())
{
return idb_strcoll(s1, s2);
}
// Compare the (trimmed) strings
const std::collate<char>& coll = std::use_facet<std::collate<char> >(loc);
int rtn = coll.compare(s1, s1+found1, s2, s2+found2);
return rtn;
}
// BUG 5241
// Infinidb specific mbstowcs(). This will handle both windows and unix platforms

5
utils/rowgroup/rowgroup.h
View File

@@ -1035,12 +1035,13 @@ inline void Row::setFloatField(float val, uint32_t colIndex)
inline void Row::setLongDoubleField(long double val, uint32_t colIndex)
{
uint8_t* p = &data[offsets[colIndex]];
*((long double*)p) = val;
if (sizeof(long double) == 16)
{
// zero out the unused portion as there may be garbage there.
*((uint64_t*)&val+1) &= 0x000000000000FFFFULL;
*((uint64_t*)p+1) &= 0x000000000000FFFFULL;
}
*((long double*) &data[offsets[colIndex]]) = val;
}
inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex)

7
utils/windowfunction/wf_stats.cpp
View File

@@ -140,6 +140,7 @@ void WF_stats<T>::resetData()
template<typename T>
void WF_stats<T>::operator()(int64_t b, int64_t e, int64_t c)
{
CDT cdt;
if ((fFrameUnit == WF__FRAME_ROWS) ||
(fPrev == -1) ||
(!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev)))))
@@ -163,7 +164,7 @@ void WF_stats<T>::operator()(int64_t b, int64_t e, int64_t c)
continue;
T valIn;
getValue(colIn, valIn);
getValue(colIn, valIn, &cdt);
long double val = (long double) valIn;
fSum1 += val;
@@ -177,7 +178,9 @@ void WF_stats<T>::operator()(int64_t b, int64_t e, int64_t c)
int scale = fRow.getScale(colIn);
long double factor = pow(10.0, scale);
if (scale != 0) // adjust the scale if necessary
// adjust the scale if necessary
if (scale != 0 &&
cdt != CalpontSystemCatalog::LONGDOUBLE)
{
fSum1 /= factor;
fSum2 /= factor * factor;

6
utils/windowfunction/wf_sum_avg.cpp
View File

@@ -264,13 +264,15 @@ void WF_sum_avg<T>::operator()(int64_t b, int64_t e, int64_t c)
continue;
T valIn;
getValue(colIn, valIn);
CDT cdt;
getValue(colIn, valIn, &cdt);
// checkSumLimit(fSum, valIn);
if ((!fDistinct) || (fSet.find(valIn) == fSet.end()))
{
long double val = valIn;
if (scale)
if (scale &&
cdt != CalpontSystemCatalog::LONGDOUBLE)
{
val /= pow(10.0, scale);
}