diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index a15773bc8..c644bddf0 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -65,7 +65,7 @@ double_quote \" grave_accent ` comment ("--"{non_newline}*) -extended_ident_cont [A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\] +extended_ident_cont [ A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\] self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] whitespace ({space}+|{comment}) @@ -75,10 +75,8 @@ ident_cont [A-Za-z\200-\377_0-9\$] identifier {ident_start}{ident_cont}* extended_identifier {ident_start}{extended_ident_cont}* /* fully qualified names regexes */ -ident_w_spaces {identifier}\x20* identifier_quoted {grave_accent}{extended_identifier}{grave_accent} identifier_double_quoted {double_quote}{extended_identifier}{double_quote} -column_ident_quoted {grave_accent}{ident_w_spaces}+{grave_accent} integer [-+]?{digit}+ decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*)) @@ -192,8 +190,6 @@ MEDIUMINT {return MEDIUMINT;} \n { lineno++;} -{column_ident_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES); return IDENT;} - {whitespace} { /* ignore */ } diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index 8b7e9fc42..13b8f72ae 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -206,11 +206,11 @@ ConstantColumn::ConstantColumn( const ConstantColumn& rhs): if (fRegex.get() != NULL) { fRegex.reset(new CNX_Regex()); -#ifdef _MSC_VER - *fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal); -#else +#ifdef POSIX_REGEX string str = dataconvert::DataConvert::constructRegexp(fResult.strVal); regcomp(fRegex.get(), str.c_str(), REG_NOSUB | REG_EXTENDED); +#else + *fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal); #endif } } @@ -257,7 +257,7 @@ ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) : ConstantColumn::~ConstantColumn() { -#ifndef _MSC_VER +#ifdef POSIX_REGEX if (fRegex.get() != NULL) regfree(fRegex.get()); @@ -400,11 +400,11 @@ void ConstantColumn::constructRegex() { //fRegex = new regex_t(); fRegex.reset(new CNX_Regex()); -#ifdef _MSC_VER - *fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal); -#else +#ifdef POSIX_REGEX string str = dataconvert::DataConvert::constructRegexp(fResult.strVal); regcomp(fRegex.get(), str.c_str(), REG_NOSUB | REG_EXTENDED); +#else + *fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal); #endif } diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 02a38efe4..bf51cbeed 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -35,6 +35,7 @@ #include #endif #include +#include #include #include "expressionparser.h" @@ -113,6 +114,8 @@ private: template inline bool numericCompare(result_t op1, result_t op2); inline bool strCompare(const std::string& op1, const std::string& op2); + // MCOL-1559 + inline bool strTrimCompare(const std::string& op1, const std::string& op2); }; inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop) @@ -126,31 +129,31 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu // considers these nulls significant, but they're not in the pattern, so we need to strip // them off... const std::string& v = lop->getStrVal(row, isNull); - char* c = (char*)alloca(v.length() + 1); - memcpy(c, v.c_str(), v.length()); - c[v.length()] = 0; - std::string vv(c); +// char* c = (char*)alloca(v.length() + 1); +// memcpy(c, v.c_str(), v.length()); +// c[v.length()] = 0; +// std::string vv(c); if (regex) { -#ifdef _MSC_VER - bool ret = boost::regex_match(vv, *regex); +#ifdef POSIX_REGEX + bool ret = regexec(regex.get(), v.c_str(), 0, NULL, 0) == 0; #else - bool ret = regexec(regex.get(), vv.c_str(), 0, NULL, 0) == 0; + bool ret = boost::regex_match(v.c_str(), *regex); #endif return (((fOp == OP_LIKE) ? ret : !ret) && !isNull); } else { -#ifdef _MSC_VER - boost::regex regex(dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull))); - bool ret = boost::regex_match(vv, regex); -#else +#ifdef POSIX_REGEX regex_t regex; std::string str = dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull)); regcomp(®ex, str.c_str(), REG_NOSUB | REG_EXTENDED); - bool ret = regexec(®ex, vv.c_str(), 0, NULL, 0) == 0; + bool ret = regexec(®ex, v.c_str(), 0, NULL, 0) == 0; regfree(®ex); +#else + boost::regex regex(dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull))); + bool ret = boost::regex_match(v.c_str(), regex); #endif return (((fOp == OP_LIKE) ? ret : !ret) && !isNull); } @@ -489,11 +492,12 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return false; const std::string& val1 = lop->getStrVal(row, isNull); - if (isNull) return false; - return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; + return strTrimCompare(val1, rop->getStrVal(row, isNull)) && !isNull; +// return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; + } //FIXME: ??? @@ -577,6 +581,37 @@ inline bool PredicateOperator::strCompare(const std::string& op1, const std::str } } +inline bool PredicateOperator::strTrimCompare(const std::string& op1, const std::string& op2) +{ + switch (fOp) + { + case OP_EQ: + return funcexp::utf8::idb_strtrimcoll(op1, op2) == 0; + + case OP_NE: + return funcexp::utf8::idb_strtrimcoll(op1, op2) != 0; + + case OP_GT: + return funcexp::utf8::idb_strtrimcoll(op1, op2) > 0; + + case OP_GE: + return funcexp::utf8::idb_strtrimcoll(op1, op2) >= 0; + + case OP_LT: + return funcexp::utf8::idb_strtrimcoll(op1, op2) < 0; + + case OP_LE: + return funcexp::utf8::idb_strtrimcoll(op1, op2) <= 0; + + default: + { + std::ostringstream oss; + oss << "Non support predicate operation: " << fOp; + throw logging::InvalidOperationExcept(oss.str()); + } + } +} + std::ostream& operator<<(std::ostream& os, const PredicateOperator& rhs); } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 606819dcb..f2e1c1076 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -164,10 +164,10 @@ typedef IDB_Decimal CNX_Decimal; * @brief IDB_Regex struct * */ -#ifdef _MSC_VER -typedef boost::regex IDB_Regex; -#else +#ifdef POSIX_REGEX typedef regex_t IDB_Regex; +#else +typedef boost::regex IDB_Regex; #endif typedef IDB_Regex CNX_Regex; diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 90d70b9b1..8ce36a478 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1644,7 +1644,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) string constval(cc->constval()); - CalpontSystemCatalog::OID dictOid = 0; CalpontSystemCatalog::ColType ct = sc->colType(); const PseudoColumn* pc = dynamic_cast(sc); @@ -1653,8 +1652,13 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) // type of pseudo column is set by connector if (!sc->schemaName().empty() && sc->isColumnStore() && !pc) ct = jobInfo.csc->colType(sc->oid()); - //X + + // Because, on a filter, we want to compare ignoring trailing spaces in many cases + if (sf->op()->op() != execplan::OP_LIKE) + { + boost::algorithm::trim_right_if(constval, boost::is_any_of(" ")); + } //@bug 339 nulls are not stored in dictionary if ((dictOid = isDictCol(ct)) > 0 && ConstantColumn::NULLDATA != cc->type()) { @@ -2778,7 +2782,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) cop = COMPARE_NIL; string value = cc->constval(); - + // Because, on a filter, we want to compare ignoring trailing spaces + boost::algorithm::trim_right_if(value, boost::is_any_of(" ")); pds->addFilter(cop, value); } @@ -2861,7 +2866,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) cop = COMPARE_NIL; string value = cc->constval(); - + // Because, on a filter, we want to compare ignoring trailing spaces + boost::algorithm::trim_right_if(value, boost::is_any_of(" ")); pds->addFilter(cop, value); } @@ -2968,7 +2974,6 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) int64_t value = 0; string constval = cc->constval(); - // @bug 1151 string longer than colwidth of char/varchar. uint8_t rf = 0; bool isNull = ConstantColumn::NULLDATA == cc->type(); diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 26242640e..7f01dccb7 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -400,6 +400,7 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) void preProcessFunctionOnAggregation(const vector& scs, const vector& aggs, + const vector& wcs, JobInfo& jobInfo) { // append the simple columns if not already projected @@ -433,6 +434,10 @@ void preProcessFunctionOnAggregation(const vector& scs, for (vector::const_iterator i = aggs.begin(); i != aggs.end(); i++) { addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo); + if (wcs.size() > 0) + { + jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone())); + } } } @@ -484,12 +489,12 @@ void checkReturnedColumns(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) if (ac != NULL && ac->aggColumnList().size() > 0) { jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), jobInfo); + preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), ac->windowfunctionColumnList(), jobInfo); } else if (fc != NULL && fc->aggColumnList().size() > 0) { jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), jobInfo); + preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), fc->windowfunctionColumnList(), jobInfo); } } } @@ -600,7 +605,8 @@ void checkAggregation(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) jobInfo.hasDistinct = csep->distinct(); - if (csep->distinct() == true) + // DISTINCT with window functions must be done in tupleannexstep + if (csep->distinct() == true && jobInfo.windowDels.size() == 0) { jobInfo.hasAggregation = true; } @@ -874,6 +880,10 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo const SimpleColumn* sc = dynamic_cast(srcp.get()); AggregateColumn* aggc = dynamic_cast(srcp.get()); bool doDistinct = (csep->distinct() && csep->groupByCols().empty()); + // Use this instead of the above line to mimic MariaDB's sql_mode = 'ONLY_FULL_GROUP_BY' + // bool doDistinct = (csep->distinct() && + // csep->groupByCols().empty() && + // !jobInfo.hasAggregation); uint32_t tupleKey = -1; string alias; string view; @@ -1122,7 +1132,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo // remember the columns to be returned jobInfo.returnedColVec.push_back(make_pair(tupleKey, op)); - // bug 1499 distinct processing, save unique distinct columns + // bug 1499 distinct processing, save unique distinct columns if (doDistinct && (jobInfo.distinctColVec.end() == find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), tupleKey))) @@ -1521,7 +1531,7 @@ void parseExecutionPlan(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, // bug4531, window function support WindowFunctionStep::checkWindowFunction(csep, jobInfo); - // bug3391, move forward the aggregation check for no aggregte having clause. + // bug3391, move forward the aggregation check for no aggregate having clause. checkAggregation(csep, jobInfo); // include filters in having clause, if any. diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 81d190644..ec9562b6d 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -840,6 +840,7 @@ const string TupleAggregateStep::toString() const SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo) { SJSTEP spjs; + TupleDeliveryStep* tds = dynamic_cast(step.get()); TupleBPS* tbps = dynamic_cast(step.get()); TupleHashJoinStep* thjs = dynamic_cast(step.get()); SubAdapterStep* sas = dynamic_cast(step.get()); @@ -917,171 +918,83 @@ SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo) // preprocess the columns used by group_concat jobInfo.groupConcatInfo.prepGroupConcat(jobInfo); - bool doGroupConcat = false; + bool doUMOnly = jobInfo.groupConcatInfo.columns().size() > 0 +// || jobInfo.windowSet.size() > 0 + || sas + || ces; + + rgs.push_back(tds->getDeliveredRowGroup()); + + // get rowgroup and aggregator + // For TupleHashJoin, we prepare for both PM and UM only aggregation + if (doUMOnly || thjs) + { + if (distinctAgg == true) + prep1PhaseDistinctAggregate(jobInfo, rgs, aggs); + else + prep1PhaseAggregate(jobInfo, rgs, aggs); + + // TODO: fix this + if (doUMOnly) + rgs.push_back(rgs[0]); + } + + if (!doUMOnly) + { + if (distinctAgg == true) + prep2PhasesDistinctAggregate(jobInfo, rgs, aggs); + else + prep2PhasesAggregate(jobInfo, rgs, aggs); + } if (tbps != NULL) { - // get rowgroup and aggregator - rgs.push_back(tbps->getDeliveredRowGroup()); - - if (jobInfo.groupConcatInfo.columns().size() == 0) - { - if (distinctAgg == true) - prep2PhasesDistinctAggregate(jobInfo, rgs, aggs); - else - prep2PhasesAggregate(jobInfo, rgs, aggs); - } - else - { - if (distinctAgg == true) - prep1PhaseDistinctAggregate(jobInfo, rgs, aggs); - else - prep1PhaseAggregate(jobInfo, rgs, aggs); - - // TODO: fix this - rgs.push_back(rgs[0]); - doGroupConcat = true; - } - - // make sure connected by a RowGroupDL - JobStepAssociation tbpsJsa; - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - dl->OID(execplan::CNX_VTABLE_ID); - spdl->rowGroupDL(dl); - tbpsJsa.outAdd(spdl); - // create delivery step aggUM = dynamic_pointer_cast(aggs[0]); spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[2], jobInfo)); - spjs->inputAssociation(tbpsJsa); - // step id?? - spjs->stepId(step->stepId() + 1); - - // set the PM/UM side aggregate structs - tbps->outputAssociation(tbpsJsa); - - if (doGroupConcat) + if (doUMOnly) dynamic_cast(spjs.get())->umOnly(true); else tbps->setAggregateStep(aggs[1], rgs[2]); } else if (thjs != NULL) { - // default to UM aggregation - rgs.push_back(thjs->getDeliveredRowGroup()); - - if (distinctAgg == true) - prep1PhaseDistinctAggregate(jobInfo, rgs, aggs); - else - prep1PhaseAggregate(jobInfo, rgs, aggs); - - // also prepare for PM aggregation - // rowgroups -- 0-proj, 1-um, [2-phase case: 2-um, 3-pm] - // aggregators -- 0-um, [2-phase case: 1-um, 2-pm] - if (jobInfo.groupConcatInfo.columns().size() == 0) - { - if (distinctAgg == true) - prep2PhasesDistinctAggregate(jobInfo, rgs, aggs); - else - prep2PhasesAggregate(jobInfo, rgs, aggs); - } - else - { - // TODO: fix this - rgs.push_back(rgs[0]); - doGroupConcat = true; - } - - // make sure connected by a RowGroupDL - JobStepAssociation thjsJsa; - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - dl->OID(execplan::CNX_VTABLE_ID); - spdl->rowGroupDL(dl); - thjsJsa.outAdd(spdl); - // create delivery step aggUM = dynamic_pointer_cast(aggs[0]); spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo)); - spjs->inputAssociation(thjsJsa); - if (doGroupConcat) + if (doUMOnly) dynamic_cast(spjs.get())->umOnly(true); else dynamic_cast(spjs.get())->savePmHJData(aggs[1], aggs[2], rgs[3]); - - // step id?? - spjs->stepId(step->stepId() + 1); - // set input side - thjs->outputAssociation(thjsJsa); thjs->deliveryStep(spjs); } - else if (sas != NULL) + else { - // UM aggregation - // rowgroups -- 0-proj, 1-um - // aggregators -- 0-um - rgs.push_back(sas->getDeliveredRowGroup()); - - if (distinctAgg == true) - prep1PhaseDistinctAggregate(jobInfo, rgs, aggs); - else - prep1PhaseAggregate(jobInfo, rgs, aggs); - - // make sure connected by a RowGroupDL - JobStepAssociation sasJsa; - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - dl->OID(execplan::CNX_VTABLE_ID); - spdl->rowGroupDL(dl); - sasJsa.outAdd(spdl); - - // create delivery step aggUM = dynamic_pointer_cast(aggs[0]); spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo)); - spjs->inputAssociation(sasJsa); - - // step id?? - spjs->stepId(step->stepId() + 1); - - // set input side - sas->outputAssociation(sasJsa); } - else if (ces != NULL) - { - // UM aggregation - // rowgroups -- 0-proj, 1-um - // aggregators -- 0-um - rgs.push_back(ces->getDeliveredRowGroup()); - if (distinctAgg == true) - prep1PhaseDistinctAggregate(jobInfo, rgs, aggs); - else - prep1PhaseAggregate(jobInfo, rgs, aggs); + // Setup the input JobstepAssoctiation -- the mechanism + // whereby the previous step feeds data to this step. + // Otherwise, we need to create one and hook to the + // previous step as well as this aggregate step. + spjs->stepId(step->stepId() + 1); - // make sure connected by a RowGroupDL - JobStepAssociation cesJsa; - AnyDataListSPtr spdl(new AnyDataList()); - RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); - dl->OID(execplan::CNX_VTABLE_ID); - spdl->rowGroupDL(dl); - cesJsa.outAdd(spdl); + JobStepAssociation jsa; + AnyDataListSPtr spdl(new AnyDataList()); + RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize); + dl->OID(execplan::CNX_VTABLE_ID); + spdl->rowGroupDL(dl); + jsa.outAdd(spdl); - // create delivery step - aggUM = dynamic_pointer_cast(aggs[0]); - spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo)); - spjs->inputAssociation(cesJsa); + spjs->inputAssociation(jsa); // Aggregate input - // step id?? - spjs->stepId(step->stepId() + 1); - - // set input side - ces->outputAssociation(cesJsa); - } + //Previous step output + step->outputAssociation(jsa); // add the aggregate on constants if (constAggDataVec.size() > 0) diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index b145bfa12..6daa6ee02 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -477,6 +477,17 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J colSet.insert(key); } + // MCOL-3435 We haven't yet checked for aggregate, but we need to know + bool hasAggregation = false; + for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++) + { + if (dynamic_cast(jobInfo.deliveredCols[i].get()) != NULL) + { + hasAggregation = true; + break; + } + } + // add non-duplicate auxiliary columns RetColsVector colsInAf; @@ -499,10 +510,30 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J if (colSet.find(key) == colSet.end()) { jobInfo.deliveredCols.push_back(*j); -// MCOL-3343 Enable this if we decide to allow Window Functions to run with -// aggregates with no group by. MariaDB allows this. Nobody else in the world does. -// There will be more work to get it to function if we try this. -// jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true)); + // MCOL-3435 Allow Window Functions to run with aggregates with + // no group by by inserting a group by for window parameters. + if (hasAggregation) + { + // If an argument is an AggregateColumn, don't group by it. + if (dynamic_cast(j->get()) == NULL) + { + bool bFound = false; + for (std::vector::iterator igpc = csep->groupByCols().begin(); + igpc < csep->groupByCols().end(); + ++igpc) + { + if (*igpc->get() == *j->get()) + { + bFound = true; + break; + } + } + if (!bFound) + { + csep->groupByCols().push_back(*j); + } + } + } } colSet.insert(key); diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 5e0905596..325caee48 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -3160,7 +3160,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp String val, *str = item->val_str(&val); string valStr; valStr.assign(str->ptr(), str->length()); - rc = new ConstantColumn(valStr); + rc = new ConstantColumn(valStr, ConstantColumn::NUM); break; } case REAL_RESULT: diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index 366e90daf..a02f88b29 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -33,9 +33,9 @@ #include #endif -#ifdef __linux__ -#define POSIX_REGEX -#endif +//#ifdef __linux__ +//#define POSIX_REGEX +//#endif #ifdef POSIX_REGEX #include diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 967430853..a35d784dc 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -2616,11 +2616,17 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, tmp = tmp.filename(); DBRMroot = (bf::path(DBRMDir) / tmp).string(); + // DBRMDir might have changed, so need to change DBRMroot + bf::path tmp(DBRMroot); + tmp = tmp.filename(); + DBRMroot = (bf::path(DBRMDir) / tmp).string(); + sendAlarm("DBRM", DBRM_LOAD_DATA_ERROR, CLEAR); // change DBRMroot to temp DBRMDir path // DBRMroot = tempDBRMDir + "/BRM_saves"; } + // // run the 'load_brm' script first if files exist // @@ -4420,6 +4426,12 @@ int ProcessMonitor::getDBRMdata(string *path) *path = pTmp.string(); log.writeLog(__LINE__, "Downloading DBRM files to " + *path, LOG_TYPE_DEBUG); + boost::uuids::uuid u = boost::uuids::random_generator()(); + bf::path pTmp = bf::path(*path) / boost::uuids::to_string(u); + bf::create_directories(pTmp); + *path = pTmp.string(); + log.writeLog(__LINE__, "Downloading DBRM files to " + *path, LOG_TYPE_DEBUG); + for ( int i = 0 ; i < numFiles ; i ++ ) { string fileName; @@ -4464,6 +4476,10 @@ int ProcessMonitor::getDBRMdata(string *path) pFilename = pTmp / pFilename.filename(); const char *cFilename = pFilename.string().c_str(); + bf::path pFilename(fileName); + pFilename = pTmp / pFilename.filename(); + const char *cFilename = pFilename.string().c_str(); + boost::scoped_ptr out(IDBDataFile::open( IDBPolicy::getType(cFilename, IDBPolicy::WRITEENG), @@ -6359,7 +6375,6 @@ int ProcessMonitor::checkDataMount() /* StorageManager isn't running yet. Can't check for writability here. */ return API_SUCCESS; } - //go unmount disk NOT assigned to this pm unmountExtraDBroots(); diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 2cc3dbb77..8b065e15f 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -94,10 +94,6 @@ bool getUIntValFromParm( { isNull = true; } - else - { - value = 0; - } } break; @@ -111,9 +107,9 @@ bool getUIntValFromParm( { d.value = 0; } - - int64_t tmpval = d.value / helpers::power(d.scale); - int lefto = (d.value - tmpval * helpers::power(d.scale)) / helpers::power(d.scale - 1); + double dscale = d.scale; + int64_t tmpval = d.value / pow(10.0, dscale); + int lefto = (d.value - tmpval * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( tmpval >= 0 && lefto > 4 ) tmpval++; diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 18a9b19aa..d335ddc81 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -180,8 +180,9 @@ int64_t Func_cast_signed::getIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / helpers::power(d.scale); - int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1); + double dscale = d.scale; + int64_t value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( value >= 0 && lefto > 4 ) value++; @@ -337,14 +338,15 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + double dscale = d.scale; if (d.value < 0) { return 0; } - uint64_t value = d.value / helpers::power(d.scale); - int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1); + uint64_t value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( value >= 0 && lefto > 4 ) value++; diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index 86cec9f32..fa1dc4e1c 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -156,9 +156,10 @@ string Func_char::getStrVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + double dscale = d.scale; // get decimal and round up - int value = d.value / helpers::power(d.scale); - int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1); + int value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( lefto > 4 ) value++; diff --git a/utils/funcexp/func_div.cpp b/utils/funcexp/func_div.cpp index 327f5e71e..b82cfaa4b 100644 --- a/utils/funcexp/func_div.cpp +++ b/utils/funcexp/func_div.cpp @@ -51,6 +51,16 @@ int64_t Func_div::getIntVal(rowgroup::Row& row, { double val1 = parm[0]->data()->getDoubleVal(row, isNull); double val2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (val2 == 0 || val2 == NAN) + { + isNull = true; + return 0; + } + // MCOL-179 InnoDB doesn't round or convert to int before dividing. + return static_cast(val1 / val2); + +#if 0 int64_t int_val2 = (int64_t)(val2 > 0 ? val2 + 0.5 : val2 - 0.5); if (int_val2 == 0) @@ -69,6 +79,7 @@ int64_t Func_div::getIntVal(rowgroup::Row& row, } return int_val1 / int_val2; +#endif } diff --git a/utils/funcexp/func_elt.cpp b/utils/funcexp/func_elt.cpp index d34dafa78..99c86af75 100644 --- a/utils/funcexp/func_elt.cpp +++ b/utils/funcexp/func_elt.cpp @@ -71,8 +71,9 @@ string Func_elt::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - number = d.value / helpers::power(d.scale); - int lefto = (d.value - number * helpers::power(d.scale)) / helpers::power(d.scale - 1); + double dscale = d.scale; + number = d.value / pow(10.0, dscale); + int lefto = (d.value - number * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( number >= 0 && lefto > 4 ) number++; diff --git a/utils/funcexp/func_makedate.cpp b/utils/funcexp/func_makedate.cpp index 5d013728f..8c8c50abf 100644 --- a/utils/funcexp/func_makedate.cpp +++ b/utils/funcexp/func_makedate.cpp @@ -68,8 +68,9 @@ uint64_t makedate(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - year = d.value / helpers::power(d.scale); - int lefto = (d.value - year * helpers::power(d.scale)) / helpers::power(d.scale - 1); + double dscale = d.scale; + year = d.value / pow(10.0, dscale); + int lefto = (d.value - year * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( year >= 0 && lefto > 4 ) year++; @@ -127,8 +128,9 @@ uint64_t makedate(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - int64_t tmp = d.value / helpers::power(d.scale); - int lefto = (d.value - tmp * helpers::power(d.scale)) / helpers::power(d.scale - 1); + double dscale = d.scale; + int64_t tmp = d.value / pow(10.0, dscale); + int lefto = (d.value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( tmp >= 0 && lefto > 4 ) tmp++; diff --git a/utils/funcexp/func_maketime.cpp b/utils/funcexp/func_maketime.cpp index 694e995a7..b43fed4ab 100644 --- a/utils/funcexp/func_maketime.cpp +++ b/utils/funcexp/func_maketime.cpp @@ -74,8 +74,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - hour = d.value / helpers::power(d.scale); - int lefto = (d.value - hour * helpers::power(d.scale)) / helpers::power(d.scale - 1); + double dscale = d.scale; + hour = d.value / pow(10.0, dscale); + int lefto = (d.value - hour * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( hour >= 0 && lefto > 4 ) hour++; @@ -113,8 +114,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - min = d.value / helpers::power(d.scale); - int lefto = (d.value - min * helpers::power(d.scale)) / helpers::power(d.scale - 1); + double dscale = d.scale; + min = d.value / pow(10.0, dscale); + int lefto = (d.value - min * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( min >= 0 && lefto > 4 ) min++; @@ -158,8 +160,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[2]->data()->getDecimalVal(row, isNull); - sec = d.value / helpers::power(d.scale); - int lefto = (d.value - sec * helpers::power(d.scale)) / helpers::power(d.scale - 1); + double dscale = d.scale; + sec = d.value / pow(10.0, dscale); + int lefto = (d.value - sec * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( sec >= 0 && lefto > 4 ) sec++; diff --git a/utils/funcexp/func_mod.cpp b/utils/funcexp/func_mod.cpp index 23e4df008..214fd2b6c 100644 --- a/utils/funcexp/func_mod.cpp +++ b/utils/funcexp/func_mod.cpp @@ -74,10 +74,10 @@ IDB_Decimal Func_mod::getDecimalVal(Row& row, } IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / helpers::power(d.scale); - int lefto = d.value % helpers::power(d.scale); + int64_t value = d.value / pow(10.0, d.scale); + int lefto = d.value % (int)pow(10.0, d.scale); - int64_t mod = (value % div) * helpers::power(d.scale) + lefto; + int64_t mod = (value % div) * pow(10.0, d.scale) + lefto; retValue.value = mod; retValue.scale = d.scale; @@ -164,7 +164,7 @@ double Func_mod::getDoubleVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / helpers::power(d.scale); + int64_t value = d.value / pow(10.0, d.scale); mod = value % div; } @@ -268,7 +268,7 @@ long double Func_mod::getLongDoubleVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / helpers::power(d.scale); + int64_t value = d.value / pow(10.0, d.scale); mod = value % div; } @@ -375,7 +375,7 @@ int64_t Func_mod::getIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / helpers::power(d.scale); + int64_t value = d.value / pow(10.0, d.scale); mod = value % div; } @@ -473,7 +473,7 @@ uint64_t Func_mod::getUIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / helpers::power(d.scale); + int64_t value = d.value / pow(10.0, d.scale); mod = value % div; } diff --git a/utils/funcexp/func_period_diff.cpp b/utils/funcexp/func_period_diff.cpp index d66d50de7..0f1b923c9 100644 --- a/utils/funcexp/func_period_diff.cpp +++ b/utils/funcexp/func_period_diff.cpp @@ -86,7 +86,7 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - period1 = d.value / helpers::power(d.scale); + period1 = d.value / pow(10.0, d.scale); break; } @@ -135,7 +135,7 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - period2 = d.value / helpers::power(d.scale); + period2 = d.value / pow(10.0, d.scale); break; } diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index e92bbcbcc..be50ca2cf 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -46,6 +46,9 @@ namespace funcexp { namespace utf8 { +// A global loc object so we don't construct one at every compare +std::locale loc; + bool JPcodePoint = false; // extern-ed in utils_utf8.h } diff --git a/utils/funcexp/utils_utf8.h b/utils/funcexp/utils_utf8.h index e88772bc4..23d41ce7e 100644 --- a/utils/funcexp/utils_utf8.h +++ b/utils/funcexp/utils_utf8.h @@ -36,7 +36,6 @@ #include #include - #include "liboamcpp.h" /** @file */ @@ -49,6 +48,10 @@ extern bool JPcodePoint; // code point ordering (Japanese UTF) flag, used in id const int MAX_UTF8_BYTES_PER_CHAR = 4; +// A global loc object so we don't construct one at every compare +extern std::locale loc; +// Is there a way to construct a global reference to a facet? +// const std::collate& coll = std::use_facet >(loc); //Infinidb version of strlocale BUG 5362 //set System Locale "C" by default @@ -110,6 +113,10 @@ std::string idb_setlocale() if (systemLang.find("ja_JP") != std::string::npos) JPcodePoint = true; + // MCOL-1559 Save off the locale to save runtime cpus + std::locale localloc(systemLang.c_str()); + loc = localloc; + return systemLang; } @@ -125,6 +132,51 @@ int idb_strcoll(const char* str1, const char* str2) return strcoll(str1, str2); } +// MCOL-1559 Add a trimmed version of strcoll +// The intent here is to make no copy of the original strings and +// not modify them, so we can't use trim to deal with the spaces. +inline +int idb_strtrimcoll(const std::string& str1, const std::string& str2) +{ + static const std::string whitespaces (" "); + const char* s1 = str1.c_str(); + const char* s2 = str2.c_str(); + + // Set found1 to the last non-whitespace char in str1 + std::size_t found1 = str1.find_last_not_of(whitespaces); + // Set found2 to the first whitespace char in str2 + std::size_t found2 = str2.find_last_not_of(whitespaces); + + // Are both strings empty or all whitespace? + if (found1 == std::string::npos && found2 == std::string::npos) + { + return 0; // they match + } + // If str1 is empty or all spaces + if (found1 == std::string::npos) + { + return -1; + } + // If str2 is empty or all spaces + if (found2 == std::string::npos) + { + return 1; + } + + // found1 and found2 point to the character that is not a space. + // compare wants it to point to one past. + found1 += 1; + found2 += 1; + // If no trimming needs doing, then strcoll is faster + if (found1 == str1.size() && found2 == str2.size()) + { + return idb_strcoll(s1, s2); + } + // Compare the (trimmed) strings + const std::collate& coll = std::use_facet >(loc); + int rtn = coll.compare(s1, s1+found1, s2, s2+found2); + return rtn; +} // BUG 5241 // Infinidb specific mbstowcs(). This will handle both windows and unix platforms diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 0990384cb..373269e13 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1035,12 +1035,13 @@ inline void Row::setFloatField(float val, uint32_t colIndex) inline void Row::setLongDoubleField(long double val, uint32_t colIndex) { + uint8_t* p = &data[offsets[colIndex]]; + *((long double*)p) = val; if (sizeof(long double) == 16) { // zero out the unused portion as there may be garbage there. - *((uint64_t*)&val+1) &= 0x000000000000FFFFULL; + *((uint64_t*)p+1) &= 0x000000000000FFFFULL; } - *((long double*) &data[offsets[colIndex]]) = val; } inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex) diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index db4b107ee..8b06eda61 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -140,6 +140,7 @@ void WF_stats::resetData() template void WF_stats::operator()(int64_t b, int64_t e, int64_t c) { + CDT cdt; if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) @@ -163,7 +164,7 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) continue; T valIn; - getValue(colIn, valIn); + getValue(colIn, valIn, &cdt); long double val = (long double) valIn; fSum1 += val; @@ -177,7 +178,9 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) int scale = fRow.getScale(colIn); long double factor = pow(10.0, scale); - if (scale != 0) // adjust the scale if necessary + // adjust the scale if necessary + if (scale != 0 && + cdt != CalpontSystemCatalog::LONGDOUBLE) { fSum1 /= factor; fSum2 /= factor * factor; diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index 4632496df..d0d5f46f7 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -264,13 +264,15 @@ void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) continue; T valIn; - getValue(colIn, valIn); + CDT cdt; + getValue(colIn, valIn, &cdt); // checkSumLimit(fSum, valIn); if ((!fDistinct) || (fSet.find(valIn) == fSet.end())) { long double val = valIn; - if (scale) + if (scale && + cdt != CalpontSystemCatalog::LONGDOUBLE) { val /= pow(10.0, scale); }