Merge pull request #873 from mariadb-corporation/develop-merge-up-20190924-2

Merge develop-1.2 into develop

dbcon/ddlpackage/ddl.l

@@ -65,7 +65,7 @@ double_quote \"
 grave_accent `
 
 comment ("--"{non_newline}*)
-extended_ident_cont [A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\]
+extended_ident_cont [ A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\]
 self			[,()\[\].;\:\+\-\*\/\%\^\<\>\=]
 whitespace ({space}+|{comment})
 
@@ -75,10 +75,8 @@ ident_cont [A-Za-z\200-\377_0-9\$]
 identifier {ident_start}{ident_cont}*
 extended_identifier {ident_start}{extended_ident_cont}*
 /* fully qualified names regexes */
-ident_w_spaces {identifier}\x20*
 identifier_quoted {grave_accent}{extended_identifier}{grave_accent}
 identifier_double_quoted {double_quote}{extended_identifier}{double_quote}
-column_ident_quoted {grave_accent}{ident_w_spaces}+{grave_accent}
 
 integer			[-+]?{digit}+
 decimal			([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*))
@@ -192,8 +190,6 @@ MEDIUMINT {return MEDIUMINT;}
 
 \n { lineno++;}
 
-{column_ident_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES); return IDENT;}
-
 {whitespace} {
     /* ignore */
 }

dbcon/execplan/constantcolumn.cpp

@@ -206,11 +206,11 @@ ConstantColumn::ConstantColumn( const ConstantColumn& rhs):
     if (fRegex.get() != NULL)
     {
         fRegex.reset(new CNX_Regex());
-#ifdef _MSC_VER
+#ifdef POSIX_REGEX
-        *fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal);
-#else
         string str = dataconvert::DataConvert::constructRegexp(fResult.strVal);
         regcomp(fRegex.get(), str.c_str(), REG_NOSUB | REG_EXTENDED);
+#else
+        *fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal);
 #endif
     }
 }
@@ -257,7 +257,7 @@ ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) :
 
 ConstantColumn::~ConstantColumn()
 {
-#ifndef _MSC_VER
+#ifdef POSIX_REGEX
 
     if (fRegex.get() != NULL)
         regfree(fRegex.get());
@@ -400,11 +400,11 @@ void ConstantColumn::constructRegex()
 {
     //fRegex = new regex_t();
     fRegex.reset(new CNX_Regex());
-#ifdef _MSC_VER
+#ifdef POSIX_REGEX
-    *fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal);
-#else
     string str = dataconvert::DataConvert::constructRegexp(fResult.strVal);
     regcomp(fRegex.get(), str.c_str(), REG_NOSUB | REG_EXTENDED);
+#else
+    *fRegex = dataconvert::DataConvert::constructRegexp(fResult.strVal);
 #endif
 }
 

dbcon/execplan/predicateoperator.h

@@ -35,6 +35,7 @@
 #include <alloca.h>
 #endif
 #include <cstring>
+#include <cmath>
 #include <boost/regex.hpp>
 
 #include "expressionparser.h"
@@ -113,6 +114,8 @@ private:
     template <typename result_t>
     inline bool numericCompare(result_t op1, result_t op2);
     inline bool strCompare(const std::string& op1, const std::string& op2);
+    // MCOL-1559
+    inline bool strTrimCompare(const std::string& op1, const std::string& op2);
 };
 
 inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop)
@@ -126,31 +129,31 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu
         //  considers these nulls significant, but they're not in the pattern, so we need to strip
         //   them off...
         const std::string& v = lop->getStrVal(row, isNull);
-        char* c = (char*)alloca(v.length() + 1);
+//        char* c = (char*)alloca(v.length() + 1);
-        memcpy(c, v.c_str(), v.length());
+//        memcpy(c, v.c_str(), v.length());
-        c[v.length()] = 0;
+//        c[v.length()] = 0;
-        std::string vv(c);
+//        std::string vv(c);
 
         if (regex)
         {
-#ifdef _MSC_VER
+#ifdef POSIX_REGEX
-            bool ret = boost::regex_match(vv, *regex);
+            bool ret = regexec(regex.get(), v.c_str(), 0, NULL, 0) == 0;
 #else
-            bool ret = regexec(regex.get(), vv.c_str(), 0, NULL, 0) == 0;
+            bool ret = boost::regex_match(v.c_str(), *regex);
 #endif
             return (((fOp == OP_LIKE) ? ret : !ret) && !isNull);
         }
         else
         {
-#ifdef _MSC_VER
+#ifdef POSIX_REGEX
-            boost::regex regex(dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull)));
-            bool ret = boost::regex_match(vv, regex);
-#else
             regex_t regex;
             std::string str = dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull));
             regcomp(&regex, str.c_str(), REG_NOSUB | REG_EXTENDED);
-            bool ret = regexec(&regex, vv.c_str(), 0, NULL, 0) == 0;
+            bool ret = regexec(&regex, v.c_str(), 0, NULL, 0) == 0;
             regfree(&regex);
+#else
+            boost::regex regex(dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull)));
+            bool ret = boost::regex_match(v.c_str(), regex);
 #endif
             return (((fOp == OP_LIKE) ? ret : !ret) && !isNull);
         }
@@ -489,11 +492,12 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu
                 return false;
 
             const std::string& val1 = lop->getStrVal(row, isNull);
-
             if (isNull)
                 return false;
 
-            return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull;
+            return strTrimCompare(val1, rop->getStrVal(row, isNull)) && !isNull;
+//            return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull;
+
         }
 
         //FIXME: ???
@@ -577,6 +581,37 @@ inline bool PredicateOperator::strCompare(const std::string& op1, const std::str
     }
 }
 
+inline bool PredicateOperator::strTrimCompare(const std::string& op1, const std::string& op2)
+{
+    switch (fOp)
+    {
+        case OP_EQ:
+            return funcexp::utf8::idb_strtrimcoll(op1, op2) == 0;
+
+        case OP_NE:
+            return funcexp::utf8::idb_strtrimcoll(op1, op2) != 0;
+
+        case OP_GT:
+            return funcexp::utf8::idb_strtrimcoll(op1, op2) > 0;
+
+        case OP_GE:
+            return funcexp::utf8::idb_strtrimcoll(op1, op2) >= 0;
+
+        case OP_LT:
+            return funcexp::utf8::idb_strtrimcoll(op1, op2) < 0;
+
+        case OP_LE:
+            return funcexp::utf8::idb_strtrimcoll(op1, op2) <= 0;
+
+        default:
+        {
+            std::ostringstream oss;
+            oss << "Non support predicate operation: " << fOp;
+            throw logging::InvalidOperationExcept(oss.str());
+        }
+    }
+}
+
 std::ostream& operator<<(std::ostream& os, const PredicateOperator& rhs);
 }
 

dbcon/execplan/treenode.h

@@ -164,10 +164,10 @@ typedef IDB_Decimal CNX_Decimal;
  * @brief IDB_Regex struct
  *
  */
-#ifdef _MSC_VER
+#ifdef POSIX_REGEX
-typedef boost::regex IDB_Regex;
-#else
 typedef regex_t IDB_Regex;
+#else
+typedef boost::regex IDB_Regex;
 #endif
 
 typedef IDB_Regex CNX_Regex;

dbcon/joblist/jlf_execplantojoblist.cpp

@@ -1644,7 +1644,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
 
         string constval(cc->constval());
 
-
         CalpontSystemCatalog::OID dictOid = 0;
         CalpontSystemCatalog::ColType ct = sc->colType();
         const PseudoColumn* pc = dynamic_cast<const PseudoColumn*>(sc);
@@ -1653,8 +1652,13 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
 //    type of pseudo column is set by connector
         if (!sc->schemaName().empty() && sc->isColumnStore() && !pc)
             ct = jobInfo.csc->colType(sc->oid());
-
 //X
+
+        // Because, on a filter, we want to compare ignoring trailing spaces in many cases
+        if (sf->op()->op() != execplan::OP_LIKE)
+        {
+            boost::algorithm::trim_right_if(constval, boost::is_any_of(" "));
+        }
         //@bug 339 nulls are not stored in dictionary
         if ((dictOid = isDictCol(ct)) > 0  && ConstantColumn::NULLDATA != cc->type())
         {
@@ -2778,7 +2782,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
                         cop = COMPARE_NIL;
 
                     string value = cc->constval();
-
+                    // Because, on a filter, we want to compare ignoring trailing spaces
+                    boost::algorithm::trim_right_if(value, boost::is_any_of(" "));
 
                     pds->addFilter(cop, value);
                 }
@@ -2861,7 +2866,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
                         cop = COMPARE_NIL;
 
                     string value = cc->constval();
-
+                    // Because, on a filter, we want to compare ignoring trailing spaces
+                    boost::algorithm::trim_right_if(value, boost::is_any_of(" "));
 
                     pds->addFilter(cop, value);
                 }
@@ -2968,7 +2974,6 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
                     int64_t value = 0;
                     string constval = cc->constval();
 
-
                     // @bug 1151 string longer than colwidth of char/varchar.
                     uint8_t rf = 0;
                     bool isNull = ConstantColumn::NULLDATA == cc->type();

dbcon/joblist/joblistfactory.cpp

@@ -400,6 +400,7 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
 
 void preProcessFunctionOnAggregation(const vector<SimpleColumn*>& scs,
                                      const vector<AggregateColumn*>& aggs,
+                                     const vector<WindowFunctionColumn*>& wcs,
                                      JobInfo& jobInfo)
 {
     // append the simple columns if not already projected
@@ -433,6 +434,10 @@ void preProcessFunctionOnAggregation(const vector<SimpleColumn*>& scs,
     for (vector<AggregateColumn*>::const_iterator i = aggs.begin(); i != aggs.end(); i++)
     {
         addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo);
+        if (wcs.size() > 0)
+        {
+            jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone()));
+        }
     }
 }
 
@@ -484,12 +489,12 @@ void checkReturnedColumns(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
         if (ac != NULL && ac->aggColumnList().size() > 0)
         {
             jobInfo.nonConstCols[i]->outputIndex(i);
-            preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), jobInfo);
+            preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), ac->windowfunctionColumnList(), jobInfo);
         }
         else if (fc != NULL && fc->aggColumnList().size() > 0)
         {
             jobInfo.nonConstCols[i]->outputIndex(i);
-            preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), jobInfo);
+            preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), fc->windowfunctionColumnList(), jobInfo);
         }
     }
 }
@@ -600,7 +605,8 @@ void checkAggregation(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
 
     jobInfo.hasDistinct = csep->distinct();
 
-    if (csep->distinct() == true)
+    // DISTINCT with window functions must be done in tupleannexstep
+    if (csep->distinct() == true && jobInfo.windowDels.size() == 0)
     {
         jobInfo.hasAggregation = true;
     }
@@ -874,6 +880,10 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo
         const SimpleColumn* sc = dynamic_cast<const SimpleColumn*>(srcp.get());
         AggregateColumn* aggc = dynamic_cast<AggregateColumn*>(srcp.get());
         bool doDistinct = (csep->distinct() && csep->groupByCols().empty());
+    //    Use this instead of the above line to mimic MariaDB's sql_mode = 'ONLY_FULL_GROUP_BY'
+    //    bool doDistinct = (csep->distinct() && 
+    //                       csep->groupByCols().empty() &&
+    //                       !jobInfo.hasAggregation);
         uint32_t tupleKey = -1;
         string alias;
         string view;
@@ -1122,7 +1132,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo
                     // remember the columns to be returned
                     jobInfo.returnedColVec.push_back(make_pair(tupleKey, op));
 
-                    // bug 1499 distinct processing, save unique distinct columns
+                   // bug 1499 distinct processing, save unique distinct columns
                     if (doDistinct &&
                             (jobInfo.distinctColVec.end() ==
                              find(jobInfo.distinctColVec.begin(), jobInfo.distinctColVec.end(), tupleKey)))
@@ -1521,7 +1531,7 @@ void parseExecutionPlan(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo,
     // bug4531, window function support
     WindowFunctionStep::checkWindowFunction(csep, jobInfo);
 
-    // bug3391, move forward the aggregation check for no aggregte having clause.
+    // bug3391, move forward the aggregation check for no aggregate having clause.
     checkAggregation(csep, jobInfo);
 
     // include filters in having clause, if any.

dbcon/joblist/tupleaggregatestep.cpp

@@ -840,6 +840,7 @@ const string TupleAggregateStep::toString() const
 SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo)
 {
     SJSTEP spjs;
+    TupleDeliveryStep* tds = dynamic_cast<TupleDeliveryStep*>(step.get());
     TupleBPS* tbps = dynamic_cast<TupleBPS*>(step.get());
     TupleHashJoinStep* thjs = dynamic_cast<TupleHashJoinStep*>(step.get());
     SubAdapterStep* sas = dynamic_cast<SubAdapterStep*>(step.get());
@@ -917,171 +918,83 @@ SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo)
 
     // preprocess the columns used by group_concat
     jobInfo.groupConcatInfo.prepGroupConcat(jobInfo);
-    bool doGroupConcat = false;
+    bool doUMOnly = jobInfo.groupConcatInfo.columns().size() > 0
+//                 || jobInfo.windowSet.size() > 0
+                 || sas
+                 || ces;
+
+    rgs.push_back(tds->getDeliveredRowGroup());
+
+    // get rowgroup and aggregator
+    // For TupleHashJoin, we prepare for both PM and UM only aggregation
+    if (doUMOnly || thjs)
+    {
+        if (distinctAgg == true)
+            prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
+        else
+            prep1PhaseAggregate(jobInfo, rgs, aggs);
+
+        // TODO: fix this
+        if (doUMOnly)
+            rgs.push_back(rgs[0]);
+    }
+    
+    if (!doUMOnly)
+    {
+        if (distinctAgg == true)
+            prep2PhasesDistinctAggregate(jobInfo, rgs, aggs);
+        else
+            prep2PhasesAggregate(jobInfo, rgs, aggs);
+    }
 
     if (tbps != NULL)
     {
-        // get rowgroup and aggregator
-        rgs.push_back(tbps->getDeliveredRowGroup());
-
-        if (jobInfo.groupConcatInfo.columns().size() == 0)
-        {
-            if (distinctAgg == true)
-                prep2PhasesDistinctAggregate(jobInfo, rgs, aggs);
-            else
-                prep2PhasesAggregate(jobInfo, rgs, aggs);
-        }
-        else
-        {
-            if (distinctAgg == true)
-                prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
-            else
-                prep1PhaseAggregate(jobInfo, rgs, aggs);
-
-            // TODO: fix this
-            rgs.push_back(rgs[0]);
-            doGroupConcat = true;
-        }
-
-        // make sure connected by a RowGroupDL
-        JobStepAssociation tbpsJsa;
-        AnyDataListSPtr spdl(new AnyDataList());
-        RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
-        dl->OID(execplan::CNX_VTABLE_ID);
-        spdl->rowGroupDL(dl);
-        tbpsJsa.outAdd(spdl);
-
         // create delivery step
         aggUM = dynamic_pointer_cast<RowAggregationUM>(aggs[0]);
         spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[2], jobInfo));
-        spjs->inputAssociation(tbpsJsa);
 
-        // step id??
+        if (doUMOnly)
-        spjs->stepId(step->stepId() + 1);
-
-        // set the PM/UM side aggregate structs
-        tbps->outputAssociation(tbpsJsa);
-
-        if (doGroupConcat)
             dynamic_cast<TupleAggregateStep*>(spjs.get())->umOnly(true);
         else
             tbps->setAggregateStep(aggs[1], rgs[2]);
     }
     else if (thjs != NULL)
     {
-        // default to UM aggregation
-        rgs.push_back(thjs->getDeliveredRowGroup());
-
-        if (distinctAgg == true)
-            prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
-        else
-            prep1PhaseAggregate(jobInfo, rgs, aggs);
-
-        // also prepare for PM aggregation
-        // rowgroups   -- 0-proj, 1-um, [2-phase case: 2-um, 3-pm]
-        // aggregators -- 0-um, [2-phase case: 1-um, 2-pm]
-        if (jobInfo.groupConcatInfo.columns().size() == 0)
-        {
-            if (distinctAgg == true)
-                prep2PhasesDistinctAggregate(jobInfo, rgs, aggs);
-            else
-                prep2PhasesAggregate(jobInfo, rgs, aggs);
-        }
-        else
-        {
-            // TODO: fix this
-            rgs.push_back(rgs[0]);
-            doGroupConcat = true;
-        }
-
-        // make sure connected by a RowGroupDL
-        JobStepAssociation thjsJsa;
-        AnyDataListSPtr spdl(new AnyDataList());
-        RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
-        dl->OID(execplan::CNX_VTABLE_ID);
-        spdl->rowGroupDL(dl);
-        thjsJsa.outAdd(spdl);
-
         // create delivery step
         aggUM = dynamic_pointer_cast<RowAggregationUM>(aggs[0]);
         spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo));
-        spjs->inputAssociation(thjsJsa);
 
-        if (doGroupConcat)
+        if (doUMOnly)
             dynamic_cast<TupleAggregateStep*>(spjs.get())->umOnly(true);
         else
             dynamic_cast<TupleAggregateStep*>(spjs.get())->savePmHJData(aggs[1], aggs[2], rgs[3]);
 
-
-        // step id??
-        spjs->stepId(step->stepId() + 1);
-
         // set input side
-        thjs->outputAssociation(thjsJsa);
         thjs->deliveryStep(spjs);
     }
-    else if (sas != NULL)
+    else
     {
-        // UM aggregation
-        // rowgroups   -- 0-proj, 1-um
-        // aggregators -- 0-um
-        rgs.push_back(sas->getDeliveredRowGroup());
-
-        if (distinctAgg == true)
-            prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
-        else
-            prep1PhaseAggregate(jobInfo, rgs, aggs);
-
-        // make sure connected by a RowGroupDL
-        JobStepAssociation sasJsa;
-        AnyDataListSPtr spdl(new AnyDataList());
-        RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
-        dl->OID(execplan::CNX_VTABLE_ID);
-        spdl->rowGroupDL(dl);
-        sasJsa.outAdd(spdl);
-
-        // create delivery step
         aggUM = dynamic_pointer_cast<RowAggregationUM>(aggs[0]);
         spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo));
-        spjs->inputAssociation(sasJsa);
-
-        // step id??
-        spjs->stepId(step->stepId() + 1);
-
-        // set input side
-        sas->outputAssociation(sasJsa);
     }
-    else if (ces != NULL)
-    {
-        // UM aggregation
-        // rowgroups   -- 0-proj, 1-um
-        // aggregators -- 0-um
-        rgs.push_back(ces->getDeliveredRowGroup());
 
-        if (distinctAgg == true)
+    // Setup the input JobstepAssoctiation -- the mechanism
-            prep1PhaseDistinctAggregate(jobInfo, rgs, aggs);
+    // whereby the previous step feeds data to this step.
-        else
+    // Otherwise, we need to create one and hook to the 
-            prep1PhaseAggregate(jobInfo, rgs, aggs);
+    // previous step as well as this aggregate step.
+    spjs->stepId(step->stepId() + 1);
 
-        // make sure connected by a RowGroupDL
+    JobStepAssociation jsa;
-        JobStepAssociation cesJsa;
+    AnyDataListSPtr spdl(new AnyDataList());
-        AnyDataListSPtr spdl(new AnyDataList());
+    RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
-        RowGroupDL* dl = new RowGroupDL(1, jobInfo.fifoSize);
+    dl->OID(execplan::CNX_VTABLE_ID);
-        dl->OID(execplan::CNX_VTABLE_ID);
+    spdl->rowGroupDL(dl);
-        spdl->rowGroupDL(dl);
+    jsa.outAdd(spdl);
-        cesJsa.outAdd(spdl);
 
-        // create delivery step
+    spjs->inputAssociation(jsa); // Aggregate input
-        aggUM = dynamic_pointer_cast<RowAggregationUM>(aggs[0]);
-        spjs.reset(new TupleAggregateStep(aggUM, rgs[1], rgs[0], jobInfo));
-        spjs->inputAssociation(cesJsa);
 
-        // step id??
+    //Previous step output
-        spjs->stepId(step->stepId() + 1);
+    step->outputAssociation(jsa);
-
-        // set input side
-        ces->outputAssociation(cesJsa);
-    }
 
     // add the aggregate on constants
     if (constAggDataVec.size() > 0)

dbcon/joblist/windowfunctionstep.cpp

@@ -477,6 +477,17 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J
         colSet.insert(key);
     }
 
+    // MCOL-3435 We haven't yet checked for aggregate, but we need to know
+    bool hasAggregation = false;
+    for (uint64_t i = 0; i < jobInfo.deliveredCols.size(); i++)
+    {
+        if (dynamic_cast<AggregateColumn*>(jobInfo.deliveredCols[i].get()) != NULL)
+        {
+            hasAggregation = true;
+            break;
+        }
+    }
+
     // add non-duplicate auxiliary columns
     RetColsVector colsInAf;
 
@@ -499,10 +510,30 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J
             if (colSet.find(key) == colSet.end())
             {
                 jobInfo.deliveredCols.push_back(*j);
-// MCOL-3343 Enable this if we decide to allow Window Functions to run with
+                // MCOL-3435 Allow Window Functions to run with aggregates with 
-// aggregates with no group by. MariaDB allows this. Nobody else in the world does.
+                // no group by by inserting a group by for window parameters.
-// There will be more work to get it to function if we try this.                
+                if (hasAggregation)
-//                jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true));
+                {
+                    // If an argument is an AggregateColumn, don't group by it.
+                    if (dynamic_cast<AggregateColumn*>(j->get()) == NULL)
+                    {
+                        bool bFound = false;
+                        for (std::vector<SRCP>::iterator igpc = csep->groupByCols().begin(); 
+                                                         igpc < csep->groupByCols().end();
+                                                         ++igpc)
+                        {
+                            if (*igpc->get() == *j->get())
+                            {
+                                bFound = true;
+                                break;
+                            }
+                        }
+                        if (!bFound)
+                        {
+                            csep->groupByCols().push_back(*j);
+                        }
+                    }
+                }
             }
 
             colSet.insert(key);

dbcon/mysql/ha_calpont_execplan.cpp

@@ -3160,7 +3160,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp
                     String val, *str = item->val_str(&val);
                     string valStr;
                     valStr.assign(str->ptr(), str->length());
-                    rc = new ConstantColumn(valStr);
+            rc = new ConstantColumn(valStr, ConstantColumn::NUM);
                     break;
                 }
                 case REAL_RESULT:

primitives/linux-port/primitiveprocessor.h

@@ -33,9 +33,9 @@
 #include <unordered_set>
 #endif
 
-#ifdef __linux__
+//#ifdef __linux__
-#define POSIX_REGEX
+//#define POSIX_REGEX
-#endif
+//#endif
 
 #ifdef POSIX_REGEX
 #include <regex.h>

procmon/processmonitor.cpp

@@ -2621,6 +2621,7 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName,
 //				DBRMroot = tempDBRMDir + "/BRM_saves";
             }
             
+            
             //
             // run the 'load_brm' script first if files exist
             //
@@ -4419,7 +4420,6 @@ int ProcessMonitor::getDBRMdata(string *path)
                         bf::create_directories(pTmp);
                     *path = pTmp.string();
                     log.writeLog(__LINE__, "Downloading DBRM files to " + *path, LOG_TYPE_DEBUG);
-                    
                     for ( int i = 0 ; i < numFiles ; i ++ )
                     {
                         string fileName;
@@ -6359,7 +6359,6 @@ int ProcessMonitor::checkDataMount()
         /*  StorageManager isn't running yet.  Can't check for writability here. */
         return API_SUCCESS;
     }
-        
     //go unmount disk NOT assigned to this pm
     unmountExtraDBroots();
 

utils/funcexp/func_bitwise.cpp

@@ -94,10 +94,6 @@ bool getUIntValFromParm(
             {
                 isNull = true;
             }
-            else
-            {
-                value = 0;
-            }
         }
         break;
 
@@ -111,9 +107,9 @@ bool getUIntValFromParm(
             {
                 d.value = 0;
             }
-
+            double dscale = d.scale;
-            int64_t tmpval = d.value / helpers::power(d.scale);
+            int64_t tmpval = d.value / pow(10.0, dscale);
-            int lefto = (d.value - tmpval * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            int lefto = (d.value - tmpval * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( tmpval >= 0 && lefto > 4 )
                 tmpval++;

utils/funcexp/func_cast.cpp

@@ -180,8 +180,9 @@ int64_t Func_cast_signed::getIntVal(Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            int64_t value = d.value / helpers::power(d.scale);
+            double dscale = d.scale;
-            int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            int64_t value = d.value / pow(10.0, dscale);
+            int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( value >= 0 && lefto > 4 )
                 value++;
@@ -337,14 +338,15 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
+            double dscale = d.scale;
 
             if (d.value < 0)
             {
                 return 0;
             }
 
-            uint64_t value = d.value / helpers::power(d.scale);
+            uint64_t value = d.value / pow(10.0, dscale);
-            int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( value >= 0 && lefto > 4 )
                 value++;

utils/funcexp/func_char.cpp

@@ -156,9 +156,10 @@ string Func_char::getStrVal(Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
+            double dscale = d.scale;
             // get decimal and round up
-            int value = d.value / helpers::power(d.scale);
+            int value = d.value / pow(10.0, dscale);
-            int lefto = (d.value - value * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( lefto > 4 )
                 value++;

utils/funcexp/func_div.cpp

@@ -51,6 +51,16 @@ int64_t Func_div::getIntVal(rowgroup::Row& row,
 {
     double val1 = parm[0]->data()->getDoubleVal(row, isNull);
     double val2 = parm[1]->data()->getDoubleVal(row, isNull);
+    
+    if (val2 == 0 || val2 == NAN)
+    {
+        isNull = true;
+        return 0;
+    }
+    // MCOL-179 InnoDB doesn't round or convert to int before dividing.
+    return static_cast<int64_t>(val1 / val2);
+
+#if 0    
     int64_t int_val2 = (int64_t)(val2 > 0 ? val2 + 0.5 : val2 - 0.5);
 
     if (int_val2 == 0)
@@ -69,6 +79,7 @@ int64_t Func_div::getIntVal(rowgroup::Row& row,
     }
 
     return int_val1 / int_val2;
+#endif    
 }
 
 

utils/funcexp/func_elt.cpp

@@ -71,8 +71,9 @@ string Func_elt::getStrVal(rowgroup::Row& row,
         case CalpontSystemCatalog::DECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            number = d.value / helpers::power(d.scale);
+            double dscale = d.scale;
-            int lefto = (d.value - number * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            number = d.value / pow(10.0, dscale);
+            int lefto = (d.value - number * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( number >= 0 && lefto > 4 )
                 number++;

utils/funcexp/func_makedate.cpp

@@ -68,8 +68,9 @@ uint64_t makedate(rowgroup::Row& row,
         case CalpontSystemCatalog::DECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            year = d.value / helpers::power(d.scale);
+            double dscale = d.scale;
-            int lefto = (d.value - year * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            year = d.value / pow(10.0, dscale);
+            int lefto = (d.value - year * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( year >= 0 && lefto > 4 )
                 year++;
@@ -127,8 +128,9 @@ uint64_t makedate(rowgroup::Row& row,
         case CalpontSystemCatalog::DECIMAL:
         {
             IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull);
-            int64_t tmp = d.value / helpers::power(d.scale);
+            double dscale = d.scale;
-            int lefto = (d.value - tmp * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            int64_t tmp = d.value / pow(10.0, dscale);
+            int lefto = (d.value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( tmp >= 0 && lefto > 4 )
                 tmp++;

utils/funcexp/func_maketime.cpp

@@ -74,8 +74,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row,
         case CalpontSystemCatalog::DECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            hour = d.value / helpers::power(d.scale);
+            double dscale = d.scale;
-            int lefto = (d.value - hour * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            hour = d.value / pow(10.0, dscale);
+            int lefto = (d.value - hour * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( hour >= 0 && lefto > 4 )
                 hour++;
@@ -113,8 +114,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row,
         case CalpontSystemCatalog::DECIMAL:
         {
             IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull);
-            min = d.value / helpers::power(d.scale);
+            double dscale = d.scale;
-            int lefto = (d.value - min * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            min = d.value / pow(10.0, dscale);
+            int lefto = (d.value - min * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( min >= 0 && lefto > 4 )
                 min++;
@@ -158,8 +160,9 @@ string Func_maketime::getStrVal(rowgroup::Row& row,
         case CalpontSystemCatalog::DECIMAL:
         {
             IDB_Decimal d = parm[2]->data()->getDecimalVal(row, isNull);
-            sec = d.value / helpers::power(d.scale);
+            double dscale = d.scale;
-            int lefto = (d.value - sec * helpers::power(d.scale)) / helpers::power(d.scale - 1);
+            sec = d.value / pow(10.0, dscale);
+            int lefto = (d.value - sec * pow(10.0, dscale)) / pow(10.0, dscale - 1);
 
             if ( sec >= 0 && lefto > 4 )
                 sec++;

utils/funcexp/func_mod.cpp

@@ -74,10 +74,10 @@ IDB_Decimal Func_mod::getDecimalVal(Row& row,
     }
 
     IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-    int64_t value = d.value / helpers::power(d.scale);
+    int64_t value = d.value / pow(10.0, d.scale);
-    int lefto = d.value % helpers::power(d.scale);
+    int lefto = d.value % (int)pow(10.0, d.scale);
 
-    int64_t mod = (value % div) * helpers::power(d.scale) + lefto;
+    int64_t mod = (value % div) * pow(10.0, d.scale) + lefto;
 
     retValue.value = mod;
     retValue.scale = d.scale;
@@ -164,7 +164,7 @@ double Func_mod::getDoubleVal(Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            int64_t value = d.value / helpers::power(d.scale);
+            int64_t value = d.value / pow(10.0, d.scale);
 
             mod = value % div;
         }
@@ -268,7 +268,7 @@ long double Func_mod::getLongDoubleVal(Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            int64_t value = d.value / helpers::power(d.scale);
+            int64_t value = d.value / pow(10.0, d.scale);
 
             mod = value % div;
         }
@@ -375,7 +375,7 @@ int64_t Func_mod::getIntVal(Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            int64_t value = d.value / helpers::power(d.scale);
+            int64_t value = d.value / pow(10.0, d.scale);
 
             mod = value % div;
         }
@@ -473,7 +473,7 @@ uint64_t Func_mod::getUIntVal(Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            int64_t value = d.value / helpers::power(d.scale);
+            int64_t value = d.value / pow(10.0, d.scale);
 
             mod = value % div;
         }

utils/funcexp/func_period_diff.cpp

@@ -86,7 +86,7 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-            period1 = d.value / helpers::power(d.scale);
+            period1 = d.value / pow(10.0, d.scale);
             break;
         }
 
@@ -135,7 +135,7 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row,
         case execplan::CalpontSystemCatalog::UDECIMAL:
         {
             IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull);
-            period2 = d.value / helpers::power(d.scale);
+            period2 = d.value / pow(10.0, d.scale);
             break;
         }
 

utils/funcexp/funcexp.cpp

@@ -46,6 +46,9 @@ namespace funcexp
 {
 namespace utf8
 {
+// A global loc object so we don't construct one at every compare
+std::locale loc;
+
 bool JPcodePoint = false;		// extern-ed in utils_utf8.h
 }
 

utils/funcexp/utils_utf8.h

@@ -36,7 +36,6 @@
 #include <cstdlib>
 
 #include <clocale>
-
 #include "liboamcpp.h"
 
 /** @file */
@@ -49,6 +48,10 @@ extern bool JPcodePoint;		// code point ordering (Japanese UTF) flag, used in id
 
 const int MAX_UTF8_BYTES_PER_CHAR = 4;
 
+// A global loc object so we don't construct one at every compare
+extern std::locale loc;
+// Is there a way to construct a global reference to a facet?
+// const std::collate<char>& coll = std::use_facet<std::collate<char> >(loc);
 
 //Infinidb version of strlocale  BUG 5362
 //set System Locale "C" by default
@@ -110,6 +113,10 @@ std::string idb_setlocale()
     if (systemLang.find("ja_JP") != std::string::npos)
         JPcodePoint = true;
 
+    // MCOL-1559 Save off the locale to save runtime cpus
+    std::locale localloc(systemLang.c_str());
+    loc = localloc;
+
     return systemLang;
 }
 
@@ -125,6 +132,51 @@ int idb_strcoll(const char* str1, const char* str2)
         return strcoll(str1, str2);
 }
 
+// MCOL-1559 Add a trimmed version of strcoll
+// The intent here is to make no copy of the original strings and
+// not modify them, so we can't use trim to deal with the spaces.
+inline
+int idb_strtrimcoll(const std::string& str1, const std::string& str2)
+{
+    static const std::string whitespaces (" ");
+    const char* s1 = str1.c_str();
+    const char* s2 = str2.c_str();
+
+    // Set found1 to the last non-whitespace char in str1
+    std::size_t found1 = str1.find_last_not_of(whitespaces);
+    // Set found2 to the first whitespace char in str2
+    std::size_t found2 = str2.find_last_not_of(whitespaces);
+
+     // Are both strings empty or all whitespace?
+    if (found1 == std::string::npos && found2 == std::string::npos)
+    {
+        return 0; // they match
+    }
+    // If str1 is empty or all spaces
+    if (found1 == std::string::npos)
+    {
+        return -1;
+    }
+    // If str2 is empty or all spaces
+    if (found2 == std::string::npos)
+    {
+        return 1;
+    }
+
+    // found1 and found2 point to the character that is not a space. 
+    // compare wants it to point to one past.
+    found1 += 1;
+    found2 += 1;
+    // If no trimming needs doing, then strcoll is faster
+    if (found1 == str1.size() && found2 == str2.size())
+    {
+        return idb_strcoll(s1, s2);
+    }
+    // Compare the (trimmed) strings
+    const std::collate<char>& coll = std::use_facet<std::collate<char> >(loc);
+    int rtn = coll.compare(s1, s1+found1, s2, s2+found2);
+    return rtn;
+}
 
 // BUG 5241
 // Infinidb specific mbstowcs(). This will handle both windows and unix platforms

utils/rowgroup/rowgroup.h

@@ -1035,12 +1035,13 @@ inline void Row::setFloatField(float val, uint32_t colIndex)
 
 inline void Row::setLongDoubleField(long double val, uint32_t colIndex)
 {
+    uint8_t* p = &data[offsets[colIndex]];
+    *((long double*)p) = val;
     if (sizeof(long double) == 16)
     {
         // zero out the unused portion as there may be garbage there.
-        *((uint64_t*)&val+1) &= 0x000000000000FFFFULL;
+        *((uint64_t*)p+1) &= 0x000000000000FFFFULL;
     }
-    *((long double*) &data[offsets[colIndex]]) = val;
 }
 
 inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex)

utils/windowfunction/wf_stats.cpp

@@ -140,6 +140,7 @@ void WF_stats<T>::resetData()
 template<typename T>
 void WF_stats<T>::operator()(int64_t b, int64_t e, int64_t c)
 {
+    CDT cdt;
     if ((fFrameUnit == WF__FRAME_ROWS) ||
             (fPrev == -1) ||
             (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev)))))
@@ -163,7 +164,7 @@ void WF_stats<T>::operator()(int64_t b, int64_t e, int64_t c)
                 continue;
 
             T valIn;
-            getValue(colIn, valIn);
+            getValue(colIn, valIn, &cdt);
             long double val = (long double) valIn;
 
             fSum1 += val;
@@ -177,7 +178,9 @@ void WF_stats<T>::operator()(int64_t b, int64_t e, int64_t c)
             int scale = fRow.getScale(colIn);
             long double factor = pow(10.0, scale);
 
-            if (scale != 0) // adjust the scale if necessary
+            // adjust the scale if necessary
+            if (scale != 0 &&
+                cdt != CalpontSystemCatalog::LONGDOUBLE)
             {
                 fSum1 /= factor;
                 fSum2 /= factor * factor;

utils/windowfunction/wf_sum_avg.cpp

@@ -264,13 +264,15 @@ void WF_sum_avg<T>::operator()(int64_t b, int64_t e, int64_t c)
                 continue;
 
             T valIn;
-            getValue(colIn, valIn);
+            CDT cdt;
+            getValue(colIn, valIn, &cdt);
 //            checkSumLimit(fSum, valIn);
 
             if ((!fDistinct) || (fSet.find(valIn) == fSet.end()))
             {
                 long double val = valIn;
-                if (scale)
+                if (scale && 
+                    cdt != CalpontSystemCatalog::LONGDOUBLE)
                 {
                     val /= pow(10.0, scale);
                 }