From fa0947caaaa294ffbcd3115b2c9fd7cec47b377b Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:47:20 +0100 Subject: [PATCH 01/17] Revert "MCOL-1559 remove unused boost/trim header" This reverts commit 28e743bf38e6cc143d21e047cfcc7af105a665c5. --- dbcon/joblist/jlf_execplantojoblist.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index fff1b12fb..e9068b5e8 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -36,6 +36,7 @@ using namespace std; #include #include #include +#include namespace ba = boost::algorithm; #include "calpontexecutionplan.h" From 182e31e2829cc26aa4bf5103a64f1abe55951264 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:47:42 +0100 Subject: [PATCH 02/17] Revert "MCOL-1559 backout trim before setting compare. Shouldn't do this in all cases." This reverts commit d78944d9ff894f677b63726d3e4f0a89763c8521. --- dbcon/joblist/jlf_execplantojoblist.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index e9068b5e8..2cad5870e 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1674,6 +1674,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) pds->cardinality(sc->cardinality()); //Add the filter + // MCOL-1559 trim before adding. + boost::trim_right_if(constval, boost::is_any_of(" ")); pds->addFilter(cop, constval); // data list for pcolstep output From 8b1c0f610164cc8ca14e1f701a5f98412d82fc71 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:00 +0100 Subject: [PATCH 03/17] Revert "MCOL-1559 trim constant varchar string before adding filter" This reverts commit b6484dda4aa5703606cf74fe77e29101625cff33. --- dbcon/joblist/jlf_execplantojoblist.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 2cad5870e..4bd41530d 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1636,6 +1636,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); +// boost::trim_right_if(constval, boost::is_any_of(" ")); + CalpontSystemCatalog::OID dictOid = 0; CalpontSystemCatalog::ColType ct = sc->colType(); @@ -1674,8 +1676,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) pds->cardinality(sc->cardinality()); //Add the filter - // MCOL-1559 trim before adding. - boost::trim_right_if(constval, boost::is_any_of(" ")); pds->addFilter(cop, constval); // data list for pcolstep output From 73f7307ca264d775c9b636bbadb813df5da3f998 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:18 +0100 Subject: [PATCH 04/17] Revert "MCOL-1559 trailing space compare" This reverts commit e2cb6444843494e62ae8eacb0a1c3425400b7f69. --- dbcon/execplan/predicateoperator.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 08f0c40cf..daa078948 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -456,8 +456,18 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu isNull = false; return !ret; } +#if 0 + if (isNull) + return false; - // MCOL-1559 + const std::string& val1 = lop->getStrVal(row, isNull); + + if (isNull) + return false; + + return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; +#endif + // MCOL-1559 std::string val1 = lop->getStrVal(row, isNull); if (isNull) return false; From f3449fe2f47c3f1f15ba88d4e65ae463b8fab3ae Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:31 +0100 Subject: [PATCH 05/17] Revert "MCOL-1559 trailing space compare" This reverts commit 5b581f53cbe1c0a7b27ac63fdddcffeccd866103. --- dbcon/execplan/predicateoperator.h | 12 ++---------- dbcon/joblist/jlf_execplantojoblist.cpp | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index daa078948..70209df5b 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -456,19 +456,11 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu isNull = false; return !ret; } -#if 0 - if (isNull) - return false; - - const std::string& val1 = lop->getStrVal(row, isNull); if (isNull) return false; - return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; -#endif - // MCOL-1559 - std::string val1 = lop->getStrVal(row, isNull); + std::string val1 = lop->getStrVal(row, isNull); if (isNull) return false; @@ -480,7 +472,7 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu boost::trim_right_if(val2, boost::is_any_of(" ")); return strCompare(val1, val2); - } + } //FIXME: ??? case execplan::CalpontSystemCatalog::VARBINARY: diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 4bd41530d..24ec7f50e 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1636,7 +1636,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); -// boost::trim_right_if(constval, boost::is_any_of(" ")); + boost::trim_right_if(constval, boost::is_any_of(" ")); CalpontSystemCatalog::OID dictOid = 0; From e7a331f3f86290972973f717a4dc2e235c1c9ced Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:43 +0100 Subject: [PATCH 06/17] Revert "MCOL-1559 trim spaces before compare" This reverts commit 8d553ae9fbbe8ddcc984966a45f632a1896fdf81. --- dbcon/execplan/predicateoperator.h | 4 ++-- dbcon/mysql/ha_calpont_execplan.cpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 70209df5b..f81f189bb 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -460,11 +460,11 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu if (isNull) return false; - std::string val1 = lop->getStrVal(row, isNull); + const std::string& val1 = lop->getStrVal(row, isNull); if (isNull) return false; - std::string val2 = rop->getStrVal(row, isNull); + const std::string& val2 = rop->getStrVal(row, isNull); if (isNull) return false; diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 65d0ac4a6..239ae9f0c 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -49,6 +49,7 @@ using namespace std; #include #include +#include #include #include @@ -4868,6 +4869,7 @@ void gp_walk(const Item* item, void* arg) { cval.assign(str->ptr(), str->length()); } +// boost::trim_right_if(cval, boost::is_any_of(" ")); gwip->rcWorkStack.push(new ConstantColumn(cval)); break; From 922e828d6c52a722ba9c4f74c16269de301facba Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:56 +0100 Subject: [PATCH 07/17] Revert "MCOL-1559 trailing white space comparison" This reverts commit aa802f44c534a21ff447b3025ae3e07ac323e337. --- dbcon/execplan/predicateoperator.h | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index f81f189bb..b83931394 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -36,7 +36,6 @@ #endif #include #include -#include #include "expressionparser.h" #include "returnedcolumn.h" @@ -461,17 +460,11 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return false; const std::string& val1 = lop->getStrVal(row, isNull); + if (isNull) return false; - const std::string& val2 = rop->getStrVal(row, isNull); - if (isNull) - return false; - - boost::trim_right_if(val1, boost::is_any_of(" ")); - boost::trim_right_if(val2, boost::is_any_of(" ")); - - return strCompare(val1, val2); + return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; } //FIXME: ??? From 9390ee05fb74c7edb28ff4f3e80bdb2f4b62aeaf Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:49:08 +0100 Subject: [PATCH 08/17] Revert "MCOL-1559 Some string trailing blank stuff" This reverts commit e5d76e142be7edcbbf0817f5bea594ddffc84973. --- dbcon/joblist/jlf_execplantojoblist.cpp | 2 -- dbcon/joblist/lbidlist.cpp | 1 - dbcon/mysql/ha_calpont_execplan.cpp | 2 -- primitives/primproc/dictstep.cpp | 2 -- primitives/primproc/primitiveserver.cpp | 2 -- 5 files changed, 9 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 24ec7f50e..f3782c9d5 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -36,7 +36,6 @@ using namespace std; #include #include #include -#include namespace ba = boost::algorithm; #include "calpontexecutionplan.h" @@ -1636,7 +1635,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); - boost::trim_right_if(constval, boost::is_any_of(" ")); CalpontSystemCatalog::OID dictOid = 0; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 7852562ef..c317defc9 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -749,7 +749,6 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, int64_t tMax = Max; dataconvert::DataConvert::trimWhitespace(tMin); dataconvert::DataConvert::trimWhitespace(tMax); - dataconvert::DataConvert::trimWhitespace(value); scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value), op, lcf); diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 239ae9f0c..65d0ac4a6 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -49,7 +49,6 @@ using namespace std; #include #include -#include #include #include @@ -4869,7 +4868,6 @@ void gp_walk(const Item* item, void* arg) { cval.assign(str->ptr(), str->length()); } -// boost::trim_right_if(cval, boost::is_any_of(" ")); gwip->rcWorkStack.push(new ConstantColumn(cval)); break; diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index 47bfeac25..abd99ada3 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -30,7 +30,6 @@ #include #include -#include #include "bpp.h" #include "primitiveserver.h" @@ -94,7 +93,6 @@ void DictStep::createCommand(ByteStream& bs) for (uint32_t i = 0; i < filterCount; i++) { bs >> strTmp; - boost::trim_right_if(strTmp, boost::is_any_of(" ")); //cout << " " << strTmp << endl; eqFilter->insert(strTmp); } diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index 54f4b7fb6..1e497122e 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -28,7 +28,6 @@ #include #include #include -#include //#define NDEBUG #include #include @@ -1778,7 +1777,6 @@ private: for (i = 0; i < count; i++) { *bs >> str; - boost::trim_right_if(str, boost::is_any_of(" ")); filter->insert(str); } From 6b2f67a23747942861d347d15f5cab7c55bc70a5 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 23 May 2019 14:15:58 -0500 Subject: [PATCH 09/17] MCOL-3314 Use something useful for stats functions oid and key during prep1PhaseAggregate --- dbcon/joblist/tupleaggregatestep.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 143af5de0..396e6a6ae 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -1583,8 +1583,8 @@ void TupleAggregateStep::prep1PhaseAggregate( } functionVec[i]->fAuxColumnIndex = lastCol++; - oidsAgg.push_back(oidsAgg[j]); - keysAgg.push_back(keysAgg[j]); + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); scaleAgg.push_back(0); precisionAgg.push_back(0); precisionAgg.push_back(0); @@ -1599,8 +1599,8 @@ void TupleAggregateStep::prep1PhaseAggregate( functionVec[i]->fAuxColumnIndex = lastCol; // sum(x) - oidsAgg.push_back(oidsAgg[j]); - keysAgg.push_back(keysAgg[j]); + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); scaleAgg.push_back(0); precisionAgg.push_back(-1); typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -1608,8 +1608,8 @@ void TupleAggregateStep::prep1PhaseAggregate( ++lastCol; // sum(x**2) - oidsAgg.push_back(oidsAgg[j]); - keysAgg.push_back(keysAgg[j]); + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); scaleAgg.push_back(0); precisionAgg.push_back(-1); typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); From 42f29096f0cc9f639ddc46e0ea0854eebdcf3d6b Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 23 May 2019 22:22:55 +0300 Subject: [PATCH 10/17] MCOL-3239 CS pushes relevant filter predicates into derived tables. This change disables predicate push for function column that contains both derived table column and non-derived table column. --- dbcon/execplan/functioncolumn.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dbcon/execplan/functioncolumn.cpp b/dbcon/execplan/functioncolumn.cpp index 487d42c0b..665e4a4c3 100644 --- a/dbcon/execplan/functioncolumn.cpp +++ b/dbcon/execplan/functioncolumn.cpp @@ -420,6 +420,16 @@ void FunctionColumn::setDerivedTable() break; } } + // MCOL-3239 Block for func column with both + // derived table column and normal table column. + else if (derivedTableAlias == "") + { + if (sc->tableAlias().length()) + { + derivedTableAlias = ""; + break; + } + } } fDerivedTable = derivedTableAlias; From 23b594eb284add47f3b97a6f20dd00eb2cf3a640 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 28 May 2019 15:56:30 -0500 Subject: [PATCH 11/17] MCOL-3304 During prepAggregate, if you set type to LONGDOUBLE, set scale to 0 --- dbcon/execplan/windowfunctioncolumn.cpp | 1 + dbcon/joblist/tupleaggregatestep.cpp | 36 ++++++++++++------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index e01c5c010..5c3e57407 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -386,6 +386,7 @@ void WindowFunctionColumn::adjustResultType() fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; fResultType.colWidth = sizeof(long double); fResultType.precision = -1; + fResultType.scale = 0; } } diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 143af5de0..23a8e8782 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -1406,7 +1406,7 @@ void TupleAggregateStep::prep1PhaseAggregate( typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAgg.push_back(-1); widthAgg.push_back(sizeof(long double)); - scaleAgg.push_back(scaleProj[colProj]); + scaleAgg.push_back(0); } break; @@ -1935,7 +1935,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAgg.push_back(-1); widthAgg.push_back(sizeof(long double)); - scaleAgg.push_back(scaleProj[colProj]); + scaleAgg.push_back(0); colAgg++; // has distinct step, put the count column for avg next to the sum @@ -2255,7 +2255,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAggDist.push_back(-1); widthAggDist.push_back(sizeof(long double)); - scaleAggDist.push_back(scaleProj[colAgg]); + scaleAggDist.push_back(0); } break; @@ -2327,10 +2327,10 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( { oidsAggDist.push_back(oidsAgg[colAgg]); keysAggDist.push_back(retKey); - scaleAggDist.push_back(scaleAgg[colAgg] >> 8); - precisionAggDist.push_back(precisionAgg[colAgg]); - typeAggDist.push_back(typeAgg[colAgg]); - widthAggDist.push_back(widthAgg[colAgg]); + scaleAggDist.push_back(0); + typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); + precisionAggDist.push_back(-1); + widthAggDist.push_back(sizeof(long double)); } else { @@ -3173,7 +3173,7 @@ void TupleAggregateStep::prep2PhasesAggregate( oidsAggPm.push_back(oidsProj[colProj]); keysAggPm.push_back(aggKey); typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); - scaleAggPm.push_back(scaleProj[colProj]); + scaleAggPm.push_back(0); precisionAggPm.push_back(-1); widthAggPm.push_back(sizeof(long double)); colAggPm++; @@ -3449,10 +3449,10 @@ void TupleAggregateStep::prep2PhasesAggregate( { oidsAggUm.push_back(oidsAggPm[colPm]); keysAggUm.push_back(retKey); - scaleAggUm.push_back(scaleAggPm[colPm] >> 8); - precisionAggUm.push_back(precisionAggPm[colPm]); - typeAggUm.push_back(typeAggPm[colPm]); - widthAggUm.push_back(widthAggPm[colPm]); + scaleAggUm.push_back(0); + typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); + precisionAggUm.push_back(-1); + widthAggUm.push_back(sizeof(long double)); } else { @@ -4027,7 +4027,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAggPm.push_back(-1); widthAggPm.push_back(sizeof(long double)); - scaleAggPm.push_back(scaleProj[colProj]); + scaleAggPm.push_back(0); colAggPm++; } @@ -4394,7 +4394,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAggDist.push_back(-1); widthAggDist.push_back(sizeof(long double)); - scaleAggDist.push_back(scaleAggUm[colUm]); + scaleAggDist.push_back(0); } // PM: put the count column for avg next to the sum // let fall through to add a count column for average function @@ -4456,10 +4456,10 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( { oidsAggDist.push_back(oidsAggUm[colUm]); keysAggDist.push_back(retKey); - scaleAggDist.push_back(scaleAggUm[colUm] >> 8); - precisionAggDist.push_back(precisionAggUm[colUm]); - typeAggDist.push_back(typeAggUm[colUm]); - widthAggDist.push_back(widthAggUm[colUm]); + scaleAggDist.push_back(0); + typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); + precisionAggDist.push_back(-1); + widthAggDist.push_back(sizeof(long double)); } else { From fff18e0dd41f22e97b3ca78497e7cb2f20c924ea Mon Sep 17 00:00:00 2001 From: David Mott Date: Mon, 3 Jun 2019 03:36:48 -0500 Subject: [PATCH 12/17] fix to possible memory corruption --- dbcon/execplan/arithmeticcolumn.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index aab6c9265..2128a47ed 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -326,7 +326,8 @@ void ArithmeticColumn::serialize(messageqcpp::ByteStream& b) const ObjectReader::writeParseTree(fExpression, b); b << fTableAlias; b << fData; - b << static_cast(fAsc); + const ByteStream::doublebyte tmp = fAsc; + b << tmp; } void ArithmeticColumn::unserialize(messageqcpp::ByteStream& b) @@ -340,7 +341,9 @@ void ArithmeticColumn::unserialize(messageqcpp::ByteStream& b) fExpression = ObjectReader::createParseTree(b); b >> fTableAlias; b >> fData; - b >> reinterpret_cast< ByteStream::doublebyte&>(fAsc); + ByteStream::doublebyte tmp; + b >> tmp; + fAsc = (tmp); fSimpleColumnList.clear(); fExpression->walk(getSimpleCols, &fSimpleColumnList); From 0983c1becf0e66de8b06024e6504f4ffb7dc245a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 10 Jun 2019 16:46:24 +0100 Subject: [PATCH 13/17] MCOL-2230 Fix TIMEDIFF() handling TIMEDIFF would not compare datetime with string correctly, causing a NULL saturation. --- utils/funcexp/func_timediff.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 369bb80a1..c4de35a0d 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -118,7 +118,10 @@ string Func_timediff::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: - if (type1 != type2) + // Diff between time and datetime returns NULL in MariaDB + if ((type2 == execplan::CalpontSystemCatalog::TIME || + type2 == execplan::CalpontSystemCatalog::DATETIME) && + type1 != type2) { isNull = true; break; From ea6592ea080640252f5eb2ff07c764ef3a350a31 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 12 Jun 2019 14:17:31 -0500 Subject: [PATCH 14/17] MCOL-3343 Try adding aggregate result to jobInfo.nonConstDelCol vector --- dbcon/joblist/joblistfactory.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 21df156a7..0a141f03a 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -397,6 +397,7 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) void preProcessFunctionOnAggregation(const vector& scs, const vector& aggs, + const vector& wcs, JobInfo& jobInfo) { // append the simple columns if not already projected @@ -426,10 +427,14 @@ void preProcessFunctionOnAggregation(const vector& scs, } } - // append the aggregate columns in arithmetic/function cloulmn to the projection list + // append the aggregate columns in arithmetic/function column to the projection list for (vector::const_iterator i = aggs.begin(); i != aggs.end(); i++) { addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo); + if (wcs.size() > 0) + { + jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone())); + } } } @@ -481,12 +486,12 @@ void checkReturnedColumns(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) if (ac != NULL && ac->aggColumnList().size() > 0) { jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), jobInfo); + preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), ac->windowfunctionColumnList(), jobInfo); } else if (fc != NULL && fc->aggColumnList().size() > 0) { jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), jobInfo); + preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), fc->windowfunctionColumnList(), jobInfo); } } } From e3e59dd167658810218a4f6c5c4b49b69ff6afba Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Mon, 17 Jun 2019 12:33:15 -0500 Subject: [PATCH 15/17] Update Version to 1.2.5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 91bc7ea08..f67d0ae71 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=2 -COLUMNSTORE_VERSION_PATCH=4 +COLUMNSTORE_VERSION_PATCH=5 COLUMNSTORE_VERSION_RELEASE=1 From 4cf2c37c1801de6c7cce02d14c439b976e17428e Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 20 Jun 2019 09:24:58 -0500 Subject: [PATCH 16/17] MCOL-3343 Handle windowfunction simple column --- dbcon/execplan/windowfunctioncolumn.cpp | 1 - dbcon/joblist/jlf_common.cpp | 58 ++++++++++++++++++++----- dbcon/joblist/jlf_common.h | 7 +-- dbcon/joblist/joblistfactory.cpp | 20 +++++++-- dbcon/joblist/tupleaggregatestep.cpp | 6 ++- dbcon/joblist/windowfunctionstep.cpp | 52 ++++++++++++++++++++++ dbcon/joblist/windowfunctionstep.h | 2 + 7 files changed, 126 insertions(+), 20 deletions(-) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index 5c3e57407..e01c5c010 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -386,7 +386,6 @@ void WindowFunctionColumn::adjustResultType() fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; fResultType.colWidth = sizeof(long double); fResultType.precision = -1; - fResultType.scale = 0; } } diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 7ce066427..33e7e9657 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -386,19 +386,57 @@ CalpontSystemCatalog::OID tableOid(const SimpleColumn* sc, boost::shared_ptr(sc); uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN; - return getTupleKey_(jobInfo, sc->oid(), sc->columnName(), extractTableAlias(sc), - sc->schemaName(), sc->viewName(), - ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0), - pseudoType, (sc->isInfiniDB() ? 0 : 1)); + if (sc == NULL) + { + return -1; + } + + if (add) + { + // setTupleInfo first if add is true, ok if already set. + if (sc->schemaName().empty()) + { + SimpleColumn tmp(*sc, jobInfo.sessionId); + tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + key = getTupleKey(jobInfo, &tmp); // sub-query should be there + } + else + { + CalpontSystemCatalog::ColType ct = sc->colType(); + string alias(extractTableAlias(sc)); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); + key = ti.key; + + CalpontSystemCatalog::OID dictOid = isDictCol(ct); + + if (dictOid > 0) + { + ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); + jobInfo.keyInfo->dictKeyMap[key] = ti.key; + key = ti.key; + } + } + } + else + { + // TupleInfo is expected to be set already + return getTupleKey_(jobInfo, sc->oid(), sc->columnName(), extractTableAlias(sc), + sc->schemaName(), sc->viewName(), + ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0), + pseudoType, (sc->isInfiniDB() ? 0 : 1)); + } + + return key; } - - + uint32_t getTupleKey(JobInfo& jobInfo, const SRCP& srcp, bool add) { int key = -1; @@ -608,7 +646,7 @@ uint32_t getExpTupleKey(const JobInfo& jobInfo, uint64_t eid, bool cr) } -void addAggregateColumn(AggregateColumn* agc, int idx, RetColsVector& vec, JobInfo& jobInfo) +void addAggregateColumn(ReturnedColumn* agc, int idx, RetColsVector& vec, JobInfo& jobInfo) { uint32_t eid = agc->expressionId(); setExpTupleInfo(agc->resultType(), eid, agc->alias(), jobInfo); diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index 18c1ea396..6cf28af8c 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -401,8 +401,9 @@ execplan::CalpontSystemCatalog::OID tableOid(const execplan::SimpleColumn* sc, /** @brief Returns the unique ID to be used in tupleInfo * */ -uint32_t getTupleKey(const JobInfo& jobInfo, - const execplan::SimpleColumn* sc); +uint32_t getTupleKey(JobInfo& jobInfo, + const execplan::SimpleColumn* sc, + bool add = false); uint32_t getTableKey(const JobInfo& jobInfo, execplan::CalpontSystemCatalog::OID tableOid, const std::string& alias, @@ -464,7 +465,7 @@ TupleInfo setExpTupleInfo(const execplan::ReturnedColumn* rc, JobInfo& jobInfo); /** @brief add an aggregate column info * */ -void addAggregateColumn(execplan::AggregateColumn*, int, RetColsVector&, JobInfo&); +void addAggregateColumn(execplan::ReturnedColumn*, int, RetColsVector&, JobInfo&); void makeJobSteps(execplan::CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, JobStepVector& projectSteps, diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 0a141f03a..66245a8e2 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -274,9 +274,11 @@ const JobStepVector doProject(const RetColsVector& retCols, JobInfo& jobInfo) if (retCols[i]->windowfunctionColumnList().size() > 0) jobInfo.expressionVec.push_back(key); - else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) == - jobInfo.expressionVec.end()) + else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) + == jobInfo.expressionVec.end()) + { jobInfo.returnedExpressions.push_back(sjstep); + } //put place hold column in projection list jobInfo.pjColList.push_back(ti); @@ -1047,16 +1049,21 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo const FunctionColumn* fc = NULL; const WindowFunctionColumn* wc = NULL; bool hasAggCols = false; + bool hasWndCols = false; if ((ac = dynamic_cast(srcp.get())) != NULL) { if (ac->aggColumnList().size() > 0) hasAggCols = true; + if (ac->windowfunctionColumnList().size() > 0) + hasWndCols = true; } else if ((fc = dynamic_cast(srcp.get())) != NULL) { if (fc->aggColumnList().size() > 0) hasAggCols = true; + if (fc->windowfunctionColumnList().size() > 0) + hasWndCols = true; } else if (dynamic_cast(srcp.get()) != NULL) { @@ -1081,7 +1088,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo)); tupleKey = ti.key; - if (hasAggCols) + if (hasAggCols && !hasWndCols) jobInfo.expressionVec.push_back(tupleKey); } @@ -1195,16 +1202,21 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo const FunctionColumn* fc = NULL; const WindowFunctionColumn* wc = NULL; bool hasAggCols = false; + bool hasWndCols = false; if ((ac = dynamic_cast(srcp.get())) != NULL) { if (ac->aggColumnList().size() > 0) hasAggCols = true; + if (ac->windowfunctionColumnList().size() > 0) + hasWndCols = true; } else if ((fc = dynamic_cast(srcp.get())) != NULL) { if (fc->aggColumnList().size() > 0) hasAggCols = true; + if (fc->windowfunctionColumnList().size() > 0) + hasWndCols = true; } else if (dynamic_cast(srcp.get()) != NULL) { @@ -1229,7 +1241,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo)); tupleKey = ti.key; - if (hasAggCols) + if (hasAggCols && !hasWndCols) jobInfo.expressionVec.push_back(tupleKey); } diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index db5a4e8d8..62f084412 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5011,7 +5011,8 @@ void TupleAggregateStep::prepExpressionOnAggregate(SP_ROWAGG_UM_t& aggUM, JobInf uint64_t eid = -1; if (((ac = dynamic_cast(it->get())) != NULL) && - (ac->aggColumnList().size() > 0)) + (ac->aggColumnList().size() > 0) && + (ac->windowfunctionColumnList().size() == 0)) { const vector& scols = ac->simpleColumnList(); simpleColumns.insert(simpleColumns.end(), scols.begin(), scols.end()); @@ -5020,7 +5021,8 @@ void TupleAggregateStep::prepExpressionOnAggregate(SP_ROWAGG_UM_t& aggUM, JobInf expressionVec.push_back(*it); } else if (((fc = dynamic_cast(it->get())) != NULL) && - (fc->aggColumnList().size() > 0)) + (fc->aggColumnList().size() > 0) && + (fc->windowfunctionColumnList().size() == 0)) { const vector& sCols = fc->simpleColumnList(); simpleColumns.insert(simpleColumns.end(), sCols.begin(), sCols.end()); diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index fe43faa77..172a7a8a5 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -320,6 +320,38 @@ const string WindowFunctionStep::toString() const return oss.str(); } +void WindowFunctionStep::AddSimplColumn(const vector& scs, + JobInfo& jobInfo) +{ + // append the simple columns if not already projected + set scProjected; + + for (RetColsVector::iterator i = jobInfo.projectionCols.begin(); + i != jobInfo.projectionCols.end(); + i++) + { + SimpleColumn* sc = dynamic_cast(i->get()); + + if (sc != NULL) + { + if (sc->schemaName().empty()) + sc->oid(joblist::tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + + scProjected.insert(UniqId(sc)); + } + } + + for (vector::const_iterator i = scs.begin(); i != scs.end(); i++) + { + if (scProjected.find(UniqId(*i)) == scProjected.end()) + { + jobInfo.windowDels.push_back(SRCP((*i)->clone())); + jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); + scProjected.insert(UniqId(*i)); + } + } +} + void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { // window functions in select clause, selected or in expression @@ -404,6 +436,23 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J if (jobInfo.windowCols.empty()) return; + // Add in the non-window side of arithmetic columns and functions + for (uint64_t i = 0; i < jobInfo.windowExps.size(); i++) + { + const ArithmeticColumn* ac = + dynamic_cast(jobInfo.windowExps[i].get()); + const FunctionColumn* fc = + dynamic_cast(jobInfo.windowExps[i].get()); + + if (ac != NULL && ac->windowfunctionColumnList().size() > 0) + { + AddSimplColumn(ac->simpleColumnList(), jobInfo); + } + else if (fc != NULL && fc->windowfunctionColumnList().size() > 0) + { + AddSimplColumn(fc->simpleColumnList(), jobInfo); + } + } // reconstruct the delivered column list with auxiliary columns set colSet; jobInfo.deliveredCols.resize(0); @@ -445,7 +494,10 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J key = getTupleKey(jobInfo, *j, true); if (colSet.find(key) == colSet.end()) + { jobInfo.deliveredCols.push_back(*j); + jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true)); + } colSet.insert(key); } diff --git a/dbcon/joblist/windowfunctionstep.h b/dbcon/joblist/windowfunctionstep.h index f483c6027..f6a695a65 100644 --- a/dbcon/joblist/windowfunctionstep.h +++ b/dbcon/joblist/windowfunctionstep.h @@ -148,6 +148,8 @@ private: void formatMiniStats(); void printCalTrace(); + static void AddSimplColumn(const vector& scs, JobInfo& jobInfo); + class Runner { public: From 2255d41eaf057d5e91be26c3c483aa813f74d35f Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 20 Jun 2019 15:54:05 -0500 Subject: [PATCH 17/17] MCOL-3343 Error when no group by for unaggregated fields in window functions. --- dbcon/joblist/windowfunctionstep.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index 172a7a8a5..1a603a98e 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -346,7 +346,10 @@ void WindowFunctionStep::AddSimplColumn(const vector& scs, if (scProjected.find(UniqId(*i)) == scProjected.end()) { jobInfo.windowDels.push_back(SRCP((*i)->clone())); - jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); +// MCOL-3343 Enable this if we decide to allow Window Functions to run with +// aggregates with no group by. MariaDB allows this. Nobody else in the world does. +// There will be more work to get it to function if we try this. +// jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); scProjected.insert(UniqId(*i)); } } @@ -496,7 +499,10 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J if (colSet.find(key) == colSet.end()) { jobInfo.deliveredCols.push_back(*j); - jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true)); +// MCOL-3343 Enable this if we decide to allow Window Functions to run with +// aggregates with no group by. MariaDB allows this. Nobody else in the world does. +// There will be more work to get it to function if we try this. +// jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true)); } colSet.insert(key);