1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00

Merge branch 'develop-1.2' into MCOL-1559

Conflicts:
	dbcon/execplan/predicateoperator.h
This commit is contained in:
David Hall
2019-06-24 17:08:33 -05:00
15 changed files with 185 additions and 56 deletions

View File

@ -1,4 +1,4 @@
COLUMNSTORE_VERSION_MAJOR=1
COLUMNSTORE_VERSION_MINOR=2
COLUMNSTORE_VERSION_PATCH=4
COLUMNSTORE_VERSION_PATCH=5
COLUMNSTORE_VERSION_RELEASE=1

View File

@ -326,7 +326,8 @@ void ArithmeticColumn::serialize(messageqcpp::ByteStream& b) const
ObjectReader::writeParseTree(fExpression, b);
b << fTableAlias;
b << fData;
b << static_cast<const ByteStream::doublebyte>(fAsc);
const ByteStream::doublebyte tmp = fAsc;
b << tmp;
}
void ArithmeticColumn::unserialize(messageqcpp::ByteStream& b)
@ -340,7 +341,9 @@ void ArithmeticColumn::unserialize(messageqcpp::ByteStream& b)
fExpression = ObjectReader::createParseTree(b);
b >> fTableAlias;
b >> fData;
b >> reinterpret_cast< ByteStream::doublebyte&>(fAsc);
ByteStream::doublebyte tmp;
b >> tmp;
fAsc = (tmp);
fSimpleColumnList.clear();
fExpression->walk(getSimpleCols, &fSimpleColumnList);

View File

@ -420,6 +420,16 @@ void FunctionColumn::setDerivedTable()
break;
}
}
// MCOL-3239 Block for func column with both
// derived table column and normal table column.
else if (derivedTableAlias == "")
{
if (sc->tableAlias().length())
{
derivedTableAlias = "";
break;
}
}
}
fDerivedTable = derivedTableAlias;

View File

@ -36,7 +36,6 @@
#endif
#include <cstring>
#include <boost/regex.hpp>
#include <boost/algorithm/string/trim.hpp>
#include "expressionparser.h"
#include "returnedcolumn.h"

View File

@ -386,19 +386,57 @@ CalpontSystemCatalog::OID tableOid(const SimpleColumn* sc, boost::shared_ptr<Cal
return p.objnum;
}
uint32_t getTupleKey(const JobInfo& jobInfo,
const execplan::SimpleColumn* sc)
uint32_t getTupleKey(JobInfo& jobInfo,
const execplan::SimpleColumn* sc,
bool add)
{
int key = -1;
const PseudoColumn* pc = dynamic_cast<const execplan::PseudoColumn*>(sc);
uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN;
return getTupleKey_(jobInfo, sc->oid(), sc->columnName(), extractTableAlias(sc),
sc->schemaName(), sc->viewName(),
((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0),
pseudoType, (sc->isInfiniDB() ? 0 : 1));
if (sc == NULL)
{
return -1;
}
if (add)
{
// setTupleInfo first if add is true, ok if already set.
if (sc->schemaName().empty())
{
SimpleColumn tmp(*sc, jobInfo.sessionId);
tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition());
key = getTupleKey(jobInfo, &tmp); // sub-query should be there
}
else
{
CalpontSystemCatalog::ColType ct = sc->colType();
string alias(extractTableAlias(sc));
CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc);
TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias));
key = ti.key;
CalpontSystemCatalog::OID dictOid = isDictCol(ct);
if (dictOid > 0)
{
ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias);
jobInfo.keyInfo->dictKeyMap[key] = ti.key;
key = ti.key;
}
}
}
else
{
// TupleInfo is expected to be set already
return getTupleKey_(jobInfo, sc->oid(), sc->columnName(), extractTableAlias(sc),
sc->schemaName(), sc->viewName(),
((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0),
pseudoType, (sc->isInfiniDB() ? 0 : 1));
}
return key;
}
uint32_t getTupleKey(JobInfo& jobInfo, const SRCP& srcp, bool add)
{
int key = -1;
@ -608,7 +646,7 @@ uint32_t getExpTupleKey(const JobInfo& jobInfo, uint64_t eid, bool cr)
}
void addAggregateColumn(AggregateColumn* agc, int idx, RetColsVector& vec, JobInfo& jobInfo)
void addAggregateColumn(ReturnedColumn* agc, int idx, RetColsVector& vec, JobInfo& jobInfo)
{
uint32_t eid = agc->expressionId();
setExpTupleInfo(agc->resultType(), eid, agc->alias(), jobInfo);

View File

@ -401,8 +401,9 @@ execplan::CalpontSystemCatalog::OID tableOid(const execplan::SimpleColumn* sc,
/** @brief Returns the unique ID to be used in tupleInfo
*
*/
uint32_t getTupleKey(const JobInfo& jobInfo,
const execplan::SimpleColumn* sc);
uint32_t getTupleKey(JobInfo& jobInfo,
const execplan::SimpleColumn* sc,
bool add = false);
uint32_t getTableKey(const JobInfo& jobInfo,
execplan::CalpontSystemCatalog::OID tableOid,
const std::string& alias,
@ -464,7 +465,7 @@ TupleInfo setExpTupleInfo(const execplan::ReturnedColumn* rc, JobInfo& jobInfo);
/** @brief add an aggregate column info
*
*/
void addAggregateColumn(execplan::AggregateColumn*, int, RetColsVector&, JobInfo&);
void addAggregateColumn(execplan::ReturnedColumn*, int, RetColsVector&, JobInfo&);
void makeJobSteps(execplan::CalpontSelectExecutionPlan* csep, JobInfo& jobInfo,
JobStepVector& querySteps, JobStepVector& projectSteps,

View File

@ -1636,6 +1636,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
string constval(cc->constval());
CalpontSystemCatalog::OID dictOid = 0;
CalpontSystemCatalog::ColType ct = sc->colType();
const PseudoColumn* pc = dynamic_cast<const PseudoColumn*>(sc);

View File

@ -274,9 +274,11 @@ const JobStepVector doProject(const RetColsVector& retCols, JobInfo& jobInfo)
if (retCols[i]->windowfunctionColumnList().size() > 0)
jobInfo.expressionVec.push_back(key);
else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) ==
jobInfo.expressionVec.end())
else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key)
== jobInfo.expressionVec.end())
{
jobInfo.returnedExpressions.push_back(sjstep);
}
//put place hold column in projection list
jobInfo.pjColList.push_back(ti);
@ -397,6 +399,7 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
void preProcessFunctionOnAggregation(const vector<SimpleColumn*>& scs,
const vector<AggregateColumn*>& aggs,
const vector<WindowFunctionColumn*>& wcs,
JobInfo& jobInfo)
{
// append the simple columns if not already projected
@ -426,10 +429,14 @@ void preProcessFunctionOnAggregation(const vector<SimpleColumn*>& scs,
}
}
// append the aggregate columns in arithmetic/function cloulmn to the projection list
// append the aggregate columns in arithmetic/function column to the projection list
for (vector<AggregateColumn*>::const_iterator i = aggs.begin(); i != aggs.end(); i++)
{
addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo);
if (wcs.size() > 0)
{
jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone()));
}
}
}
@ -481,12 +488,12 @@ void checkReturnedColumns(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
if (ac != NULL && ac->aggColumnList().size() > 0)
{
jobInfo.nonConstCols[i]->outputIndex(i);
preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), jobInfo);
preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), ac->windowfunctionColumnList(), jobInfo);
}
else if (fc != NULL && fc->aggColumnList().size() > 0)
{
jobInfo.nonConstCols[i]->outputIndex(i);
preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), jobInfo);
preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), fc->windowfunctionColumnList(), jobInfo);
}
}
}
@ -1042,16 +1049,21 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo
const FunctionColumn* fc = NULL;
const WindowFunctionColumn* wc = NULL;
bool hasAggCols = false;
bool hasWndCols = false;
if ((ac = dynamic_cast<const ArithmeticColumn*>(srcp.get())) != NULL)
{
if (ac->aggColumnList().size() > 0)
hasAggCols = true;
if (ac->windowfunctionColumnList().size() > 0)
hasWndCols = true;
}
else if ((fc = dynamic_cast<const FunctionColumn*>(srcp.get())) != NULL)
{
if (fc->aggColumnList().size() > 0)
hasAggCols = true;
if (fc->windowfunctionColumnList().size() > 0)
hasWndCols = true;
}
else if (dynamic_cast<const AggregateColumn*>(srcp.get()) != NULL)
{
@ -1076,7 +1088,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo
TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo));
tupleKey = ti.key;
if (hasAggCols)
if (hasAggCols && !hasWndCols)
jobInfo.expressionVec.push_back(tupleKey);
}
@ -1190,16 +1202,21 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo
const FunctionColumn* fc = NULL;
const WindowFunctionColumn* wc = NULL;
bool hasAggCols = false;
bool hasWndCols = false;
if ((ac = dynamic_cast<const ArithmeticColumn*>(srcp.get())) != NULL)
{
if (ac->aggColumnList().size() > 0)
hasAggCols = true;
if (ac->windowfunctionColumnList().size() > 0)
hasWndCols = true;
}
else if ((fc = dynamic_cast<const FunctionColumn*>(srcp.get())) != NULL)
{
if (fc->aggColumnList().size() > 0)
hasAggCols = true;
if (fc->windowfunctionColumnList().size() > 0)
hasWndCols = true;
}
else if (dynamic_cast<const AggregateColumn*>(srcp.get()) != NULL)
{
@ -1224,7 +1241,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo
TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo));
tupleKey = ti.key;
if (hasAggCols)
if (hasAggCols && !hasWndCols)
jobInfo.expressionVec.push_back(tupleKey);
}

View File

@ -749,7 +749,6 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min,
int64_t tMax = Max;
dataconvert::DataConvert::trimWhitespace(tMin);
dataconvert::DataConvert::trimWhitespace(tMax);
dataconvert::DataConvert::trimWhitespace(value);
scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value),
op, lcf);

View File

@ -1406,7 +1406,7 @@ void TupleAggregateStep::prep1PhaseAggregate(
typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE);
precisionAgg.push_back(-1);
widthAgg.push_back(sizeof(long double));
scaleAgg.push_back(scaleProj[colProj]);
scaleAgg.push_back(0);
}
break;
@ -1583,8 +1583,8 @@ void TupleAggregateStep::prep1PhaseAggregate(
}
functionVec[i]->fAuxColumnIndex = lastCol++;
oidsAgg.push_back(oidsAgg[j]);
keysAgg.push_back(keysAgg[j]);
oidsAgg.push_back(oidsProj[j]);
keysAgg.push_back(keysProj[j]);
scaleAgg.push_back(0);
precisionAgg.push_back(0);
precisionAgg.push_back(0);
@ -1599,8 +1599,8 @@ void TupleAggregateStep::prep1PhaseAggregate(
functionVec[i]->fAuxColumnIndex = lastCol;
// sum(x)
oidsAgg.push_back(oidsAgg[j]);
keysAgg.push_back(keysAgg[j]);
oidsAgg.push_back(oidsProj[j]);
keysAgg.push_back(keysProj[j]);
scaleAgg.push_back(0);
precisionAgg.push_back(-1);
typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE);
@ -1608,8 +1608,8 @@ void TupleAggregateStep::prep1PhaseAggregate(
++lastCol;
// sum(x**2)
oidsAgg.push_back(oidsAgg[j]);
keysAgg.push_back(keysAgg[j]);
oidsAgg.push_back(oidsProj[j]);
keysAgg.push_back(keysProj[j]);
scaleAgg.push_back(0);
precisionAgg.push_back(-1);
typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE);
@ -1935,7 +1935,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate(
typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE);
precisionAgg.push_back(-1);
widthAgg.push_back(sizeof(long double));
scaleAgg.push_back(scaleProj[colProj]);
scaleAgg.push_back(0);
colAgg++;
// has distinct step, put the count column for avg next to the sum
@ -2255,7 +2255,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate(
typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE);
precisionAggDist.push_back(-1);
widthAggDist.push_back(sizeof(long double));
scaleAggDist.push_back(scaleProj[colAgg]);
scaleAggDist.push_back(0);
}
break;
@ -2327,10 +2327,10 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate(
{
oidsAggDist.push_back(oidsAgg[colAgg]);
keysAggDist.push_back(retKey);
scaleAggDist.push_back(scaleAgg[colAgg] >> 8);
precisionAggDist.push_back(precisionAgg[colAgg]);
typeAggDist.push_back(typeAgg[colAgg]);
widthAggDist.push_back(widthAgg[colAgg]);
scaleAggDist.push_back(0);
typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE);
precisionAggDist.push_back(-1);
widthAggDist.push_back(sizeof(long double));
}
else
{
@ -3173,7 +3173,7 @@ void TupleAggregateStep::prep2PhasesAggregate(
oidsAggPm.push_back(oidsProj[colProj]);
keysAggPm.push_back(aggKey);
typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE);
scaleAggPm.push_back(scaleProj[colProj]);
scaleAggPm.push_back(0);
precisionAggPm.push_back(-1);
widthAggPm.push_back(sizeof(long double));
colAggPm++;
@ -3449,10 +3449,10 @@ void TupleAggregateStep::prep2PhasesAggregate(
{
oidsAggUm.push_back(oidsAggPm[colPm]);
keysAggUm.push_back(retKey);
scaleAggUm.push_back(scaleAggPm[colPm] >> 8);
precisionAggUm.push_back(precisionAggPm[colPm]);
typeAggUm.push_back(typeAggPm[colPm]);
widthAggUm.push_back(widthAggPm[colPm]);
scaleAggUm.push_back(0);
typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE);
precisionAggUm.push_back(-1);
widthAggUm.push_back(sizeof(long double));
}
else
{
@ -4027,7 +4027,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate(
typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE);
precisionAggPm.push_back(-1);
widthAggPm.push_back(sizeof(long double));
scaleAggPm.push_back(scaleProj[colProj]);
scaleAggPm.push_back(0);
colAggPm++;
}
@ -4394,7 +4394,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate(
typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE);
precisionAggDist.push_back(-1);
widthAggDist.push_back(sizeof(long double));
scaleAggDist.push_back(scaleAggUm[colUm]);
scaleAggDist.push_back(0);
}
// PM: put the count column for avg next to the sum
// let fall through to add a count column for average function
@ -4456,10 +4456,10 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate(
{
oidsAggDist.push_back(oidsAggUm[colUm]);
keysAggDist.push_back(retKey);
scaleAggDist.push_back(scaleAggUm[colUm] >> 8);
precisionAggDist.push_back(precisionAggUm[colUm]);
typeAggDist.push_back(typeAggUm[colUm]);
widthAggDist.push_back(widthAggUm[colUm]);
scaleAggDist.push_back(0);
typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE);
precisionAggDist.push_back(-1);
widthAggDist.push_back(sizeof(long double));
}
else
{
@ -5011,7 +5011,8 @@ void TupleAggregateStep::prepExpressionOnAggregate(SP_ROWAGG_UM_t& aggUM, JobInf
uint64_t eid = -1;
if (((ac = dynamic_cast<ArithmeticColumn*>(it->get())) != NULL) &&
(ac->aggColumnList().size() > 0))
(ac->aggColumnList().size() > 0) &&
(ac->windowfunctionColumnList().size() == 0))
{
const vector<SimpleColumn*>& scols = ac->simpleColumnList();
simpleColumns.insert(simpleColumns.end(), scols.begin(), scols.end());
@ -5020,7 +5021,8 @@ void TupleAggregateStep::prepExpressionOnAggregate(SP_ROWAGG_UM_t& aggUM, JobInf
expressionVec.push_back(*it);
}
else if (((fc = dynamic_cast<FunctionColumn*>(it->get())) != NULL) &&
(fc->aggColumnList().size() > 0))
(fc->aggColumnList().size() > 0) &&
(fc->windowfunctionColumnList().size() == 0))
{
const vector<SimpleColumn*>& sCols = fc->simpleColumnList();
simpleColumns.insert(simpleColumns.end(), sCols.begin(), sCols.end());

View File

@ -320,6 +320,41 @@ const string WindowFunctionStep::toString() const
return oss.str();
}
void WindowFunctionStep::AddSimplColumn(const vector<SimpleColumn*>& scs,
JobInfo& jobInfo)
{
// append the simple columns if not already projected
set<UniqId> scProjected;
for (RetColsVector::iterator i = jobInfo.projectionCols.begin();
i != jobInfo.projectionCols.end();
i++)
{
SimpleColumn* sc = dynamic_cast<SimpleColumn*>(i->get());
if (sc != NULL)
{
if (sc->schemaName().empty())
sc->oid(joblist::tableOid(sc, jobInfo.csc) + 1 + sc->colPosition());
scProjected.insert(UniqId(sc));
}
}
for (vector<SimpleColumn*>::const_iterator i = scs.begin(); i != scs.end(); i++)
{
if (scProjected.find(UniqId(*i)) == scProjected.end())
{
jobInfo.windowDels.push_back(SRCP((*i)->clone()));
// MCOL-3343 Enable this if we decide to allow Window Functions to run with
// aggregates with no group by. MariaDB allows this. Nobody else in the world does.
// There will be more work to get it to function if we try this.
// jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true));
scProjected.insert(UniqId(*i));
}
}
}
void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
{
// window functions in select clause, selected or in expression
@ -404,6 +439,23 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J
if (jobInfo.windowCols.empty())
return;
// Add in the non-window side of arithmetic columns and functions
for (uint64_t i = 0; i < jobInfo.windowExps.size(); i++)
{
const ArithmeticColumn* ac =
dynamic_cast<const ArithmeticColumn*>(jobInfo.windowExps[i].get());
const FunctionColumn* fc =
dynamic_cast<const FunctionColumn*>(jobInfo.windowExps[i].get());
if (ac != NULL && ac->windowfunctionColumnList().size() > 0)
{
AddSimplColumn(ac->simpleColumnList(), jobInfo);
}
else if (fc != NULL && fc->windowfunctionColumnList().size() > 0)
{
AddSimplColumn(fc->simpleColumnList(), jobInfo);
}
}
// reconstruct the delivered column list with auxiliary columns
set<uint64_t> colSet;
jobInfo.deliveredCols.resize(0);
@ -445,7 +497,13 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J
key = getTupleKey(jobInfo, *j, true);
if (colSet.find(key) == colSet.end())
{
jobInfo.deliveredCols.push_back(*j);
// MCOL-3343 Enable this if we decide to allow Window Functions to run with
// aggregates with no group by. MariaDB allows this. Nobody else in the world does.
// There will be more work to get it to function if we try this.
// jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true));
}
colSet.insert(key);
}

View File

@ -148,6 +148,8 @@ private:
void formatMiniStats();
void printCalTrace();
static void AddSimplColumn(const vector<SimpleColumn*>& scs, JobInfo& jobInfo);
class Runner
{
public:

View File

@ -30,7 +30,6 @@
#include <unistd.h>
#include <algorithm>
#include <boost/algorithm/string/trim.hpp>
#include "bpp.h"
#include "primitiveserver.h"
@ -94,7 +93,6 @@ void DictStep::createCommand(ByteStream& bs)
for (uint32_t i = 0; i < filterCount; i++)
{
bs >> strTmp;
boost::trim_right_if(strTmp, boost::is_any_of(" "));
//cout << " " << strTmp << endl;
eqFilter->insert(strTmp);
}

View File

@ -28,7 +28,6 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <stdexcept>
#include <boost/algorithm/string/trim.hpp>
//#define NDEBUG
#include <cassert>
#include <boost/thread.hpp>
@ -1778,7 +1777,6 @@ private:
for (i = 0; i < count; i++)
{
*bs >> str;
boost::trim_right_if(str, boost::is_any_of(" "));
filter->insert(str);
}

View File

@ -118,7 +118,10 @@ string Func_timediff::getStrVal(rowgroup::Row& row,
case execplan::CalpontSystemCatalog::TIME:
case execplan::CalpontSystemCatalog::DATETIME:
if (type1 != type2)
// Diff between time and datetime returns NULL in MariaDB
if ((type2 == execplan::CalpontSystemCatalog::TIME ||
type2 == execplan::CalpontSystemCatalog::DATETIME) &&
type1 != type2)
{
isNull = true;
break;