database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-27 21:01:50 +03:00
Code Activity

MCOL-894 Upmerged post review changes.

Browse Source 
Raised the default for orderby threads from 4 to 16.
    Removed if LIMIT check block in makeVtableModeSteps().
    Removed a duplicate of TimeCompare class and methods.

MCOL-3536 Upmerged the change.

MCOL-894 Post review changes.
    Uncomment if LIMIT check block in makeVtableModeSteps().

    TupleAnnexStep dtor now uses vector::size() as a boundary.

    Removed useless try-catch blocks.

    executeParallelOrderBy() now calculates rowSize only once.

    Removed forward declaration in the unexisting namespace.

    Trim UTs a bit.
This commit is contained in:
Roman Nozdrin
2019-10-30 17:00:51 +03:00
parent 7b5e5f0eb6
commit 0696696cf6
6 changed files with 212 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
dbcon/joblist/orderby-tests.cpp
View File

@ -315,7 +315,7 @@ private:
void ORDERBY_TIME_TEST() void ORDERBY_TIME_TEST()
{ {
uint64_t numRows = 8192; uint64_t numRows = 8192;
uint64_t maxThreads = 16; uint64_t maxThreads = 8;
// limit == 100000 is still twice as good to sort in parallel // limit == 100000 is still twice as good to sort in parallel
// limit == 1000000 however is better to sort using single threaded sorting // limit == 1000000 however is better to sort using single threaded sorting
uint64_t limit = 100000; uint64_t limit = 100000;
@ -324,9 +324,10 @@ private:
bool generateRandValues = true; bool generateRandValues = true;
bool hasDistinct = true; bool hasDistinct = true;
bool noDistinct = false; bool noDistinct = false;
//orderByTest_nRGs(numRows * 14400, limit, maxThreads, woParallel, generateRandValues, noDistinct); orderByTest_nRGs(numRows * 14400, limit, maxThreads, woParallel, generateRandValues, noDistinct);
//orderByTest_nRGs(numRows * 14400, limit, maxThreads, woParallel, generateRandValues, hasDistinct);
orderByTest_nRGs(numRows * 14400, limit, maxThreads, parallel, generateRandValues, noDistinct); orderByTest_nRGs(numRows * 14400, limit, maxThreads, parallel, generateRandValues, noDistinct);
orderByTest_nRGs(numRows * 14400, limit, maxThreads, woParallel, generateRandValues, hasDistinct);
orderByTest_nRGs(numRows * 14400, limit, maxThreads, parallel, generateRandValues, hasDistinct);
} }
void QUICK_TEST() void QUICK_TEST()
{ {

476
dbcon/joblist/tupleannexstep.cpp
View File

@ -138,7 +138,7 @@ TupleAnnexStep::~TupleAnnexStep()
{ {
if(fOrderByList.size() > 0) if(fOrderByList.size() > 0)
{ {
for(uint64_t id = 0; id <= fMaxThreads; id++) for(uint64_t id = 0; id < fOrderByList.size(); id++)
{ {
delete fOrderByList[id]; delete fOrderByList[id];
} }
@ -704,7 +704,8 @@ void TupleAnnexStep::finalizeParallelOrderByDistinct()
fRowGroupOut.getRow(0, &fRowOut); fRowGroupOut.getRow(0, &fRowOut);
ordering::SortingPQ finalPQ; ordering::SortingPQ finalPQ;
scoped_ptr<DistinctMap_t> distinctMap(new DistinctMap_t(10, TAHasher(this), TAEq(this))); scoped_ptr<DistinctMap_t> distinctMap(new DistinctMap_t(10, TAHasher(this), TAEq(this)));
fRowGroupIn.initRow(&row1);
fRowGroupIn.initRow(&row2);
try try
{ {
@ -744,156 +745,120 @@ void TupleAnnexStep::finalizeParallelOrderByDistinct()
ERR_IN_PROCESS, fErrorInfo, fSessionId); ERR_IN_PROCESS, fErrorInfo, fSessionId);
} }
// OFFSET processing
while (finalPQ.size() && offset < fLimitStart)
{
offset++;
finalPQ.pop();
}
try // Calculate rowSize only once
if (finalPQ.size())
{ {
// OFFSET processing ordering::OrderByRow& topOBRow =
while (finalPQ.size() && offset < fLimitStart) const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
if (!fConstant)
{ {
offset++; copyRow(fRowIn, &fRowOut);
finalPQ.pop(); }
} else
} {
catch (const std::exception& ex) fConstant->fillInConstants(fRowIn, fRowOut);
{ }
catchHandler(ex.what(),ERR_IN_PROCESS, fErrorInfo, fSessionId); rowSize = fRowOut.getSize();
} fRowGroupOut.incRowCount();
catch (...) fRowOut.nextRow(rowSize);
{ finalPQ.pop();
catchHandler("TupleAnnexStep::finalizeParallelOrderByDistinct execute\ count++;
caught an unknown exception 2",
ERR_IN_PROCESS, fErrorInfo, fSessionId);
} }
try if (!fConstant)
{ {
// Calculate rowSize only once while(finalPQ.size())
if (finalPQ.size())
{ {
ordering::OrderByRow& topOBRow = if (cancelled())
const_cast<ordering::OrderByRow&>(finalPQ.top()); {
fRowIn.setData(topOBRow.fData); break;
if (!fConstant) }
while (count < fLimitCount && finalPQ.size()
&& fRowGroupOut.getRowCount() < rowgroup::rgCommonSize)
{
ordering::OrderByRow &topOBRow =
const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
copyRow(fRowIn, &fRowOut); copyRow(fRowIn, &fRowOut);
fRowGroupOut.incRowCount();
fRowOut.nextRow(rowSize);
finalPQ.pop();
count++;
if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize)
{
break;
}
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
rgDataOut.reinit(fRowGroupIn, rowgroup::rgCommonSize);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.getRow(0, &fRowOut);
}
else else
{
break;
}
} // end of limit bound while loop
}
else // Add ConstantColumns striped earlier
{
while(finalPQ.size())
{
if (cancelled())
{
break;
}
while (count < fLimitCount && finalPQ.size()
&& fRowGroupOut.getRowCount() < rowgroup::rgCommonSize)
{
ordering::OrderByRow &topOBRow =
const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
fConstant->fillInConstants(fRowIn, fRowOut); fConstant->fillInConstants(fRowIn, fRowOut);
rowSize = fRowOut.getSize(); fRowGroupOut.incRowCount();
fRowGroupOut.incRowCount(); fRowOut.nextRow(rowSize);
fRowOut.nextRow(rowSize);
finalPQ.pop();
count++;
}
}
catch (const std::exception& ex)
{
catchHandler(ex.what(),ERR_IN_PROCESS, fErrorInfo, fSessionId);
}
catch (...)
{
catchHandler("TupleAnnexStep::finalizeParallelOrderByDistinct execute\
caught an unknown exception 3",
ERR_IN_PROCESS, fErrorInfo, fSessionId);
}
try finalPQ.pop();
{ count++;
if (!fConstant) if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize)
{ {
while(finalPQ.size()) break;
}
}
if (fRowGroupOut.getRowCount() > 0)
{ {
if (cancelled()) fRowsReturned += fRowGroupOut.getRowCount();
{ fOutputDL->insert(rgDataOut);
break; rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize);
} fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
while (count < fLimitCount && finalPQ.size() fRowGroupOut.getRow(0, &fRowOut);
&& fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) }
{ else
ordering::OrderByRow &topOBRow =
const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
copyRow(fRowIn, &fRowOut);
fRowGroupOut.incRowCount();
fRowOut.nextRow(rowSize);
finalPQ.pop();
count++;
if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize)
{
break;
}
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
rgDataOut.reinit(fRowGroupIn, rowgroup::rgCommonSize);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.getRow(0, &fRowOut);
}
else
{
break;
}
} // end of limit bound while loop
}
else // Add ConstantColumns striped earlier
{
while(finalPQ.size())
{ {
if (cancelled())
{
break; break;
} }
} // end of limit bound while loop
while (count < fLimitCount && finalPQ.size() } // end of if-else
&& fRowGroupOut.getRowCount() < rowgroup::rgCommonSize)
{
ordering::OrderByRow &topOBRow =
const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
fConstant->fillInConstants(fRowIn, fRowOut);
fRowGroupOut.incRowCount();
fRowOut.nextRow(rowSize);
finalPQ.pop();
count++;
if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize)
{
break;
}
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.getRow(0, &fRowOut);
}
else
{
break;
}
} // end of limit bound while loop
} // end of if-else
}
catch (const std::exception& ex)
{
catchHandler(ex.what(),ERR_IN_PROCESS, fErrorInfo, fSessionId);
}
catch (...)
{
catchHandler("TupleAnnexStep::finalizeParallelOrderByDistinct execute\
caught an unknown exception 4",
ERR_IN_PROCESS, fErrorInfo, fSessionId);
}
if (fRowGroupOut.getRowCount() > 0) if (fRowGroupOut.getRowCount() > 0)
{ {
@ -982,152 +947,116 @@ void TupleAnnexStep::finalizeParallelOrderBy()
ERR_IN_PROCESS, fErrorInfo, fSessionId); ERR_IN_PROCESS, fErrorInfo, fSessionId);
} }
// OFFSET processing
while (finalPQ.size() && offset < fLimitStart)
{
offset++;
finalPQ.pop();
}
try // Calculate rowSize only once
if (finalPQ.size())
{ {
// OFFSET processing ordering::OrderByRow& topOBRow =
while (finalPQ.size() && offset < fLimitStart) const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
if (!fConstant)
{ {
offset++; copyRow(fRowIn, &fRowOut);
finalPQ.pop(); }
} else
} {
catch (const std::exception& ex) fConstant->fillInConstants(fRowIn, fRowOut);
{ }
catchHandler(ex.what(),ERR_IN_PROCESS, fErrorInfo, fSessionId); rowSize = fRowOut.getSize();
} fRowGroupOut.incRowCount();
catch (...) fRowOut.nextRow(rowSize);
{ finalPQ.pop();
catchHandler("TupleAnnexStep::finalizeParallelOrderBy execute\ count++;
caught an unknown exception 2",
ERR_IN_PROCESS, fErrorInfo, fSessionId);
} }
try if (!fConstant)
{ {
// Calculate rowSize only once while(finalPQ.size())
if (finalPQ.size())
{ {
ordering::OrderByRow& topOBRow = if (cancelled())
const_cast<ordering::OrderByRow&>(finalPQ.top()); {
fRowIn.setData(topOBRow.fData); break;
if (!fConstant) }
while (count < fLimitCount && finalPQ.size()
&& fRowGroupOut.getRowCount() < rowgroup::rgCommonSize)
{
ordering::OrderByRow &topOBRow =
const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
copyRow(fRowIn, &fRowOut); copyRow(fRowIn, &fRowOut);
fRowGroupOut.incRowCount();
fRowOut.nextRow(rowSize);
finalPQ.pop();
count++;
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
rgDataOut.reinit(fRowGroupIn, rowgroup::rgCommonSize);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.getRow(0, &fRowOut);
}
else else
{
break;
}
} // end of limit bound while loop
}
else // Add ConstantColumns striped earlier
{
while(finalPQ.size())
{
if (cancelled())
{
break;
}
while (count < fLimitCount && finalPQ.size()
&& fRowGroupOut.getRowCount() < rowgroup::rgCommonSize)
{
ordering::OrderByRow &topOBRow =
const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
fConstant->fillInConstants(fRowIn, fRowOut); fConstant->fillInConstants(fRowIn, fRowOut);
rowSize = fRowOut.getSize(); fRowGroupOut.incRowCount();
fRowGroupOut.incRowCount(); fRowOut.nextRow(rowSize);
fRowOut.nextRow(rowSize);
finalPQ.pop();
count++;
}
}
catch (const std::exception& ex)
{
catchHandler(ex.what(),ERR_IN_PROCESS, fErrorInfo, fSessionId);
}
catch (...)
{
catchHandler("TupleAnnexStep::finalizeParallelOrderBy execute\
caught an unknown exception 3",
ERR_IN_PROCESS, fErrorInfo, fSessionId);
}
try finalPQ.pop();
{ count++;
if (!fConstant) if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize)
{ {
while(finalPQ.size()) break;
}
}
if (fRowGroupOut.getRowCount() > 0)
{ {
if (cancelled()) fRowsReturned += fRowGroupOut.getRowCount();
{ fOutputDL->insert(rgDataOut);
break; rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize);
} fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
while (count < fLimitCount && finalPQ.size() fRowGroupOut.getRow(0, &fRowOut);
&& fRowGroupOut.getRowCount() < rowgroup::rgCommonSize) }
{ else
ordering::OrderByRow &topOBRow =
const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
copyRow(fRowIn, &fRowOut);
fRowGroupOut.incRowCount();
fRowOut.nextRow(rowSize);
finalPQ.pop();
count++;
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
rgDataOut.reinit(fRowGroupIn, rowgroup::rgCommonSize);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.getRow(0, &fRowOut);
}
else
{
break;
}
} // end of limit bound while loop
}
else // Add ConstantColumns striped earlier
{
while(finalPQ.size())
{ {
if (cancelled()) break;
{ }
break; } // end of limit bound while loop
} } // end of if-else
while (count < fLimitCount && finalPQ.size()
&& fRowGroupOut.getRowCount() < rowgroup::rgCommonSize)
{
ordering::OrderByRow &topOBRow =
const_cast<ordering::OrderByRow&>(finalPQ.top());
fRowIn.setData(topOBRow.fData);
fConstant->fillInConstants(fRowIn, fRowOut);
fRowGroupOut.incRowCount();
fRowOut.nextRow(rowSize);
finalPQ.pop();
count++;
if (fRowGroupOut.getRowCount() == rowgroup::rgCommonSize)
{
break;
}
}
if (fRowGroupOut.getRowCount() > 0)
{
fRowsReturned += fRowGroupOut.getRowCount();
fOutputDL->insert(rgDataOut);
rgDataOut.reinit(fRowGroupOut, rowgroup::rgCommonSize);
fRowGroupOut.setData(&rgDataOut);
fRowGroupOut.resetRowGroup(0);
fRowGroupOut.getRow(0, &fRowOut);
}
else
{
break;
}
} // end of limit bound while loop
} // end of if-else
}
catch (const std::exception& ex)
{
catchHandler(ex.what(),ERR_IN_PROCESS, fErrorInfo, fSessionId);
}
catch (...)
{
catchHandler("TupleAnnexStep::finalizeParallelOrderBy execute\
caught an unknown exception 4",
ERR_IN_PROCESS, fErrorInfo, fSessionId);
}
if (fRowGroupOut.getRowCount() > 0) if (fRowGroupOut.getRowCount() > 0)
{ {
@ -1200,7 +1129,10 @@ void TupleAnnexStep::executeParallelOrderBy(uint64_t id)
rg.setData(&rgDataIn); rg.setData(&rgDataIn);
rg.getRow(0, &r); rg.getRow(0, &r);
rowSize = r.getSize(); if (!rowSize)
{
rowSize = r.getSize();
}
rowCount = rg.getRowCount(); rowCount = rg.getRowCount();
for (uint64_t i = 0; i < rowCount; ++i) for (uint64_t i = 0; i < rowCount; ++i)

7
dbcon/joblist/tupleannexstep.h
View File

@ -28,13 +28,6 @@
#include "jobstep.h" #include "jobstep.h"
#include "limitedorderby.h" #include "limitedorderby.h"
// forward reference
namespace fucexp
{
class FuncExp;
}
namespace joblist namespace joblist
{ {
class TupleConstantStep; class TupleConstantStep;

3
dbcon/joblist/tupleconstantstep.cpp
View File

@ -472,7 +472,7 @@ void TupleConstantStep::execute()
fOutputDL->endOfInput(); fOutputDL->endOfInput();
} }
// *DRRTUY Copy row at once not one field at a time
void TupleConstantStep::fillInConstants() void TupleConstantStep::fillInConstants()
{ {
fRowGroupIn.getRow(0, &fRowIn); fRowGroupIn.getRow(0, &fRowIn);
@ -495,7 +495,6 @@ void TupleConstantStep::fillInConstants()
} }
else // only first column is constant else // only first column is constant
{ {
//size_t n = fRowOut.getOffset(fRowOut.getColumnCount()) - fRowOut.getOffset(1);
for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i) for (uint64_t i = 0; i < fRowGroupIn.getRowCount(); ++i)
{ {
fRowOut.setRid(fRowIn.getRelRid()); fRowOut.setRid(fRowIn.getRelRid());

38
utils/windowfunction/idborderby.cpp
View File

@ -468,37 +468,6 @@ int TimeCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
} }
else else
{ {
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int TimeCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
bool b1 = l->row1().isNullValue(fSpec.fIndex);
bool b2 = l->row2().isNullValue(fSpec.fIndex);
int ret = 0;
if (b1 == true || b2 == true)
{
if (b1 == false && b2 == true)
ret = fSpec.fNf;
else if (b1 == true && b2 == false)
ret = -fSpec.fNf;
}
else
{
int64_t v1 = l->row1().getIntField(fSpec.fIndex);
int64_t v2 = l->row2().getIntField(fSpec.fIndex);
// ((int64_t) -00:00:26) > ((int64_t) -00:00:25) // ((int64_t) -00:00:26) > ((int64_t) -00:00:25)
// i.e. For 2 negative TIME values, we invert the order of // i.e. For 2 negative TIME values, we invert the order of
// comparison operations to force "-00:00:26" to appear before // comparison operations to force "-00:00:26" to appear before
@ -684,13 +653,6 @@ void CompareRule::compileRules(const std::vector<IdbSortSpec>& spec, const rowgr
break; break;
} }
case CalpontSystemCatalog::TIME:
{
Compare* c = new TimeCompare(*i);
fCompares.push_back(c);
break;
}
default: default:
{ {
break; break;

14
utils/windowfunction/idborderby.h
View File

@ -51,8 +51,8 @@ class ResourceManager;
namespace ordering namespace ordering
{ {
template<typename _Tp, typename _Sequence = vector<_Tp>, template<typename _Tp, typename _Sequence = std::vector<_Tp>,
typename _Compare = less<typename _Sequence::value_type> > typename _Compare = std::less<typename _Sequence::value_type> >
class reservablePQ: private std::priority_queue<_Tp, _Sequence, _Compare> class reservablePQ: private std::priority_queue<_Tp, _Sequence, _Compare>
{ {
public: public:
@ -132,6 +132,7 @@ public:
void revertSortSpec() void revertSortSpec()
{ {
fSpec.fAsc = -fSpec.fAsc; fSpec.fAsc = -fSpec.fAsc;
fSpec.fNf = -fSpec.fNf;
} }
protected: protected:
@ -289,15 +290,6 @@ public:
// End of comparators for variable sized types // End of comparators for variable sized types
class TimeCompare : public Compare
{
public:
TimeCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
class CompareRule class CompareRule
{ {
public: public: