database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-05 16:15:50 +03:00
Code Activity

fix(aggregation, RAM): MCOL-5715 Changes the second phase aggregation. (#3171)

Browse Source 
This patch changes the second phase aggregation pipeline - takes into
account current memory consumption.

Co-authored-by: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com>
Co-authored-by: drrtuy <roman.nozdrin@mariadb.com>
This commit is contained in:
Denis Khalikov
2024-08-29 14:24:47 +03:00
committed by drrtuy
parent e0a01c6cf4
commit 928678499a
1 changed files with 131 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
dbcon/joblist/tupleaggregatestep.cpp
View File

@@ -360,9 +360,17 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID)
if (threadID >= fNumOfBuckets) if (threadID >= fNumOfBuckets)
return; return;
bool finishedSecondPhase = false;
bool diskAggAllowed = fRm->getAllowDiskAggregation();
const uint32_t maxRowsSize = rowgroup::rgCommonSize;
while (!finishedSecondPhase && !fEndOfResult)
{
scoped_array<RowBucketVec> rowBucketVecs(new RowBucketVec[fNumOfBuckets]); scoped_array<RowBucketVec> rowBucketVecs(new RowBucketVec[fNumOfBuckets]);
scoped_array<bool> bucketDone(new bool[fNumOfBuckets]); scoped_array<bool> bucketDone(new bool[fNumOfBuckets]);
uint32_t hashlen = fAggregator->aggMapKeyLength(); uint32_t hashlen = fAggregator->aggMapKeyLength();
bool outOfMemory = false;
size_t totalMemSizeConsumed = 0;
try try
{ {
@@ -411,6 +419,15 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID)
rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash); rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash);
rowIn.nextRow(); rowIn.nextRow();
} }
const auto rgSize = diskAggAllowed ? rowGroupIn->getSizeWithStrings(maxRowsSize)
: rowGroupIn->getSizeWithStrings();
totalMemSizeConsumed += rgSize;
if (!fRm->getMemory(rgSize, fSessionMemLimit, !diskAggAllowed))
{
outOfMemory = true;
break;
}
} }
} }
} }
@@ -435,11 +452,22 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID)
rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash); rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash);
rowIn.nextRow(); rowIn.nextRow();
} }
const auto rgSize =
diskAggAllowed ? rowGroupIn->getSizeWithStrings(maxRowsSize) : rowGroupIn->getSizeWithStrings();
totalMemSizeConsumed += rgSize;
if (!fRm->getMemory(rgSize, fSessionMemLimit, !diskAggAllowed))
{
outOfMemory = true;
break;
} }
} }
}
if (!outOfMemory)
finishedSecondPhase = true;
bool done = false; bool done = false;
// reset bucketDone[] to be false // reset bucketDone[] to be false
// memset(bucketDone, 0, sizeof(bucketDone)); // memset(bucketDone, 0, sizeof(bucketDone));
fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false); fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false);
@@ -478,17 +506,23 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID)
} }
} }
fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit);
if (cancelled()) if (cancelled())
{ {
fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit);
finishedSecondPhase = true;
fEndOfResult = true; fEndOfResult = true;
} }
} // try } // try
catch (...) catch (...)
{ {
fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit);
handleException(std::current_exception(), logging::tupleAggregateStepErr, handleException(std::current_exception(), logging::tupleAggregateStepErr,
logging::ERR_AGGREGATION_TOO_BIG, "TupleAggregateStep::doThreadedSecondPhaseAggregate()"); logging::ERR_AGGREGATION_TOO_BIG,
"TupleAggregateStep::doThreadedSecondPhaseAggregate()");
fEndOfResult = true; fEndOfResult = true;
finishedSecondPhase = true;
}
} }
fDoneAggregate = true; fDoneAggregate = true;