database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

fix(aggregation, RAM): MCOL-5715 Changes the second phase aggregation. (#3171)

Browse Source 
This patch changes the second phase aggregation pipeline - takes into
account current memory consumption.

Co-authored-by: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com>
Co-authored-by: drrtuy <roman.nozdrin@mariadb.com>
This commit is contained in:
Denis Khalikov
2024-08-29 14:24:47 +03:00
committed by drrtuy
parent e0a01c6cf4
commit 928678499a
1 changed files with 131 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
dbcon/joblist/tupleaggregatestep.cpp
View File

@ -360,9 +360,17 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID)
if (threadID >= fNumOfBuckets)
return;
bool finishedSecondPhase = false;
bool diskAggAllowed = fRm->getAllowDiskAggregation();
const uint32_t maxRowsSize = rowgroup::rgCommonSize;
while (!finishedSecondPhase && !fEndOfResult)
{
scoped_array<RowBucketVec> rowBucketVecs(new RowBucketVec[fNumOfBuckets]);
scoped_array<bool> bucketDone(new bool[fNumOfBuckets]);
uint32_t hashlen = fAggregator->aggMapKeyLength();
bool outOfMemory = false;
size_t totalMemSizeConsumed = 0;
try
{
@ -411,6 +419,15 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID)
rowBucketVecs[bucketID][j].emplace_back(rowIn.getPointer(), hash);
rowIn.nextRow();
}
const auto rgSize = diskAggAllowed ? rowGroupIn->getSizeWithStrings(maxRowsSize)
: rowGroupIn->getSizeWithStrings();
totalMemSizeConsumed += rgSize;
if (!fRm->getMemory(rgSize, fSessionMemLimit, !diskAggAllowed))
{
outOfMemory = true;
break;
}
}
}
}
@ -435,11 +452,22 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID)
rowBucketVecs[bucketID][0].emplace_back(rowIn.getPointer(), hash);
rowIn.nextRow();
}
const auto rgSize =
diskAggAllowed ? rowGroupIn->getSizeWithStrings(maxRowsSize) : rowGroupIn->getSizeWithStrings();
totalMemSizeConsumed += rgSize;
if (!fRm->getMemory(rgSize, fSessionMemLimit, !diskAggAllowed))
{
outOfMemory = true;
break;
}
}
}
if (!outOfMemory)
finishedSecondPhase = true;
bool done = false;
// reset bucketDone[] to be false
// memset(bucketDone, 0, sizeof(bucketDone));
fill(&bucketDone[0], &bucketDone[fNumOfBuckets], false);
@ -478,17 +506,23 @@ void TupleAggregateStep::doThreadedSecondPhaseAggregate(uint32_t threadID)
}
}
fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit);
if (cancelled())
{
fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit);
finishedSecondPhase = true;
fEndOfResult = true;
}
} // try
catch (...)
{
fRm->returnMemory(totalMemSizeConsumed, fSessionMemLimit);
handleException(std::current_exception(), logging::tupleAggregateStepErr,
logging::ERR_AGGREGATION_TOO_BIG, "TupleAggregateStep::doThreadedSecondPhaseAggregate()");
logging::ERR_AGGREGATION_TOO_BIG,
"TupleAggregateStep::doThreadedSecondPhaseAggregate()");
fEndOfResult = true;
finishedSecondPhase = true;
}
}
fDoneAggregate = true;