database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL4841 dev port run large join without OOM

Browse Source
This commit is contained in:
David Hall
2022-02-09 17:33:55 -06:00
parent d30e140dc3
commit 27dea733c5
34 changed files with 821 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
dbcon/joblist/diskjoinstep.cpp
View File

@ -419,8 +419,9 @@ void DiskJoinStep::joinFcn()
while (largeData)
{
l_largeRG.setData(largeData.get());
thjs->joinOneRG(0, &joinResults, l_largeRG, l_outputRG, l_largeRow, l_joinFERow, l_outputRow, baseRow,
joinMatches, smallRowTemplates, &joiners, &colMappings, &fergMappings, &smallNullMem);
thjs->joinOneRG(0, joinResults, l_largeRG, l_outputRG, l_largeRow, l_joinFERow, l_outputRow, baseRow,
joinMatches, smallRowTemplates, outputDL.get(), &joiners, &colMappings, &fergMappings,
&smallNullMem);
for (j = 0; j < (int)joinResults.size(); j++)
{
@ -428,7 +429,7 @@ void DiskJoinStep::joinFcn()
// cout << "got joined output " << l_outputRG.toString() << endl;
outputDL->insert(joinResults[j]);
}
thjs->returnMemory();
joinResults.clear();
largeData = in->jp->getNextLargeRGData();
}
@ -477,6 +478,22 @@ void DiskJoinStep::joinFcn()
{
outputDL->insert(rgData);
// cout << "inserting a full RG" << endl;
if (thjs)
{
if (!thjs->getMemory(l_outputRG.getMaxDataSize()))
{
// calculate guess of size required for error message
uint64_t memReqd = (unmatched.size() * outputRG.getDataSize(1)) / 1048576;
Message::Args args;
args.add(memReqd);
args.add(thjs->resourceManager->getConfiguredUMMemLimit() / 1048576);
std::cerr << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_RESULT_TOO_BIG,
args)
<< " @" << __FILE__ << ":" << __LINE__;
throw logging::IDBExcept(logging::ERR_JOIN_RESULT_TOO_BIG, args);
}
}
rgData.reinit(l_outputRG);
l_outputRG.setData(&rgData);
l_outputRG.resetRowGroup(0);
@ -491,6 +508,10 @@ void DiskJoinStep::joinFcn()
// cout << "inserting an rg with " << l_outputRG.getRowCount() << endl;
outputDL->insert(rgData);
}
if (thjs)
{
thjs->returnMemory();
}
}
}
}