database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

feat(): replace getMaxDataSize with getMaxDataSizeWithStrings to accurately account for mem

Browse Source
This commit is contained in:
drrtuy
2025-03-04 21:10:39 +00:00
parent da28750ffc
commit be5711cf0d
3 changed files with 13 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
dbcon/joblist/diskjoinstep.cpp
View File

@ -519,16 +519,18 @@ void DiskJoinStep::joinFcn(const uint32_t threadID)
// cout << "inserting a full RG" << endl;
if (thjs)
{
// FIXME: Possible false positive. Something wrong with this calculation, just put a warning
// until fixed.
if (!thjs->getMemory(l_outputRG.getMaxDataSize()))
if (!thjs->getMemory(l_outputRG.getMaxDataSizeWithStrings()))
{
// FIXME: This is also looks wrong.
// calculate guess of size required for error message
uint64_t memReqd = (unmatched.size() * outputRG.getDataSize(1)) / 1048576;
uint64_t memLimit = thjs->resourceManager->getConfiguredUMMemLimit() / 1048576;
std::cerr << "DiskJoin::joinFcn() possible OOM for the join result, mem required: "
<< memReqd << " mem limit: " << memLimit << std::endl;
uint64_t memReqd = (l_outputRG.getMaxDataSizeWithStrings()) / 1048576;
Message::Args args;
args.add(memReqd);
args.add(thjs->resourceManager->getConfiguredUMMemLimit() / 1048576);
std::cerr << logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_RESULT_TOO_BIG,
args)
<< " @" << __FILE__ << ":" << __LINE__;
throw logging::IDBExcept(logging::ERR_JOIN_RESULT_TOO_BIG, args);
}
}

5
dbcon/joblist/tuple-bps.cpp
View File

@ -275,9 +275,10 @@ uint64_t TupleBPS::JoinLocalData::generateJoinResultSet(const uint32_t depth,
uint64_t baseRid = local_outputRG.getBaseRid();
outputData.push_back(joinedData);
// Don't let the join results buffer get out of control.
if (tbps->resourceManager()->getMemory(local_outputRG.getMaxDataSize(), false))
auto outputDataSize = local_outputRG.getMaxDataSizeWithStrings();
if (tbps->resourceManager()->getMemory(outputDataSize, false))
{
memSizeForOutputRG += local_outputRG.getMaxDataSize();
memSizeForOutputRG += outputDataSize;
}
else
{

2
utils/rowgroup/rowstorage.cpp
View File

@ -1361,7 +1361,7 @@ class RowGroupStorage
{
messageqcpp::ByteStream bs;
fRowGroupOut->setData(rgdata);
rgdata->serialize(bs, fRowGroupOut->getDataSize());
rgdata->serialize(bs, fRowGroupOut->getSizeWithStrings());
int errNo;
if ((errNo = fDumper->write(makeRGFilename(rgid), (char*)bs.buf(), bs.length())) != 0)