database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Fix/mcol 5787 rgdata buffer max size dev (#3325)

Browse Source 
* fix(rowgroup): RGData now uses uint64_t counter for the fixed sizes columns data buf.
	The buffer can utilize > 4GB RAM that is necessary for PM side join.
	RGData ctor uses uint32_t allocating data buffer.
 	This fact causes implicit heap overflow.

* feat(bytestream,serdes): BS buffer size type is uint64_t
	This necessary to handle 64bit RGData, that comes as
	a separate patch. The pair of patches would allow to
	have PM joins when SmallSide size > 4GB.

* feat(bytestream,serdes): Distribute BS buf size data type change to avoid implicit data type narrowing

* feat(rowgroup): this returns bits lost during cherry-pick. The bits lost caused the first RGData::serialize to crash a process
This commit is contained in:
drrtuy
2024-11-09 19:44:02 +00:00
committed by GitHub
parent 842a3c8a40
commit 8ae5a3da40
28 changed files with 1130 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
dbcon/joblist/batchprimitiveprocessor-jl.cpp
View File

@ -72,7 +72,6 @@ BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm)
, LBIDTrace(false)
, tupleLength(0)
, status(0)
, sendRowGroups(false)
, valueColumn(0)
, sendTupleJoinRowGroupData(false)
, bop(BOP_AND)
@ -147,7 +146,7 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pDictionaryStep& step)
tableOID = step.tableOid();
if (filterCount == 0 && !sendRowGroups)
if (filterCount == 0)
{
sendAbsRids = true;
sendValues = true;
@ -244,7 +243,7 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& step)
if (utils::isWide(cc->getWidth()))
wideColumnsWidths |= cc->getWidth();
if (filterCount == 0 && !sendRowGroups)
if (filterCount == 0)
sendValues = true;
idbassert(sessionID == step.sessionId());
@ -283,7 +282,7 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& p, const pDic
projectCount++;
needStrValues = true;
if (filterCount == 0 && !sendRowGroups)
if (filterCount == 0)
{
sendValues = true;
sendAbsRids = true;
@ -1054,9 +1053,6 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const
if (tJoiners.size() > 0)
flags |= HAS_JOINER;
if (sendRowGroups)
flags |= HAS_ROWGROUP;
if (sendTupleJoinRowGroupData)
flags |= JOIN_ROWGROUP_DATA;
@ -1071,12 +1067,6 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const
bs << bop;
bs << (uint8_t)(forHJ ? 1 : 0);
if (sendRowGroups)
{
bs << valueColumn;
bs << inputRG;
}
if (ot == ROW_GROUP)
{
bs << projectionRG;
@ -1248,6 +1238,7 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const
* (projection count)x run msgs for projection Commands
*/
// The deser counterpart function is BPP::resetBPP
void BatchPrimitiveProcessorJL::runBPP(ByteStream& bs, uint32_t pmNum, bool isExeMgrDEC)
{
ISMPacketHeader ism;
@ -1289,35 +1280,28 @@ void BatchPrimitiveProcessorJL::runBPP(ByteStream& bs, uint32_t pmNum, bool isEx
bs << sentByEM;
if (_hasScan)
{
idbassert(ridCount == 0);
else if (!sendRowGroups)
}
else
{
idbassert(ridCount > 0 && (ridMap != 0 || sendAbsRids));
else
idbassert(inputRG.getRowCount() > 0);
if (sendRowGroups)
{
uint32_t rgSize = inputRG.getDataSize();
bs << rgSize;
bs.append(inputRG.getData(), rgSize);
}
bs << ridCount;
if (sendAbsRids)
bs.append((uint8_t*)absRids.get(), ridCount << 3);
else
{
bs << ridCount;
if (sendAbsRids)
bs.append((uint8_t*)absRids.get(), ridCount << 3);
else
{
bs << ridMap;
bs << baseRid;
bs.append((uint8_t*)relRids, ridCount << 1);
}
if (sendValues)
bs.append((uint8_t*)values, ridCount << 3);
bs << ridMap;
bs << baseRid;
bs.append((uint8_t*)relRids, ridCount << 1);
}
if (sendValues)
bs.append((uint8_t*)values, ridCount << 3);
for (i = 0; i < filterCount; i++)
filterSteps[i]->runCommand(bs);
@ -1667,7 +1651,6 @@ void BatchPrimitiveProcessorJL::setJoinedRowGroup(const rowgroup::RowGroup& rg)
void BatchPrimitiveProcessorJL::setInputRowGroup(const rowgroup::RowGroup& rg)
{
sendRowGroups = true;
sendAbsRids = false;
sendValues = false;
inputRG = rg;