database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Fix/mcol 5787 rgdata buffer max size dev (#3325)

Browse Source 
* fix(rowgroup): RGData now uses uint64_t counter for the fixed sizes columns data buf.
	The buffer can utilize > 4GB RAM that is necessary for PM side join.
	RGData ctor uses uint32_t allocating data buffer.
 	This fact causes implicit heap overflow.

* feat(bytestream,serdes): BS buffer size type is uint64_t
	This necessary to handle 64bit RGData, that comes as
	a separate patch. The pair of patches would allow to
	have PM joins when SmallSide size > 4GB.

* feat(bytestream,serdes): Distribute BS buf size data type change to avoid implicit data type narrowing

* feat(rowgroup): this returns bits lost during cherry-pick. The bits lost caused the first RGData::serialize to crash a process
This commit is contained in:
drrtuy
2024-11-09 19:44:02 +00:00
committed by GitHub
parent 842a3c8a40
commit 8ae5a3da40
28 changed files with 1130 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
writeengine/server/we_dataloader.cpp
View File

@ -437,7 +437,7 @@ void WEDataLoader::pushData2Cpimport(ByteStream& Ibs)
{
if (Ibs.length() > 0)
{
int aLen = Ibs.length();
messageqcpp::BSSizeType aLen = Ibs.length();
char* pStart = reinterpret_cast<char*>(Ibs.buf());
char* pEnd = pStart + aLen;
char* pPtr = pStart;

49
writeengine/server/we_dataloader.h
View File

@ -30,6 +30,7 @@
#pragma once
#include "bytestream.h"
#include "rwlock_local.h"
#include "resourcemanager.h"
@ -58,32 +59,32 @@ class WEDataLoader : public Observer
public:
bool setupCpimport(); // fork the cpimport
void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267
void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue
void pushData2Cpimport(messageqcpp::ByteStream& Ibs); // push data to cpimport from the queue
void closeWritePipe();
void str2Argv(std::string CmdLine, std::vector<char*>& V);
public:
void onReceiveKeepAlive(ByteStream& Ibs);
void onReceiveData(ByteStream& Ibs);
void onReceiveEod(ByteStream& Ibs); // end of data
void onReceiveMode(ByteStream& Ibs);
void onReceiveKeepAlive(messageqcpp::ByteStream& Ibs);
void onReceiveData(messageqcpp::ByteStream& Ibs);
void onReceiveEod(messageqcpp::ByteStream& Ibs); // end of data
void onReceiveMode(messageqcpp::ByteStream& Ibs);
// void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs);
void onReceiveCmd(ByteStream& bs); // {(ByteStream& Ibs);
void onReceiveAck(ByteStream& Ibs);
void onReceiveNak(ByteStream& Ibs);
void onReceiveError(ByteStream& Ibs);
void onReceiveCmd(messageqcpp::ByteStream& bs); // {(ByteStream& Ibs);
void onReceiveAck(messageqcpp::ByteStream& Ibs);
void onReceiveNak(messageqcpp::ByteStream& Ibs);
void onReceiveError(messageqcpp::ByteStream& Ibs);
void onReceiveJobId(ByteStream& Ibs);
void onReceiveJobData(ByteStream& Ibs);
void onReceiveImportFileName(ByteStream& Ibs);
void onReceiveCmdLineArgs(ByteStream& Ibs);
void onReceiveJobId(messageqcpp::ByteStream& Ibs);
void onReceiveJobData(messageqcpp::ByteStream& Ibs);
void onReceiveImportFileName(messageqcpp::ByteStream& Ibs);
void onReceiveCmdLineArgs(messageqcpp::ByteStream& Ibs);
void onReceiveStartCpimport();
void onReceiveBrmRptFileName(ByteStream& Ibs);
void onReceiveCleanup(ByteStream& Ibs);
void onReceiveRollback(ByteStream& Ibs);
void onReceiveBrmRptFileName(messageqcpp::ByteStream& Ibs);
void onReceiveCleanup(messageqcpp::ByteStream& Ibs);
void onReceiveRollback(messageqcpp::ByteStream& Ibs);
void onReceiveErrFileRqst(ByteStream& Ibs);
void onReceiveBadFileRqst(ByteStream& Ibs);
void onReceiveErrFileRqst(messageqcpp::ByteStream& Ibs);
void onReceiveBadFileRqst(messageqcpp::ByteStream& Ibs);
void onCpimportSuccess();
void onCpimportFailure();
@ -103,11 +104,11 @@ class WEDataLoader : public Observer
{
fMode = Mode;
}
void updateTxBytes(unsigned int Tx)
void updateTxBytes(messageqcpp::BSSizeType Tx)
{
fTxBytes += Tx;
}
void updateRxBytes(unsigned int Rx)
void updateRxBytes(messageqcpp::BSSizeType Rx)
{
fRxBytes += Rx;
}
@ -132,11 +133,11 @@ class WEDataLoader : public Observer
fObjId = ObjId;
}
unsigned int getTxBytes()
messageqcpp::BSSizeType getTxBytes()
{
return fTxBytes;
}
unsigned int getRxBytes()
messageqcpp::BSSizeType getRxBytes()
{
return fRxBytes;
}
@ -172,8 +173,8 @@ class WEDataLoader : public Observer
int fMode;
std::ofstream fDataDumpFile;
std::ofstream fJobFile;
unsigned int fTxBytes;
unsigned int fRxBytes;
messageqcpp::BSSizeType fTxBytes;
messageqcpp::BSSizeType fRxBytes;
char fPmId;
int fObjId; // Object Identifier for logging

2
writeengine/splitter/we_sdhandler.cpp
View File

@ -2289,7 +2289,7 @@ int WESDHandler::getNextDbrPm2Send()
int WESDHandler::leastDataSendPm()
{
unsigned int aTx = 0;
messageqcpp::BSSizeType aTx = 0;
int aPmId = 0;
for (int aCnt = 1; aCnt <= fPmCount; aCnt++)

4
writeengine/splitter/we_splclient.cpp
View File

@ -208,7 +208,7 @@ void WESplClient::send()
messageqcpp::SBS aSbs = fSendQueue.front();
fSendQueue.pop();
aLock.unlock();
int aLen = (*aSbs).length();
messageqcpp::BSSizeType aLen = (*aSbs).length();
if (aLen > 0)
{
@ -241,7 +241,7 @@ void WESplClient::recv()
rm_ts.tv_sec = fRdSecTo; // 0 when data sending otherwise 1- second
rm_ts.tv_nsec = 20000000; // 20 milliSec
bool isTimeOut = false;
int aLen = 0;
messageqcpp::BSSizeType aLen = 0;
try
{

15
writeengine/splitter/we_splclient.h
View File

@ -29,7 +29,6 @@
#pragma once
#include "threadsafequeue.h"
#include "resourcemanager.h"
#include "we_messages.h"
@ -122,11 +121,11 @@ class WESplClient
{
return fRowTx;
}
uint32_t getBytesRcv() const
messageqcpp::BSSizeType getBytesRcv() const
{
return fBytesRcv;
}
uint32_t getBytesTx()
messageqcpp::BSSizeType getBytesTx()
{
boost::mutex::scoped_lock aLock(fTxMutex);
return fBytesTx;
@ -214,17 +213,17 @@ class WESplClient
{
return fIpAddress;
}
void setBytesRcv(uint32_t BytesRcv)
void setBytesRcv(messageqcpp::BSSizeType BytesRcv)
{
fBytesRcv = BytesRcv;
}
void setBytesTx(uint32_t BytesTx)
void setBytesTx(messageqcpp::BSSizeType BytesTx)
{
boost::mutex::scoped_lock aLock(fTxMutex);
fBytesTx = BytesTx;
aLock.unlock();
}
void updateBytesTx(uint32_t fBytes)
void updateBytesTx(messageqcpp::BSSizeType fBytes)
{
boost::mutex::scoped_lock aLock(fTxMutex);
fBytesTx += fBytes;
@ -358,8 +357,8 @@ class WESplClient
int fDataRqstCnt; // Data request count
long fRdSecTo; // read timeout sec
unsigned int fRowTx; // No. Of Rows Transmitted
uint32_t fBytesTx;
uint32_t fBytesRcv;
messageqcpp::BSSizeType fBytesTx;
messageqcpp::BSSizeType fBytesRcv;
time_t fLastInTime;
time_t fStartTime;
bool fSend;