database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Merge pull request #1842 from denis0x0D/MCOL-987_LZ

Browse Source 
MCOL-987 LZ4 compression support.
This commit is contained in:
Roman Nozdrin
2021-07-07 13:13:18 +03:00
committed by GitHub
parent 7b4f759592 cc1c3629c5
commit 866dc25729
45 changed files with 1311 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
writeengine/wrapper/we_colopcompress.cpp
View File

@ -121,9 +121,9 @@ int ColumnOpCompress0::saveBlock(IDBDataFile* pFile, const unsigned char* writeB
* Constructor
*/
ColumnOpCompress1::ColumnOpCompress1(Log* logger)
ColumnOpCompress1::ColumnOpCompress1(uint32_t compressionType, Log* logger)
{
m_compressionType = 1;
m_compressionType = compressionType;
m_chunkManager = new ChunkManager();
if (logger)
@ -164,11 +164,7 @@ bool ColumnOpCompress1::abbreviatedExtent(IDBDataFile* pFile, int colWidth) cons
int ColumnOpCompress1::blocksInFile(IDBDataFile* pFile) const
{
CompFileHeader compFileHeader;
readHeaders(pFile, compFileHeader.fControlData, compFileHeader.fPtrSection);
compress::IDBCompressInterface compressor;
return compressor.getBlockCount(compFileHeader.fControlData);
return m_chunkManager->getBlockCount(pFile);
}

2
writeengine/wrapper/we_colopcompress.h
View File

@ -97,7 +97,7 @@ public:
/**
* @brief Constructor
*/
EXPORT ColumnOpCompress1(Log* logger = 0);
EXPORT ColumnOpCompress1(uint32_t compressionType, Log* logger = 0);
/**
* @brief Default Destructor

4
writeengine/wrapper/we_dctnrycompress.cpp
View File

@ -67,9 +67,9 @@ DctnryCompress0::~DctnryCompress0()
/**
* Constructor
*/
DctnryCompress1::DctnryCompress1(Log* logger)
DctnryCompress1::DctnryCompress1(uint32_t compressionType, Log* logger)
{
m_compressionType = 1;
m_compressionType = compressionType;
m_chunkManager = new ChunkManager();
if (logger)

2
writeengine/wrapper/we_dctnrycompress.h
View File

@ -62,7 +62,7 @@ public:
/**
* @brief Constructor
*/
EXPORT DctnryCompress1(Log* logger = 0);
EXPORT DctnryCompress1(uint32_t compressionType, Log* logger = 0);
/**
* @brief Default Destructor

26
writeengine/wrapper/writeengine.cpp
View File

@ -76,19 +76,25 @@ StopWatch timer;
WriteEngineWrapper::WriteEngineWrapper() : m_opType(NOOP)
{
m_colOp[UN_COMPRESSED_OP] = new ColumnOpCompress0;
m_colOp[COMPRESSED_OP] = new ColumnOpCompress1;
m_dctnry[UN_COMPRESSED_OP] = new DctnryCompress0;
m_dctnry[COMPRESSED_OP] = new DctnryCompress1;
m_colOp[COMPRESSED_OP_1] = new ColumnOpCompress1(/*comressionType=*/1);
m_dctnry[COMPRESSED_OP_1] = new DctnryCompress1(/*compressionType=*/1);
m_colOp[COMPRESSED_OP_2] = new ColumnOpCompress1(/*comressionType=*/3);
m_dctnry[COMPRESSED_OP_2] = new DctnryCompress1(/*compressionType=*/3);
}
WriteEngineWrapper::WriteEngineWrapper(const WriteEngineWrapper& rhs) : m_opType(rhs.m_opType)
{
m_colOp[UN_COMPRESSED_OP] = new ColumnOpCompress0;
m_colOp[COMPRESSED_OP] = new ColumnOpCompress1;
m_dctnry[UN_COMPRESSED_OP] = new DctnryCompress0;
m_dctnry[COMPRESSED_OP] = new DctnryCompress1;
m_colOp[COMPRESSED_OP_1] = new ColumnOpCompress1(/*compressionType=*/1);
m_dctnry[COMPRESSED_OP_1] = new DctnryCompress1(/*compressionType=*/1);
m_colOp[COMPRESSED_OP_2] = new ColumnOpCompress1(/*compressionType=*/3);
m_dctnry[COMPRESSED_OP_2] = new DctnryCompress1(/*compressionType=*/3);
}
/**@brief WriteEngineWrapper Constructor
@ -96,9 +102,13 @@ WriteEngineWrapper::WriteEngineWrapper(const WriteEngineWrapper& rhs) : m_opTyp
WriteEngineWrapper::~WriteEngineWrapper()
{
delete m_colOp[UN_COMPRESSED_OP];
delete m_colOp[COMPRESSED_OP];
delete m_dctnry[UN_COMPRESSED_OP];
delete m_dctnry[COMPRESSED_OP];
delete m_colOp[COMPRESSED_OP_1];
delete m_dctnry[COMPRESSED_OP_1];
delete m_colOp[COMPRESSED_OP_2];
delete m_dctnry[COMPRESSED_OP_2];
}
/**@brief Perform upfront initialization

53
writeengine/wrapper/writeengine.h
View File

@ -58,9 +58,10 @@ namespace WriteEngine
{
//... Total compression operation: un_compresssed, compressed
const int UN_COMPRESSED_OP = 0;
const int COMPRESSED_OP = 1;
const int TOTAL_COMPRESS_OP = 2;
const int UN_COMPRESSED_OP = 0;
const int COMPRESSED_OP_1 = 1;
const int COMPRESSED_OP_2 = 2;
const int TOTAL_COMPRESS_OP = 3;
//...Forward class declarations
class Log;
@ -446,8 +447,10 @@ public:
*/
void setIsInsert(bool bIsInsert)
{
m_colOp[COMPRESSED_OP]->chunkManager()->setIsInsert(bIsInsert);
m_dctnry[COMPRESSED_OP]->chunkManager()->setIsInsert(true);
m_colOp[COMPRESSED_OP_1]->chunkManager()->setIsInsert(bIsInsert);
m_dctnry[COMPRESSED_OP_1]->chunkManager()->setIsInsert(true);
m_colOp[COMPRESSED_OP_2]->chunkManager()->setIsInsert(bIsInsert);
m_dctnry[COMPRESSED_OP_2]->chunkManager()->setIsInsert(true);
}
/**
@ -458,7 +461,7 @@ public:
*/
bool getIsInsert()
{
return m_colOp[COMPRESSED_OP]->chunkManager()->getIsInsert();
return m_colOp[COMPRESSED_OP_1]->chunkManager()->getIsInsert();
}
std::tr1::unordered_map<TxnID, SP_TxnLBIDRec_t>& getTxnMap()
@ -475,10 +478,23 @@ public:
*/
int flushChunks(int rc, const std::map<FID, FID>& columOids)
{
int rtn1 = m_colOp[COMPRESSED_OP]->chunkManager()->flushChunks(rc, columOids);
int rtn2 = m_dctnry[COMPRESSED_OP]->chunkManager()->flushChunks(rc, columOids);
std::vector<int32_t> compressedOpIds = {COMPRESSED_OP_1,
COMPRESSED_OP_2};
return (rtn1 != NO_ERROR ? rtn1 : rtn2);
for (const auto compressedOpId : compressedOpIds)
{
auto rtn = m_colOp[compressedOpId]->chunkManager()->flushChunks(
rc, columOids);
if (rtn != NO_ERROR)
return rtn;
rtn = m_dctnry[compressedOpId]->chunkManager()->flushChunks(
rc, columOids);
if (rtn != NO_ERROR)
return rtn;
}
return NO_ERROR;
}
/**
@ -524,7 +540,7 @@ public:
int startTransaction(const TxnID& txnid)
{
int rc = 0;
rc = m_colOp[COMPRESSED_OP]->chunkManager()->startTransaction(txnid);
rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->startTransaction(txnid);
//if ( rc == 0)
// rc = m_dctnry[COMPRESSED_OP]->chunkManager()->startTransaction(txnid);
return rc;
@ -537,7 +553,8 @@ public:
int confirmTransaction (const TxnID& txnid)
{
int rc = 0;
rc = m_colOp[COMPRESSED_OP]->chunkManager()->confirmTransaction (txnid);
rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->confirmTransaction(
txnid);
return rc;
}
@ -549,7 +566,8 @@ public:
int endTransaction(const TxnID& txnid, bool success)
{
int rc = 0;
rc = m_colOp[COMPRESSED_OP]->chunkManager()->endTransaction(txnid, success);
rc = m_colOp[COMPRESSED_OP_1]->chunkManager()->endTransaction(txnid,
success);
//if ( rc == 0)
// rc = m_dctnry[COMPRESSED_OP]->chunkManager()->endTransaction(txnid, success);
return rc;
@ -785,7 +803,16 @@ private:
int op(int compressionType)
{
return (compressionType > 0 ? COMPRESSED_OP : UN_COMPRESSED_OP);
switch (compressionType)
{
case 1:
case 2:
return COMPRESSED_OP_1;
case 3:
return COMPRESSED_OP_2;
}
return 0;
}