database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-4685: Eliminate some irrelevant settings (uncompressed data and extents per file).

Browse Source 
This patch:
1. Removes the option to declare uncompressed columns (set columnstore_compression_type = 0).
2. Ignores [COMMENT '[compression=0] option at table or column level (no error messages, just disregard).
3. Removes the option to set more than 2 extents per file (ExtentsPreSegmentFile).
4. Updates rebuildEM tool to support up to 10 dictionary extent per dictionary segment file.
5. Adds check for `DBRootStorageType` for rebuildEM tool.
6. Renamed rebuildEM to mcsRebuildEM.
This commit is contained in:
Denis Khalikov
2021-05-05 15:02:08 +03:00
parent ddf19a9cc3
commit 606194e6e4
23 changed files with 170 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
writeengine/bulk/we_colbufcompressed.cpp
View File

@ -584,21 +584,18 @@ int ColumnBufferCompressed::saveCompressionHeaders( )
char hdrBuf[IDBCompressInterface::HDR_BUF_LEN * 2];
RETURN_ON_ERROR(fColInfo->colOp->readHeaders(fFile, hdrBuf));
auto lbid = fCompressor->getLBID0(hdrBuf);
auto lbid = fCompressor->getLBIDByIndex(hdrBuf, 0);
fCompressor->initHdr(hdrBuf, fColInfo->column.width,
fColInfo->column.dataType,
fColInfo->column.compressionType);
fCompressor->setBlockCount(hdrBuf,
(fColInfo->getFileSize() / BYTE_PER_BLOCK) );
// If lbid written in the header is not 0 - we are running for the next extent for column
// segment file.
if (lbid)
{
fCompressor->setLBID0(hdrBuf, lbid);
fCompressor->setLBID1(hdrBuf, fColInfo->getLastUpdatedLBID());
}
fCompressor->setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 1);
else
{
fCompressor->setLBID0(hdrBuf, fColInfo->getLastUpdatedLBID());
}
fCompressor->setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 0);
std::vector<uint64_t> ptrs;

6
writeengine/server/we_ddlcommandproc.cpp
View File

@ -59,10 +59,8 @@ WE_DDLCommandProc::WE_DDLCommandProc()
if (fpc.length() != 0)
filesPerColumnPartition = cf->uFromText(fpc);
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if (epsf.length() != 0)
extentsPerSegmentFile = cf->uFromText(epsf);
// MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
string dbct = cf->getConfig("SystemConfig", "DBRootCount");

2
writeengine/server/we_ddlcommandproc.h
View File

@ -112,7 +112,7 @@ private:
WriteEngineWrapper fWEWrapper;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
};
}
#undef EXPORT

6
writeengine/server/we_dmlcommandproc.cpp
View File

@ -70,10 +70,8 @@ WE_DMLCommandProc::WE_DMLCommandProc()
if (fpc.length() != 0)
filesPerColumnPartition = cf->uFromText(fpc);
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if (epsf.length() != 0)
extentsPerSegmentFile = cf->uFromText(epsf);
// MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
string dbct = cf->getConfig("SystemConfig", "DBRootCount");

1
writeengine/server/we_dmlcommandproc.h
View File

@ -132,6 +132,7 @@ private:
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
Log fLog;
static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
};
}

10
writeengine/shared/we_chunkmanager.cpp
View File

@ -466,7 +466,7 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
fileData->fFileHeader.fPtrSection,
/*colWidth=*/0, fileData->fColDataType,
fFileOp->compressionType(), hdrSize);
fCompressor.setLBID0(fileData->fFileHeader.fControlData, lbid);
fCompressor.setLBIDByIndex(fileData->fFileHeader.fControlData, lbid, 0);
if (writeHeader(fileData, __LINE__) != NO_ERROR)
{
@ -1423,7 +1423,7 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int6
int rc = NO_ERROR;
char* hdr = pFileData->fFileHeader.fControlData;
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
fCompressor.setLBID1(hdr, lbid);
fCompressor.setLBIDByIndex(hdr, lbid, 1);
ChunkData* chunkData = (pFileData)->findChunk(0);
if (chunkData != NULL)
@ -1514,7 +1514,11 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount,
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
if (currentBlockCount)
fCompressor.setLBID1(hdr, lbid);
{
// Append to the end.
uint64_t lbidCount = fCompressor.getLBIDCount(hdr);
fCompressor.setLBIDByIndex(hdr, lbid, lbidCount);
}
return rc;
}

4
writeengine/shared/we_config.cpp
View File

@ -153,10 +153,6 @@ void Config::checkReload( )
// Initialize extents per segment file
//--------------------------------------------------------------------------
m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if ( epsf.length() != 0 )
m_ExtentsPerSegmentFile = cf->uFromText(epsf);
//--------------------------------------------------------------------------
// Initialize bulk load process priority

8
writeengine/shared/we_fileop.cpp
View File

@ -818,7 +818,7 @@ int FileOp::extendFile(
{
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, startLbid);
compressor.setLBIDByIndex(hdrs, startLbid, 0);
}
}
@ -978,7 +978,7 @@ int FileOp::addExtentExactFile(
{
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, startLbid);
compressor.setLBIDByIndex(hdrs, startLbid, 0);
}
}
@ -1067,7 +1067,7 @@ int FileOp::initColumnExtent(
char hdrs[IDBCompressInterface::HDR_BUF_LEN * 2];
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, lbid);
compressor.setLBIDByIndex(hdrs, lbid, 0);
if (bAbbrevExtent)
compressor.setBlockCount(hdrs, nBlocks);
@ -1349,7 +1349,7 @@ int FileOp::writeInitialCompColumnChunk(
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setBlockCount(hdrs, nBlocksAllocated);
compressor.setLBID0(hdrs, startLBID);
compressor.setLBIDByIndex(hdrs, startLBID, 0);
// Store compression pointers in the header
std::vector<uint64_t> ptrs;