database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-4566: Add rebuildEM tool support to work with compressed files.

Browse Source 
* This patch adds rebuildEM tool support to work with compressed files.
* This patch increases a version of the file header.

Note: Default version of the `rebuildEM` tool was using very old API,
those functions are not present currently. So `rebuildEM` will not work with
files created without compression, because we cannot deduce some info which are
needed to create column extent.
This commit is contained in:
Denis Khalikov
2021-03-10 17:23:13 +03:00
parent 2eec956977
commit 5d497e8821
25 changed files with 1560 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
writeengine/shared/we_chunkmanager.cpp
View File

@ -290,6 +290,25 @@ IDBDataFile* ChunkManager::getFilePtr(const FID& fid,
return (fileData ? fileData->fFilePtr : NULL);
}
//------------------------------------------------------------------------------
// Get/Return IDBDataFile* for specified OID, root, partition, and segment.
// Function is to be used to open column/dict segment file.
// If the IDBDataFile* is not found, then a segment file will be opened using
// the mode (mode) and I/O buffer size (size) that is given. Name of the
// resulting file is returned in filename.
//------------------------------------------------------------------------------
IDBDataFile* ChunkManager::getSegmentFilePtr(
FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth,
std::string& filename, const char* mode, int32_t size, bool useTmpSuffix,
bool isDict) const
{
CompFileData* fileData =
getFileData(fid, root, partition, segment, filename, mode, size,
colDataType, colWidth, useTmpSuffix, isDict);
return (fileData ? fileData->fFilePtr : NULL);
}
//------------------------------------------------------------------------------
// Get/Return CompFileData* for specified column OID, root, partition, and
// segment. If the IDBDataFile* is not found, then a segment file will be opened
@ -411,7 +430,8 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
uint16_t segment,
const char* filename,
const char* mode,
int size)
int size,
BRM::LBID_t lbid)
{
FileID fileID(fid, root, partition, segment);
CompFileData* fileData = new CompFileData(fileID, fid, CalpontSystemCatalog::VARCHAR, width);
@ -440,8 +460,13 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
fileData->fFileHeader.fLongPtrSectData.reset(fileData->fFileHeader.fPtrSection);
}
fCompressor.initHdr(fileData->fFileHeader.fControlData, fileData->fFileHeader.fPtrSection,
// Dictionary store extent width == 0. See more details in function
// `createDictStoreExtent`.
fCompressor.initHdr(fileData->fFileHeader.fControlData,
fileData->fFileHeader.fPtrSection,
/*colWidth=*/0, fileData->fColDataType,
fFileOp->compressionType(), hdrSize);
fCompressor.setLBID0(fileData->fFileHeader.fControlData, lbid);
if (writeHeader(fileData, __LINE__) != NO_ERROR)
{
@ -1376,7 +1401,8 @@ int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* em
// Increment the block count stored in the chunk header used to track how many
// blocks are allocated to the corresponding segment file.
//------------------------------------------------------------------------------
int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
// same here as for dict.
int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid)
{
map<IDBDataFile*, CompFileData*>::iterator i = fFilePtrMap.find(pFile);
@ -1397,6 +1423,7 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
int rc = NO_ERROR;
char* hdr = pFileData->fFileHeader.fControlData;
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
fCompressor.setLBID1(hdr, lbid);
ChunkData* chunkData = (pFileData)->findChunk(0);
if (chunkData != NULL)
@ -1428,7 +1455,8 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
// Increment the block count stored in the chunk header used to track how many
// blocks are allocated to the corresponding segment file.
//------------------------------------------------------------------------------
int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount)
int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount,
BRM::LBID_t lbid)
{
map<IDBDataFile*, CompFileData*>::iterator i = fFilePtrMap.find(pFile);
@ -1485,6 +1513,8 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount)
if (rc == NO_ERROR)
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
if (currentBlockCount)
fCompressor.setLBID1(hdr, lbid);
return rc;
}