You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-30 19:23:07 +03:00
MCOL-4566: Add rebuildEM tool support to work with compressed files.
* This patch adds rebuildEM tool support to work with compressed files. * This patch increases a version of the file header. Note: Default version of the `rebuildEM` tool was using very old API, those functions are not present currently. So `rebuildEM` will not work with files created without compression, because we cannot deduce some info which are needed to create column extent.
This commit is contained in:
@ -290,6 +290,25 @@ IDBDataFile* ChunkManager::getFilePtr(const FID& fid,
|
||||
return (fileData ? fileData->fFilePtr : NULL);
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Get/Return IDBDataFile* for specified OID, root, partition, and segment.
|
||||
// Function is to be used to open column/dict segment file.
|
||||
// If the IDBDataFile* is not found, then a segment file will be opened using
|
||||
// the mode (mode) and I/O buffer size (size) that is given. Name of the
|
||||
// resulting file is returned in filename.
|
||||
//------------------------------------------------------------------------------
|
||||
IDBDataFile* ChunkManager::getSegmentFilePtr(
|
||||
FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
|
||||
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth,
|
||||
std::string& filename, const char* mode, int32_t size, bool useTmpSuffix,
|
||||
bool isDict) const
|
||||
{
|
||||
CompFileData* fileData =
|
||||
getFileData(fid, root, partition, segment, filename, mode, size,
|
||||
colDataType, colWidth, useTmpSuffix, isDict);
|
||||
return (fileData ? fileData->fFilePtr : NULL);
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
// Get/Return CompFileData* for specified column OID, root, partition, and
|
||||
// segment. If the IDBDataFile* is not found, then a segment file will be opened
|
||||
@ -411,7 +430,8 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
|
||||
uint16_t segment,
|
||||
const char* filename,
|
||||
const char* mode,
|
||||
int size)
|
||||
int size,
|
||||
BRM::LBID_t lbid)
|
||||
{
|
||||
FileID fileID(fid, root, partition, segment);
|
||||
CompFileData* fileData = new CompFileData(fileID, fid, CalpontSystemCatalog::VARCHAR, width);
|
||||
@ -440,8 +460,13 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
|
||||
fileData->fFileHeader.fLongPtrSectData.reset(fileData->fFileHeader.fPtrSection);
|
||||
}
|
||||
|
||||
fCompressor.initHdr(fileData->fFileHeader.fControlData, fileData->fFileHeader.fPtrSection,
|
||||
// Dictionary store extent width == 0. See more details in function
|
||||
// `createDictStoreExtent`.
|
||||
fCompressor.initHdr(fileData->fFileHeader.fControlData,
|
||||
fileData->fFileHeader.fPtrSection,
|
||||
/*colWidth=*/0, fileData->fColDataType,
|
||||
fFileOp->compressionType(), hdrSize);
|
||||
fCompressor.setLBID0(fileData->fFileHeader.fControlData, lbid);
|
||||
|
||||
if (writeHeader(fileData, __LINE__) != NO_ERROR)
|
||||
{
|
||||
@ -1376,7 +1401,8 @@ int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* em
|
||||
// Increment the block count stored in the chunk header used to track how many
|
||||
// blocks are allocated to the corresponding segment file.
|
||||
//------------------------------------------------------------------------------
|
||||
int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
|
||||
// same here as for dict.
|
||||
int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid)
|
||||
{
|
||||
map<IDBDataFile*, CompFileData*>::iterator i = fFilePtrMap.find(pFile);
|
||||
|
||||
@ -1397,6 +1423,7 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
|
||||
int rc = NO_ERROR;
|
||||
char* hdr = pFileData->fFileHeader.fControlData;
|
||||
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
|
||||
fCompressor.setLBID1(hdr, lbid);
|
||||
ChunkData* chunkData = (pFileData)->findChunk(0);
|
||||
|
||||
if (chunkData != NULL)
|
||||
@ -1428,7 +1455,8 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
|
||||
// Increment the block count stored in the chunk header used to track how many
|
||||
// blocks are allocated to the corresponding segment file.
|
||||
//------------------------------------------------------------------------------
|
||||
int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount)
|
||||
int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount,
|
||||
BRM::LBID_t lbid)
|
||||
{
|
||||
map<IDBDataFile*, CompFileData*>::iterator i = fFilePtrMap.find(pFile);
|
||||
|
||||
@ -1485,6 +1513,8 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount)
|
||||
if (rc == NO_ERROR)
|
||||
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
|
||||
|
||||
if (currentBlockCount)
|
||||
fCompressor.setLBID1(hdr, lbid);
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user