database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-4566: Add rebuildEM tool support to work with compressed files.

Browse Source 
* This patch adds rebuildEM tool support to work with compressed files.
* This patch increases a version of the file header.

Note: Default version of the `rebuildEM` tool was using very old API,
those functions are not present currently. So `rebuildEM` will not work with
files created without compression, because we cannot deduce some info which are
needed to create column extent.
This commit is contained in:
Denis Khalikov
2021-03-10 17:23:13 +03:00
parent 2eec956977
commit 5d497e8821
25 changed files with 1560 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
utils/compress/idbcompress.cpp
View File

@ -38,6 +38,7 @@ namespace
const uint64_t MAGIC_NUMBER = 0xfdc119a384d0778eULL;
const uint64_t VERSION_NUM1 = 1;
const uint64_t VERSION_NUM2 = 2;
const uint64_t VERSION_NUM3 = 3;
const int COMPRESSED_CHUNK_INCREMENT_SIZE = 8192;
const int PTR_SECTION_OFFSET = compress::IDBCompressInterface::HDR_BUF_LEN;
@ -71,6 +72,8 @@ struct CompressedDBFileHeader
uint64_t fBlockCount;
uint64_t fColumnWidth;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
uint64_t fLBID0;
uint64_t fLBID1;
};
// Make the header to be 4K, regardless number of fields being defined/used in header.
@ -80,18 +83,6 @@ union CompressedDBFileHeaderBlock
char fDummy[compress::IDBCompressInterface::HDR_BUF_LEN];
};
void initCompressedDBFileHeader(void* hdrBuf, int compressionType, int hdrSize)
{
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM2;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = 0;
hdr->fHeader.fColDataType = execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
}
void initCompressedDBFileHeader(
void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType,
@ -99,12 +90,14 @@ void initCompressedDBFileHeader(
{
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM2;
hdr->fHeader.fVersionNum = VERSION_NUM3;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = columnWidth;
hdr->fHeader.fColDataType = colDataType;
hdr->fHeader.fLBID0 = 0;
hdr->fHeader.fLBID1 = 0;
}
} // namespace
@ -351,23 +344,18 @@ void IDBCompressInterface::storePtrs(const std::vector<uint64_t>& ptrs, void* pt
storePtrs(ptrs, reinterpret_cast<char*>(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a column file.
//------------------------------------------------------------------------------
void IDBCompressInterface::initHdr(void* hdrBuf, int compressionType) const
{
memset(hdrBuf, 0, HDR_BUF_LEN * 2);
initCompressedDBFileHeader(hdrBuf, compressionType, HDR_BUF_LEN * 2);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a dictionary file.
//------------------------------------------------------------------------------
void IDBCompressInterface::initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const
void IDBCompressInterface::initHdr(
void* hdrBuf, void* ptrBuf, uint32_t colWidth,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType, int hdrSize) const
{
memset(hdrBuf, 0, HDR_BUF_LEN);
memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN);
initCompressedDBFileHeader(hdrBuf, compressionType, hdrSize);
initCompressedDBFileHeader(hdrBuf, colWidth, columnType, compressionType,
hdrSize);
}
//------------------------------------------------------------------------------
@ -383,6 +371,15 @@ void IDBCompressInterface::initHdr(
compressionType, HDR_BUF_LEN * 2);
}
//------------------------------------------------------------------------------
// Get the header's version number
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getVersionNumber(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fVersionNum);
}
//------------------------------------------------------------------------------
// Set the file's block count
//------------------------------------------------------------------------------
@ -415,6 +412,59 @@ uint64_t IDBCompressInterface::getHdrSize(const void* hdrBuf) const
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fHeaderSize);
}
//------------------------------------------------------------------------------
// Get column type
//-----------------------------------------------------------------------------
execplan::CalpontSystemCatalog::ColDataType
IDBCompressInterface::getColDataType(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColDataType);
}
//------------------------------------------------------------------------------
// Get column width
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColumnWidth);
}
//------------------------------------------------------------------------------
// Get start LBID
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getLBID0(const void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID0);
}
//------------------------------------------------------------------------------
// Set start LBID
//------------------------------------------------------------------------------
void IDBCompressInterface::setLBID0(void* hdrBuf, uint64_t lbid) const
{
if (lbid)
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID0 = lbid;
}
//------------------------------------------------------------------------------
// Get start LBID
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getLBID1(const void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID1);
}
//------------------------------------------------------------------------------
// Set start LBID
//------------------------------------------------------------------------------
void IDBCompressInterface::setLBID1(void* hdrBuf, uint64_t lbid) const
{
if (lbid)
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID1 = lbid;
}
//------------------------------------------------------------------------------
// Calculates the chunk and block offset within the chunk for the specified
// block number.