database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Merge pull request #1808 from denis0x0D/MCOL-4566/rebuild_em_compressed

Browse Source 
MCOL-4566: Add rebuildEM tool support to work with compressed files.
This commit is contained in:
Roman Nozdrin
2021-04-02 17:52:34 +03:00
committed by GitHub
parent dc1e7f14bd 5d497e8821
commit 05863a3fb5
25 changed files with 1560 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
utils/compress/idbcompress.cpp
View File

@ -38,6 +38,7 @@ namespace
const uint64_t MAGIC_NUMBER = 0xfdc119a384d0778eULL;
const uint64_t VERSION_NUM1 = 1;
const uint64_t VERSION_NUM2 = 2;
const uint64_t VERSION_NUM3 = 3;
const int COMPRESSED_CHUNK_INCREMENT_SIZE = 8192;
const int PTR_SECTION_OFFSET = compress::IDBCompressInterface::HDR_BUF_LEN;
@ -71,6 +72,8 @@ struct CompressedDBFileHeader
uint64_t fBlockCount;
uint64_t fColumnWidth;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
uint64_t fLBID0;
uint64_t fLBID1;
};
// Make the header to be 4K, regardless number of fields being defined/used in header.
@ -80,18 +83,6 @@ union CompressedDBFileHeaderBlock
char fDummy[compress::IDBCompressInterface::HDR_BUF_LEN];
};
void initCompressedDBFileHeader(void* hdrBuf, int compressionType, int hdrSize)
{
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM2;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = 0;
hdr->fHeader.fColDataType = execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
}
void initCompressedDBFileHeader(
void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType,
@ -99,12 +90,14 @@ void initCompressedDBFileHeader(
{
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM2;
hdr->fHeader.fVersionNum = VERSION_NUM3;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = columnWidth;
hdr->fHeader.fColDataType = colDataType;
hdr->fHeader.fLBID0 = 0;
hdr->fHeader.fLBID1 = 0;
}
} // namespace
@ -351,23 +344,18 @@ void IDBCompressInterface::storePtrs(const std::vector<uint64_t>& ptrs, void* pt
storePtrs(ptrs, reinterpret_cast<char*>(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a column file.
//------------------------------------------------------------------------------
void IDBCompressInterface::initHdr(void* hdrBuf, int compressionType) const
{
memset(hdrBuf, 0, HDR_BUF_LEN * 2);
initCompressedDBFileHeader(hdrBuf, compressionType, HDR_BUF_LEN * 2);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a dictionary file.
//------------------------------------------------------------------------------
void IDBCompressInterface::initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const
void IDBCompressInterface::initHdr(
void* hdrBuf, void* ptrBuf, uint32_t colWidth,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType, int hdrSize) const
{
memset(hdrBuf, 0, HDR_BUF_LEN);
memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN);
initCompressedDBFileHeader(hdrBuf, compressionType, hdrSize);
initCompressedDBFileHeader(hdrBuf, colWidth, columnType, compressionType,
hdrSize);
}
//------------------------------------------------------------------------------
@ -383,6 +371,15 @@ void IDBCompressInterface::initHdr(
compressionType, HDR_BUF_LEN * 2);
}
//------------------------------------------------------------------------------
// Get the header's version number
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getVersionNumber(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fVersionNum);
}
//------------------------------------------------------------------------------
// Set the file's block count
//------------------------------------------------------------------------------
@ -415,6 +412,59 @@ uint64_t IDBCompressInterface::getHdrSize(const void* hdrBuf) const
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fHeaderSize);
}
//------------------------------------------------------------------------------
// Get column type
//-----------------------------------------------------------------------------
execplan::CalpontSystemCatalog::ColDataType
IDBCompressInterface::getColDataType(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColDataType);
}
//------------------------------------------------------------------------------
// Get column width
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColumnWidth);
}
//------------------------------------------------------------------------------
// Get start LBID
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getLBID0(const void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID0);
}
//------------------------------------------------------------------------------
// Set start LBID
//------------------------------------------------------------------------------
void IDBCompressInterface::setLBID0(void* hdrBuf, uint64_t lbid) const
{
if (lbid)
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID0 = lbid;
}
//------------------------------------------------------------------------------
// Get start LBID
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getLBID1(const void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID1);
}
//------------------------------------------------------------------------------
// Set start LBID
//------------------------------------------------------------------------------
void IDBCompressInterface::setLBID1(void* hdrBuf, uint64_t lbid) const
{
if (lbid)
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID1 = lbid;
}
//------------------------------------------------------------------------------
// Calculates the chunk and block offset within the chunk for the specified
// block number.

65
utils/compress/idbcompress.h
View File

@ -101,20 +101,15 @@ public:
*/
EXPORT int uncompress(const char* in, size_t inLen, char* out) const;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes
*/
EXPORT void initHdr(void* hdrBuf, int compressionType) const;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN bytes
* @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes
*/
EXPORT void initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const;
EXPORT void initHdr(void* hdrBuf, void* ptrBuf, uint32_t columnWidht,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType, int hdrSize) const;
/**
* Initialize header buffer at start of compressed db file.
@ -199,6 +194,11 @@ public:
/*
* Mutator methods for the block count in the file
*/
/**
* getVersionNumber
*/
EXPORT uint64_t getVersionNumber(const void* hdrBuf) const;
/**
* setBlockCount
*/
@ -222,6 +222,37 @@ public:
*/
EXPORT uint64_t getHdrSize(const void* hdrBuf) const;
/**
* getColumnType
*/
EXPORT execplan::CalpontSystemCatalog::ColDataType
getColDataType(const void* hdrBuf) const;
/**
* getColumnWidth
*/
EXPORT uint64_t getColumnWidth(const void* hdrBuf) const;
/**
* getLBID
*/
EXPORT uint64_t getLBID0(const void* hdrBuf) const;
/**
* setBID
*/
EXPORT void setLBID0(void* hdrBuf, uint64_t lbid) const;
/**
* getLBID
*/
EXPORT uint64_t getLBID1(const void* hdrBuf) const;
/**
* setBID
*/
EXPORT void setLBID1(void* hdrBuf, uint64_t lbid) const;
/**
* Mutator methods for the user padding bytes
*/
@ -286,8 +317,7 @@ inline int IDBCompressInterface::uncompress(const char* in, size_t inLen, char*
{
return 0;
}
inline void IDBCompressInterface::initHdr(void*, int) const {}
inline void IDBCompressInterface::initHdr(void*, void*, int, int) const {}
inline void IDBCompressInterface::initHdr(void*, void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int, int) const {}
inline void initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int) const {}
inline int IDBCompressInterface::verifyHdr(const void*) const
{
@ -317,6 +347,11 @@ inline int IDBCompressInterface::padCompressedChunks(unsigned char* buf, unsigne
{
return -1;
}
inline uint64_t
IDBCompressInterface::getVersionNumber(const void* hdrBuf) const
{
return 0;
}
inline void IDBCompressInterface::setBlockCount(void* hdrBuf, uint64_t count) const {}
inline uint64_t IDBCompressInterface::getBlockCount(const void* hdrBuf) const
{
@ -327,10 +362,20 @@ inline uint64_t IDBCompressInterface::getHdrSize(const void*) const
{
return 0;
}
inline execplan::CalpontSystemCatalog::ColDataType
IDBCompressInterface::getColDataType(const void* hdrBuf) const
{
return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
}
inline uint64_t getColumnWidth(const void* hdrBuf) const { return 0; }
inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize)
{
return uncompSize;
}
inline uint64_t getLBID0(const void* hdrBuf) const { return 0; }
void setLBID0(void* hdrBuf, uint64_t lbid) const {}
inline uint64_t getLBID1(const void* hdrBuf) const { return 0; }
void setLBID1(void* hdrBuf, uint64_t lbid) const {}
inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen)
{
return false;