database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-987 Add LZ4 compression.

Browse Source 
* Adds CompressInterfaceLZ4 which uses LZ4 API for compress/uncompress.
* Adds CMake machinery to search LZ4 on running host.
* All methods which use static data and do not modify any internal data - become `static`,
  so we can use them without creation of the specific object. This is possible, because
  the header specification has not been modified. We still use 2 sections in header, first
  one with file meta data, the second one with pointers for compressed chunks.
* Methods `compress`, `uncompress`, `maxCompressedSize`, `getUncompressedSize` - become
  pure virtual, so we can override them for the other compression algos.
* Adds method `getChunkMagicNumber`, so we can verify chunk magic number
  for each compression algo.
* Renames "s/IDBCompressInterface/CompressInterface/g" according to requirement.
This commit is contained in:
Denis Khalikov
2021-04-01 17:26:38 +03:00
parent dd12bd3cd0
commit cc1c3629c5
45 changed files with 1311 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
writeengine/shared/we_rbmetawriter.cpp
View File

@ -1007,9 +1007,9 @@ void RBMetaWriter::backupHWMChunk(
}
// Read Control header
char controlHdr[ IDBCompressInterface::HDR_BUF_LEN ];
char controlHdr[ CompressInterface::HDR_BUF_LEN ];
rc = fileOp.readFile( dbFile, (unsigned char*)controlHdr,
IDBCompressInterface::HDR_BUF_LEN );
CompressInterface::HDR_BUF_LEN );
if (rc != NO_ERROR)
{
@ -1025,8 +1025,7 @@ void RBMetaWriter::backupHWMChunk(
throw WeException( oss.str(), rc );
}
IDBCompressInterface compressor;
int rc1 = compressor.verifyHdr( controlHdr );
int rc1 = compress::CompressInterface::verifyHdr(controlHdr);
if (rc1 != 0)
{
@ -1045,9 +1044,23 @@ void RBMetaWriter::backupHWMChunk(
throw WeException( oss.str(), rc );
}
auto compressionType =
compress::CompressInterface::getCompressionType(controlHdr);
std::unique_ptr<compress::CompressInterface> compressor(
compress::getCompressInterfaceByType(compressionType));
if (!compressor)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Ivalid compression type " << compressionType;
fileOp.closeFile( dbFile );
throw WeException(oss.str(), rc);
}
// Read Pointer header data
uint64_t hdrSize = compressor.getHdrSize(controlHdr);
uint64_t ptrHdrSize = hdrSize - IDBCompressInterface::HDR_BUF_LEN;
uint64_t hdrSize = compress::CompressInterface::getHdrSize(controlHdr);
uint64_t ptrHdrSize = hdrSize - CompressInterface::HDR_BUF_LEN;
char* pointerHdr = new char[ptrHdrSize];
rc = fileOp.readFile( dbFile, (unsigned char*)pointerHdr, ptrHdrSize );
@ -1067,7 +1080,8 @@ void RBMetaWriter::backupHWMChunk(
}
CompChunkPtrList chunkPtrs;
rc = compressor.getPtrList(pointerHdr, ptrHdrSize, chunkPtrs );
rc = compress::CompressInterface::getPtrList(pointerHdr, ptrHdrSize,
chunkPtrs);
delete[] pointerHdr;
if (rc != 0)
@ -1087,7 +1101,7 @@ void RBMetaWriter::backupHWMChunk(
unsigned int blockOffsetWithinChunk = 0;
unsigned char* buffer = 0;
uint64_t chunkSize = 0;
compressor.locateBlock(startingHWM, chunkIndex, blockOffsetWithinChunk);
compressor->locateBlock(startingHWM, chunkIndex, blockOffsetWithinChunk);
if (chunkIndex < chunkPtrs.size())
{