database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-987 Add LZ4 compression.

Browse Source 
* Adds CompressInterfaceLZ4 which uses LZ4 API for compress/uncompress.
* Adds CMake machinery to search LZ4 on running host.
* All methods which use static data and do not modify any internal data - become `static`,
  so we can use them without creation of the specific object. This is possible, because
  the header specification has not been modified. We still use 2 sections in header, first
  one with file meta data, the second one with pointers for compressed chunks.
* Methods `compress`, `uncompress`, `maxCompressedSize`, `getUncompressedSize` - become
  pure virtual, so we can override them for the other compression algos.
* Adds method `getChunkMagicNumber`, so we can verify chunk magic number
  for each compression algo.
* Renames "s/IDBCompressInterface/CompressInterface/g" according to requirement.
This commit is contained in:
Denis Khalikov
2021-04-01 17:26:38 +03:00
parent dd12bd3cd0
commit cc1c3629c5
45 changed files with 1311 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
writeengine/server/we_getfilesizes.cpp
View File

@ -96,7 +96,7 @@ size_t readFillBuffer(
return totalBytesRead;
}
off64_t getCompressedDataSize(string& fileName)
static off64_t getCompressedDataSize(string& fileName)
{
off64_t dataSize = 0;
IDBDataFile* pFile = 0;
@ -119,21 +119,21 @@ off64_t getCompressedDataSize(string& fileName)
throw std::runtime_error(oss.str());
}
IDBCompressInterface decompressor;
//--------------------------------------------------------------------------
// Read headers and extract compression pointers
//--------------------------------------------------------------------------
char hdr1[IDBCompressInterface::HDR_BUF_LEN];
nBytes = readFillBuffer( pFile, hdr1, IDBCompressInterface::HDR_BUF_LEN);
char hdr1[CompressInterface::HDR_BUF_LEN];
nBytes = readFillBuffer( pFile, hdr1, CompressInterface::HDR_BUF_LEN);
if ( nBytes != IDBCompressInterface::HDR_BUF_LEN )
if ( nBytes != CompressInterface::HDR_BUF_LEN )
{
std::ostringstream oss;
oss << "Error reading first header from file " << fileName;
throw std::runtime_error(oss.str());
}
int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - IDBCompressInterface::HDR_BUF_LEN;
int64_t ptrSecSize = compress::CompressInterface::getHdrSize(hdr1) -
CompressInterface::HDR_BUF_LEN;
char* hdr2 = new char[ptrSecSize];
nBytes = readFillBuffer( pFile, hdr2, ptrSecSize);
@ -145,7 +145,8 @@ off64_t getCompressedDataSize(string& fileName)
}
CompChunkPtrList chunkPtrs;
int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
int rc =
compress::CompressInterface::getPtrList(hdr2, ptrSecSize, chunkPtrs);
delete[] hdr2;
if (rc != 0)