database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-498 Returned changes in we_colbufcompressed.* b/c they are neccesary

Browse Source 
to track compressed data size.
This commit is contained in:
Roman Nozdrin
2019-01-21 11:39:15 +03:00
parent abf7ef80c2
commit ecbf6b7606
3 changed files with 16 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
writeengine/bulk/we_colbufcompressed.cpp
View File

@ -167,11 +167,16 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf(
// file, and instead buffer up the data to be compressed in 4M chunks before
// writing it out.
//------------------------------------------------------------------------------
int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize,
bool fillUpWEmpties)
{
if (writeSize == 0) // skip unnecessary write, if 0 bytes given
return NO_ERROR;
int fillUpWEmptiesWriteSize = 0;
if (fillUpWEmpties)
fillUpWEmptiesWriteSize = BYTE_PER_BLOCK - writeSize % BYTE_PER_BLOCK;
// If we are starting a new file, we need to reinit the buffer and
// find out what our file offset should be set to.
if (!fToBeCompressedCapacity)
@ -219,7 +224,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
// Expand the compression buffer size if working with an abbrev extent, and
// the bytes we are about to add will overflow the abbreviated extent.
if ((fToBeCompressedCapacity < IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) &&
((fNumBytes + writeSize) > fToBeCompressedCapacity) )
((fNumBytes + writeSize + fillUpWEmptiesWriteSize) > fToBeCompressedCapacity) )
{
std::ostringstream oss;
oss << "Expanding abbrev to-be-compressed buffer for: OID-" <<
@ -231,7 +236,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN;
}
if ((fNumBytes + writeSize) <= fToBeCompressedCapacity)
if ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) <= fToBeCompressedCapacity)
{
if (fLog->isDebug( DEBUG_2 ))
{
@ -242,12 +247,14 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; addBytes-" << writeSize <<
"; extraBytes-" << fillUpWEmptiesWriteSize <<
"; totBytes-" << (fNumBytes + writeSize);
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
memcpy(bufOffset, (fBuffer + startOffset), writeSize);
fNumBytes += writeSize;
fNumBytes += fillUpWEmptiesWriteSize;
}
else // Not enough room to add all the data to the to-be-compressed buffer
{
@ -338,6 +345,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut);
fNumBytes += writeSizeOut;
fNumBytes += fillUpWEmptiesWriteSize;
}
startOffsetX += writeSizeOut;
@ -347,7 +355,6 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Compress and write out the data in the to-be-compressed buffer.
// Also may write out the compression header.

6
writeengine/bulk/we_colbufcompressed.h
View File

@ -83,11 +83,11 @@ public:
*
* @param startOffset The buffer offset from where the write should begin
* @param writeSize The number of bytes to be written to the file
* @param fillUpWNulls The flag to fill the buffer with NULLs up to
* the block boundary.
* @param fillUpWEmpties The flag to fill the buffer with empty magic
* values up to the block boundary.
*/
virtual int writeToFile(int startOffset, int writeSize,
bool fillUpWNulls = false);
bool fillUpWEmpties = false);
private:

4
writeengine/bulk/we_colbufmgr.cpp
View File

@ -573,7 +573,7 @@ int ColumnBufferManager::writeToFileExtentCheck(
if (availableFileSize >= writeSize)
{
int rc = fCBuf->writeToFile(startOffset, writeSize);
int rc = fCBuf->writeToFile(startOffset, writeSize, fillUpWEmpties);
if (rc != NO_ERROR)
{
@ -632,7 +632,7 @@ int ColumnBufferManager::writeToFileExtentCheck(
}
int writeSize2 = writeSize - writeSize1;
rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2);
rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2, fillUpWEmpties);
if (rc != NO_ERROR)
{