database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-498 Returned changes in we_colbufcompressed.* b/c they are neccesary

Browse Source 
to track compressed data size.
This commit is contained in:
Roman Nozdrin
2019-01-21 11:39:15 +03:00
parent abf7ef80c2
commit ecbf6b7606
3 changed files with 16 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
writeengine/bulk/we_colbufcompressed.cpp
View File

@ -167,11 +167,16 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf(
// file, and instead buffer up the data to be compressed in 4M chunks before // file, and instead buffer up the data to be compressed in 4M chunks before
// writing it out. // writing it out.
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize,
bool fillUpWEmpties)
{ {
if (writeSize == 0) // skip unnecessary write, if 0 bytes given if (writeSize == 0) // skip unnecessary write, if 0 bytes given
return NO_ERROR; return NO_ERROR;
int fillUpWEmptiesWriteSize = 0;
if (fillUpWEmpties)
fillUpWEmptiesWriteSize = BYTE_PER_BLOCK - writeSize % BYTE_PER_BLOCK;
// If we are starting a new file, we need to reinit the buffer and // If we are starting a new file, we need to reinit the buffer and
// find out what our file offset should be set to. // find out what our file offset should be set to.
if (!fToBeCompressedCapacity) if (!fToBeCompressedCapacity)
@ -219,7 +224,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
// Expand the compression buffer size if working with an abbrev extent, and // Expand the compression buffer size if working with an abbrev extent, and
// the bytes we are about to add will overflow the abbreviated extent. // the bytes we are about to add will overflow the abbreviated extent.
if ((fToBeCompressedCapacity < IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) && if ((fToBeCompressedCapacity < IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) &&
((fNumBytes + writeSize) > fToBeCompressedCapacity) ) ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) > fToBeCompressedCapacity) )
{ {
std::ostringstream oss; std::ostringstream oss;
oss << "Expanding abbrev to-be-compressed buffer for: OID-" << oss << "Expanding abbrev to-be-compressed buffer for: OID-" <<
@ -231,7 +236,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN; fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN;
} }
if ((fNumBytes + writeSize) <= fToBeCompressedCapacity) if ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) <= fToBeCompressedCapacity)
{ {
if (fLog->isDebug( DEBUG_2 )) if (fLog->isDebug( DEBUG_2 ))
{ {
@ -242,12 +247,14 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
"; part-" << fColInfo->curCol.dataFile.fPartition << "; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment << "; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; addBytes-" << writeSize << "; addBytes-" << writeSize <<
"; extraBytes-" << fillUpWEmptiesWriteSize <<
"; totBytes-" << (fNumBytes + writeSize); "; totBytes-" << (fNumBytes + writeSize);
fLog->logMsg( oss.str(), MSGLVL_INFO2 ); fLog->logMsg( oss.str(), MSGLVL_INFO2 );
} }
memcpy(bufOffset, (fBuffer + startOffset), writeSize); memcpy(bufOffset, (fBuffer + startOffset), writeSize);
fNumBytes += writeSize; fNumBytes += writeSize;
fNumBytes += fillUpWEmptiesWriteSize;
} }
else // Not enough room to add all the data to the to-be-compressed buffer else // Not enough room to add all the data to the to-be-compressed buffer
{ {
@ -338,6 +345,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut);
fNumBytes += writeSizeOut; fNumBytes += writeSizeOut;
fNumBytes += fillUpWEmptiesWriteSize;
} }
startOffsetX += writeSizeOut; startOffsetX += writeSizeOut;
@ -347,7 +355,6 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Compress and write out the data in the to-be-compressed buffer. // Compress and write out the data in the to-be-compressed buffer.
// Also may write out the compression header. // Also may write out the compression header.

6
writeengine/bulk/we_colbufcompressed.h
View File

@ -83,11 +83,11 @@ public:
* *
* @param startOffset The buffer offset from where the write should begin * @param startOffset The buffer offset from where the write should begin
* @param writeSize The number of bytes to be written to the file * @param writeSize The number of bytes to be written to the file
* @param fillUpWNulls The flag to fill the buffer with NULLs up to * @param fillUpWEmpties The flag to fill the buffer with empty magic
* the block boundary. * values up to the block boundary.
*/ */
virtual int writeToFile(int startOffset, int writeSize, virtual int writeToFile(int startOffset, int writeSize,
bool fillUpWNulls = false); bool fillUpWEmpties = false);
private: private:

4
writeengine/bulk/we_colbufmgr.cpp
View File

@ -573,7 +573,7 @@ int ColumnBufferManager::writeToFileExtentCheck(
if (availableFileSize >= writeSize) if (availableFileSize >= writeSize)
{ {
int rc = fCBuf->writeToFile(startOffset, writeSize); int rc = fCBuf->writeToFile(startOffset, writeSize, fillUpWEmpties);
if (rc != NO_ERROR) if (rc != NO_ERROR)
{ {
@ -632,7 +632,7 @@ int ColumnBufferManager::writeToFileExtentCheck(
} }
int writeSize2 = writeSize - writeSize1; int writeSize2 = writeSize - writeSize1;
rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2); rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2, fillUpWEmpties);
if (rc != NO_ERROR) if (rc != NO_ERROR)
{ {