database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-12-24 14:20:59 +03:00
Code Activity

MCOL-498 Add DBRootX.PreallocSpace setting in the XML. Dict files extents now contain a correct number of blocks available.

Browse Source
This commit is contained in:
Roman Nozdrin
2018-12-20 06:52:05 +03:00
parent 29becc2971
commit cbdcdb9f10
8 changed files with 131 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
writeengine/dictionary/we_dctnry.cpp
View File

@@ -259,12 +259,14 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth,
if ( m_dFile != NULL )
{
bool optimizePrealloc = ( flag ) ? false : true;
rc = FileOp::initDctnryExtent( m_dFile,
m_dbRoot,
totalSize,
m_dctnryHeader2,
m_totalHdrBytes,
false );
false,
optimizePrealloc );
if (rc != NO_ERROR)
{

4
writeengine/server/we_dmlcommandproc.cpp
View File

@@ -102,13 +102,13 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
bs >> tmp32;
uint32_t dbroot = tmp32;
cout << "processSingleInsert received bytestream length " << bs.length() << endl;
//cout << "processSingleInsert received bytestream length " << bs.length() << endl;
messageqcpp::ByteStream::byte packageType;
bs >> packageType;
insertPkg.read( bs);
uint32_t sessionId = insertPkg.get_SessionID();
cout << " processSingleInsert for session " << sessionId << endl;
//cout << " processSingleInsert for session " << sessionId << endl;
DMLTable* tablePtr = insertPkg.get_Table();
RowList rows = tablePtr->get_RowList();

3
writeengine/shared/we_define.h
View File

@@ -45,7 +45,8 @@ const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file
const int BYTE_PER_BLOCK = 8192; // Num bytes per data block
const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block
const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block
const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024;
const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; // Used for initial number of blocks calculation
const int MAX_INITIAL_EXTENT_BLOCKS_TO_DISK = 256; // Number of blocks in abbrev extent for 8byte col.
// Num rows reserved to disk for 'initial' extent
const int FILE_NAME_SIZE = 200; // Max size of file name
const long long MAX_ALLOW_ERROR_COUNT = 100000; //Max allowable error count

65
writeengine/shared/we_fileop.cpp
View File

@@ -1046,7 +1046,7 @@ int FileOp::initColumnExtent(
// @bug5769 Don't initialize extents or truncate db files on HDFS
// MCOL-498 We don't need sequential segment files if a PM uses SSD either.
if (idbdatafile::IDBPolicy::useHdfs() || !idbdatafile::IDBPolicy::PreallocSpace())
if (idbdatafile::IDBPolicy::useHdfs())
{
//@Bug 3219. update the compression header after the extent is expanded.
if ((!bNewFile) && (m_compressionType) && (bExpandExtent))
@@ -1101,10 +1101,19 @@ int FileOp::initColumnExtent(
else
Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT);
#endif
// MCOL-498 Skip the huge preallocations if the option is set
// for the dbroot
if ( bOptExtension )
{
bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot))
? bOptExtension : false;
}
int savedErrno = 0;
// MCOL-498 Try to preallocate the space, fallback to write if fallocate has failed
if ( !bOptExtension || ( nBlocks < 300 && pFile->fallocate(0, currFileSize, writeSize) ))
// MCOL-498 fallocate the abbreviated extent,
// fallback to sequential write if fallocate failed
if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK
&& pFile->fallocate(0, currFileSize, writeSize) )
)
{
savedErrno = errno;
// Log the failed fallocate() call result
@@ -1817,7 +1826,7 @@ int FileOp::initDctnryExtent(
off64_t currFileSize = pFile->size();
// @bug5769 Don't initialize extents or truncate db files on HDFS
// MCOL-498 We don't need sequential segment files if a PM uses SSD either.
if (idbdatafile::IDBPolicy::useHdfs() || !idbdatafile::IDBPolicy::PreallocSpace())
if (idbdatafile::IDBPolicy::useHdfs())
{
if (m_compressionType)
updateDctnryExtent(pFile, nBlocks);
@@ -1867,12 +1876,21 @@ int FileOp::initDctnryExtent(
else
Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT);
#endif
int savedErrno = 0;
// MCOL-498 Try to preallocate the space, fallback to write if fallocate
// has failed
//if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize))
// MCOL-498 Skip the huge preallocations if the option is set
// for the dbroot
if ( bOptExtension )
{
// Log the failed fallocate() call result
bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot))
? bOptExtension : false;
}
int savedErrno = 0;
// MCOL-498 fallocate the abbreviated extent,
// fallback to sequential write if fallocate failed
if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK
&& pFile->fallocate(0, currFileSize, writeSize) )
)
{
// MCOL-498 Log the failed fallocate() call result
if ( bOptExtension )
{
std::ostringstream oss;
@@ -1935,23 +1953,22 @@ int FileOp::initDctnryExtent(
return ERR_FILE_WRITE;
}
}
}
if (m_compressionType)
updateDctnryExtent(pFile, nBlocks);
// Synchronize to avoid write buffer pile up too much, which could cause
// controllernode to timeout later when it needs to save a snapshot.
pFile->flush();
// CS doesn't account flush timings.
#ifdef PROFILE
if (bExpandExtent)
Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT);
else
Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT);
if (bExpandExtent)
Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT);
else
Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT);
#endif
}
}
} // preallocation fallback end
// MCOL-498 CS has to set a number of blocs in the chunk header
if ( m_compressionType )
{
updateDctnryExtent(pFile, nBlocks);
}
pFile->flush();
}
return NO_ERROR;

8
writeengine/wrapper/we_colop.cpp
View File

@@ -1527,7 +1527,7 @@ void ColumnOp::setColParam(Column& column,
* rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted
* valArray - the array of row values
* oldValArray - the array of old value
* bDelete - yet
* bDelete - yet. The flag must be useless.
* RETURN:
* NO_ERROR if success, other number otherwise
***********************************************************/
@@ -1571,7 +1571,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
}
// MCOL-498 CS hasn't touched any block yet,
// but the row fill be the first in the block.
// but the row filled will be the first in the block.
fistRowInBlock = ( !(curRowId % (rowsInBlock)) ) ? true : false;
if( fistRowInBlock && !bDelete )
fillUpWEmptyVals = true;
@@ -1708,8 +1708,6 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
if ( writeSize )
setEmptyBuf( dataBuf + dataBio + curCol.colWidth, writeSize,
emptyVal, curCol.colWidth );
//fillUpWEmptyVals = false;
//fistRowInBlock = false;
}
rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo);
@@ -1726,7 +1724,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
if( !fillUpWEmptyVals )
emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth);
// MCOL-498 Skip if this is the last block in an extent.
if ( curDataFbo != MAX_NBLOCKS - 1)
if ( curDataFbo % MAX_NBLOCKS != MAX_NBLOCKS - 1 )
{
rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo);
if ( rc != NO_ERROR)