database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-24 22:42:05 +03:00
Code Activity

MCOL-994 handle a second abbreviated extent in case it moved dbroots because of redistribute remove.

Browse Source
This commit is contained in:
David Hall
2017-12-07 10:48:06 -06:00
parent b295c8fc0f
commit 34799d8d30
6 changed files with 102 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
versioning/BRM/extentmap.h
View File

@ -684,8 +684,8 @@ public:
*
* Function should only be called for System Catalog OIDs, as it assumes
* the OID is fully contained on a single DBRoot, returning the first
* DBRoot found. This only makes since for a System Catalog OID, because
* all other column OIDs can span multiple DBRoots.
* DBRoot found. This only makes sence for a System Catalog
* OID, because all other column OIDs can span multiple DBRoots.
*
* @param oid The system catalog OID
* @param dbRoot (out) the DBRoot holding the system catalog OID

58
writeengine/bulk/we_columninfo.cpp
View File

@ -111,7 +111,7 @@ unsigned int compactVarBinary(char* charTmpBuf, int fieldLength)
}
}
namespace WriteEngine
{
@ -209,7 +209,7 @@ ColumnInfo::ColumnInfo(Log* logger,
// fAutoIncMgr = new ColumnAutoIncJob(logger);
}
}
//------------------------------------------------------------------------------
// ColumnInfo destructor
//------------------------------------------------------------------------------
@ -250,7 +250,7 @@ void ColumnInfo::clearMemory( )
fDictBlocks.clear();
}
//------------------------------------------------------------------------------
// If at the start of the job, We have encountered a PM that has no DB file for
// this column, or whose HWM extent is disabled; then this function is called
@ -281,7 +281,7 @@ void ColumnInfo::setupDelayedFileCreation(
column.compressionType,
dbRoot, partition, segment );
}
//------------------------------------------------------------------------------
// Create a DB file as part of delayed file creation. See setupDelayedFile-
// Creation for an explanation.
@ -506,7 +506,7 @@ int ColumnInfo::createDelayedFileIfNeeded( const std::string& tableName )
return rc;
}
//------------------------------------------------------------------------------
// Add an extent for this column. The next segment file in the DBRoot,
// partition, segment number rotation will be selected for the extent.
@ -613,7 +613,7 @@ int ColumnInfo::extendColumn( bool saveLBIDForCP )
return rc;
}
//------------------------------------------------------------------------------
// Add a new extent to this column, at the specified DBRoot. Partition may be
// used if DBRoot is empty.
@ -773,7 +773,7 @@ int ColumnInfo::extendColumnNewExtent(
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Fill out existing partial extent to extent boundary, so that we can resume
// inserting rows on an extent boundary basis. This use case should only take
@ -881,7 +881,7 @@ int ColumnInfo::extendColumnOldExtent(
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Either add or update the File object, so that it has the updated HWM.
// We will access this info to update the HWM in the ExtentMap at the end
@ -933,7 +933,7 @@ void ColumnInfo::addToSegFileList( File& dataFile, HWM hwm )
fSegFileUpdateList.push_back( dataFile );
}
}
//------------------------------------------------------------------------------
// Reset file offset data member attributes when we start working on the next
// extent.
@ -961,7 +961,7 @@ int ColumnInfo::resetFileOffsetsNewExtent(const char* /*hdr*/)
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Set current size of file in raw (uncompressed) bytes, given the specified
// hwm. abbrevFlag indicates whether this is a fixed size abbreviated extent.
@ -994,7 +994,7 @@ void ColumnInfo::setFileSize( HWM hwm, int abbrevFlag )
}
}
}
//------------------------------------------------------------------------------
// If we are dealing with the first extent in the first segment file for this
// column, and the segment file is still equal to 256K rows, then we set the
@ -1026,7 +1026,7 @@ void ColumnInfo::setAbbrevExtentCheck( )
}
}
}
//------------------------------------------------------------------------------
// If this is an abbreviated extent, we expand the extent to a full extent on
// disk, by initializing the necessary number of remaining blocks.
@ -1102,7 +1102,7 @@ int ColumnInfo::expandAbbrevExtent( bool bRetainFilePos )
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Close the current Column file.
//------------------------------------------------------------------------------
@ -1116,7 +1116,7 @@ int ColumnInfo::closeColumnFile(bool /*bCompletingExtent*/, bool /*bAbort*/)
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Initialize fLastInputRowInCurrentExtent used in detecting when a Read Buffer
// is crossing an extent boundary, so that we can accurately track the min/max
@ -1163,7 +1163,7 @@ void ColumnInfo::lastInputRowInExtentInit( bool bIsNewExtent )
bIsNewExtent );
}
}
//------------------------------------------------------------------------------
// Increment fLastRIDInExtent to the end of the next extent.
//------------------------------------------------------------------------------
@ -1171,7 +1171,7 @@ void ColumnInfo::lastInputRowInExtentInc( )
{
fLastInputRowInCurrentExtent += fRowsPerExtent;
}
//------------------------------------------------------------------------------
// Parsing is complete for this column. Flush pending data. Close the current
// segment file, and corresponding dictionary store file (if applicable). Also
@ -1234,7 +1234,7 @@ int ColumnInfo::finishParsing( )
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Store updated column information in BRMReporter for this column at EOJ;
// so that Extent Map CP information and HWM's can be updated.
@ -1259,7 +1259,7 @@ void ColumnInfo::getBRMUpdateInfo( BRMReporter& brmReporter )
if (entriesAdded > 0)
getCPInfoForBRM( brmReporter );
}
//------------------------------------------------------------------------------
// Get updated Casual Partition (CP) information for BRM for this column at EOJ.
//------------------------------------------------------------------------------
@ -1267,7 +1267,7 @@ void ColumnInfo::getCPInfoForBRM( BRMReporter& brmReporter )
{
fColExtInf->getCPInfoForBRM(column, brmReporter);
}
//------------------------------------------------------------------------------
// Get updated HWM information for BRM for this column at EOJ.
// Returns count of the number of HWM entries added to the BRMReporter.
@ -1340,7 +1340,7 @@ int ColumnInfo::getHWMInfoForBRM( BRMReporter& brmReporter )
return entriesAdded;
}
//------------------------------------------------------------------------------
// Setup initial extent we will begin loading at start of import.
// DBRoot, partition, segment, etc for the starting extent are specified.
@ -1460,7 +1460,7 @@ int ColumnInfo::setupInitialColumnExtent(
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Prepare the initial column segment file for import.
//------------------------------------------------------------------------------
@ -1521,7 +1521,7 @@ int ColumnInfo::setupInitialColumnFile( HWM oldHwm, HWM hwm )
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Update the number of bytes in the file, and the free space still remaining.
//------------------------------------------------------------------------------
@ -1542,7 +1542,7 @@ bool ColumnInfo::isFileComplete() const
return false;
}
//------------------------------------------------------------------------------
// Initialize last used auto-increment value from the current "next"
// auto-increment value taken from the system catalog (or BRM).
@ -1575,7 +1575,7 @@ int ColumnInfo::finishAutoInc( )
return rc;
}
//------------------------------------------------------------------------------
// Get current dbroot, partition, segment, and HWM for this column.
//
@ -1597,7 +1597,7 @@ void ColumnInfo::getSegFileInfo( DBRootExtentInfo& fileInfo )
else
fileInfo.fLocalHwm = 0;
}
//------------------------------------------------------------------------------
// Open a new or existing Dictionary store file based on the DBRoot,
// partition, and segment settings in curCol.dataFile.
@ -1741,7 +1741,7 @@ int ColumnInfo::openDctnryStore( bool bMustExist )
return rc;
}
//------------------------------------------------------------------------------
// Close the current Dictionary store file.
//------------------------------------------------------------------------------
@ -1773,7 +1773,7 @@ int ColumnInfo::closeDctnryStore(bool bAbort)
return rc;
}
//------------------------------------------------------------------------------
// Update dictionary store file with specified strings, and return the assigned
// tokens (tokenbuf) to be stored in the corresponding column token file.
@ -1835,7 +1835,7 @@ int ColumnInfo::updateDctnryStore(char* buf,
return NO_ERROR;
}
//------------------------------------------------------------------------------
// No action necessary for uncompressed dictionary files
//------------------------------------------------------------------------------
@ -1856,7 +1856,7 @@ int ColumnInfo::truncateDctnryStore(
{
return NO_ERROR;
}
//------------------------------------------------------------------------------
// utility to convert a Status enumeration to a string
//------------------------------------------------------------------------------

70
writeengine/shared/we_fileop.cpp
View File

@ -593,8 +593,7 @@ int FileOp::extendFile(
return ERR_FILE_NOT_EXIST;
}
pFile = openFile( oid, dbRoot, partition, segment,
segFile, "r+b" );//old file
pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" );//old file
if (pFile == 0)
{
ostringstream oss;
@ -613,7 +612,7 @@ int FileOp::extendFile(
if ( isDebug(DEBUG_1) && getLogger() )
{
std::ostringstream oss;
oss << "Opening existing column file" <<
oss << "Opening existing column file (extendFile)" <<
": OID-" << oid <<
"; DBRoot-" << dbRoot <<
"; part-" << partition <<
@ -651,19 +650,55 @@ int FileOp::extendFile(
ostringstream oss;
oss << "oid: " << oid << " with path " << segFile <<
"; new extent fbo " << hwm << "; number of "
"compressed chunks " << ptrCount;
"compressed chunks " << ptrCount <<
"; chunkIndex " << chunkIndex;
logging::Message::Args args;
args.add("compressed");
args.add(oss.str());
SimpleSysLog::instance()->logMsg(args,
logging::LOG_TYPE_ERROR,
logging::M0103);
return ERR_FILE_NEW_EXTENT_FBO;
// Expand the partial extent to full with emptyVal
// Since fillCompColumnExtentEmptyChunks() messes with the
// file on disk, we need to close it and reopen after or
// the cache isn't updated.
if ((pFile))
closeFile( pFile );
pFile = NULL;
string failedTask; // For return error message, if any.
rc = FileOp::fillCompColumnExtentEmptyChunks(oid,
width,
emptyVal,
dbRoot,
partition,
segment,
hwm,
segFile,
failedTask);
if (rc != NO_ERROR)
{
if (getLogger())
{
std::ostringstream oss;
oss << "FileOp::extendFile: error padding partial compressed extent for " <<
"column OID-" << oid <<
"; DBRoot-" << dbRoot <<
"; part-" << partition <<
"; seg-" << segment <<
"; hwm-" << hwm <<
" " << failedTask.c_str();
getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL );
}
return rc;
}
pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" ); // modified file
}
// Get the latest file header for the caller. If a partial extent was filled out,
// this will be different than when we first read the headers.
if (hdrs)
{
memcpy(hdrs, hdrsIn, sizeof(hdrsIn) );
RETURN_ON_ERROR( readHeaders(pFile, hdrs) );
}
}
else
@ -693,7 +728,26 @@ int FileOp::extendFile(
SimpleSysLog::instance()->logMsg(args,
logging::LOG_TYPE_ERROR,
logging::M0103);
return ERR_FILE_NEW_EXTENT_FBO;
// Expand the partial extent to full with emptyVal
// This generally won't ever happen, as uncompressed files
// are created with full extents.
rc = FileOp::expandAbbrevColumnExtent( pFile, dbRoot,
emptyVal, width);
if (rc != NO_ERROR)
{
if (getLogger())
{
std::ostringstream oss;
oss << "FileOp::extendFile: error padding partial uncompressed extent for " <<
"column OID-" << oid <<
"; DBRoot-" << dbRoot <<
"; part-" << partition <<
"; seg-" << segment <<
"; hwm-" << hwm;
getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL );
}
return rc;
}
}
}
}
@ -2598,7 +2652,7 @@ int FileOp::expandAbbrevColumnExtent(
}
// Add blocks to turn the abbreviated extent into a full extent.
int rc = initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width,
int rc = FileOp::initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width,
false, // existing file
true, // expand existing extent
false); // n/a since not adding new extent

2
writeengine/shared/we_log.h
View File

@ -47,7 +47,7 @@ namespace WriteEngine
{
const std::string MSG_LEVEL_STR[] = {
"INFO",
"INFO",
"INFO2",
"WARN",
"ERR ",
"CRIT" };

2
writeengine/splitter/we_sdhandler.cpp
View File

@ -992,7 +992,7 @@ void WESDHandler::onNakResponse(int PmId) {
// Increase the read timeout for this PM thread so that it won't
// consume too much CPU
void WESDHandler::onEodResponse(int PmId) {
if (getDebugLvl())
if (getDebugLvl()>2)
cout << "Received a EOD from " << PmId << endl;
if (fRef.fCmdArgs.getMode() == 0)

8
writeengine/wrapper/we_colopcompress.cpp
View File

@ -188,7 +188,13 @@ int ColumnOpCompress1::expandAbbrevColumnExtent(
IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width)
{
// update the uncompressed initial chunk to full chunk
RETURN_ON_ERROR(m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width));
int rc = m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width);
// ERR_COMP_FILE_NOT_FOUND is acceptable here. It just means that the
// file hasn't been loaded into the chunk manager yet. No big deal.
if (rc != NO_ERROR && rc != ERR_COMP_FILE_NOT_FOUND)
{
return rc;
}
// let the base to physically expand extent.
return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width);