MCOL-994 handle a second abbreviated extent in case it moved dbroots because of redistribute remove.

versioning/BRM/extentmap.h

@@ -684,8 +684,8 @@ public:
 	 *
 	 * Function should only be called for System Catalog OIDs, as it assumes
 	 * the OID is fully contained on a single DBRoot, returning the first
-	 * DBRoot found.  This only makes since for a System Catalog OID, because
+	 * DBRoot found.  This only makes sence for a System Catalog 
-	 * all other column OIDs can span multiple DBRoots.
+	 * OID, because all other column OIDs can span multiple DBRoots.
 	 *
 	 * @param oid The system catalog OID
 	 * @param dbRoot (out) the DBRoot holding the system catalog OID

writeengine/bulk/we_columninfo.cpp

@@ -111,7 +111,7 @@ unsigned int compactVarBinary(char* charTmpBuf, int fieldLength)
 }
 
 }
-
+
 namespace WriteEngine
 {
 
@@ -209,7 +209,7 @@ ColumnInfo::ColumnInfo(Log*             logger,
         // fAutoIncMgr = new ColumnAutoIncJob(logger);
     }
 }
-
+
 //------------------------------------------------------------------------------
 // ColumnInfo destructor
 //------------------------------------------------------------------------------
@@ -250,7 +250,7 @@ void ColumnInfo::clearMemory( )
 
     fDictBlocks.clear();
 }
-
+
 //------------------------------------------------------------------------------
 // If at the start of the job, We have encountered a PM that has no DB file for
 // this column, or whose HWM extent is disabled; then this function is called
@@ -281,7 +281,7 @@ void ColumnInfo::setupDelayedFileCreation(
         column.compressionType,
         dbRoot, partition, segment );
 }
-
+
 //------------------------------------------------------------------------------
 // Create a DB file as part of delayed file creation.  See setupDelayedFile-
 // Creation for an explanation.
@@ -506,7 +506,7 @@ int ColumnInfo::createDelayedFileIfNeeded( const std::string& tableName )
 
     return rc;
 }
-
+
 //------------------------------------------------------------------------------
 // Add an extent for this column.  The next segment file in the DBRoot,
 // partition, segment number rotation will be selected for the extent.
@@ -613,7 +613,7 @@ int ColumnInfo::extendColumn( bool saveLBIDForCP )
 
     return rc;
 }
-
+
 //------------------------------------------------------------------------------
 // Add a new extent to this column, at the specified DBRoot.  Partition may be
 // used if DBRoot is empty.
@@ -773,7 +773,7 @@ int ColumnInfo::extendColumnNewExtent(
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // Fill out existing partial extent to extent boundary, so that we can resume
 // inserting rows on an extent boundary basis.  This use case should only take
@@ -881,7 +881,7 @@ int ColumnInfo::extendColumnOldExtent(
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 //  Either add or update the File object, so that it has the updated HWM.
 //  We will access this info to update the HWM in the ExtentMap at the end
@@ -933,7 +933,7 @@ void ColumnInfo::addToSegFileList( File& dataFile, HWM hwm )
         fSegFileUpdateList.push_back( dataFile );
     }
 }
-
+
 //------------------------------------------------------------------------------
 // Reset file offset data member attributes when we start working on the next
 // extent.
@@ -961,7 +961,7 @@ int ColumnInfo::resetFileOffsetsNewExtent(const char* /*hdr*/)
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // Set current size of file in raw (uncompressed) bytes, given the specified
 // hwm.  abbrevFlag indicates whether this is a fixed size abbreviated extent.
@@ -994,7 +994,7 @@ void ColumnInfo::setFileSize( HWM hwm, int abbrevFlag )
         }
     }
 }
-
+
 //------------------------------------------------------------------------------
 // If we are dealing with the first extent in the first segment file for this
 // column, and the segment file is still equal to 256K rows, then we set the
@@ -1026,7 +1026,7 @@ void ColumnInfo::setAbbrevExtentCheck( )
         }
     }
 }
-
+
 //------------------------------------------------------------------------------
 // If this is an abbreviated extent, we expand the extent to a full extent on
 // disk, by initializing the necessary number of remaining blocks.
@@ -1102,7 +1102,7 @@ int ColumnInfo::expandAbbrevExtent( bool bRetainFilePos )
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // Close the current Column file.
 //------------------------------------------------------------------------------
@@ -1116,7 +1116,7 @@ int ColumnInfo::closeColumnFile(bool /*bCompletingExtent*/, bool /*bAbort*/)
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // Initialize fLastInputRowInCurrentExtent used in detecting when a Read Buffer
 // is crossing an extent boundary, so that we can accurately track the min/max
@@ -1163,7 +1163,7 @@ void ColumnInfo::lastInputRowInExtentInit( bool bIsNewExtent )
                                   bIsNewExtent );
     }
 }
-
+
 //------------------------------------------------------------------------------
 // Increment fLastRIDInExtent to the end of the next extent.
 //------------------------------------------------------------------------------
@@ -1171,7 +1171,7 @@ void ColumnInfo::lastInputRowInExtentInc( )
 {
     fLastInputRowInCurrentExtent += fRowsPerExtent;
 }
-
+
 //------------------------------------------------------------------------------
 // Parsing is complete for this column.  Flush pending data.  Close the current
 // segment file, and corresponding dictionary store file (if applicable).  Also
@@ -1234,7 +1234,7 @@ int ColumnInfo::finishParsing( )
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // Store updated column information in BRMReporter for this column at EOJ;
 // so that Extent Map CP information and HWM's can be updated.
@@ -1259,7 +1259,7 @@ void ColumnInfo::getBRMUpdateInfo( BRMReporter& brmReporter )
     if (entriesAdded > 0)
         getCPInfoForBRM( brmReporter );
 }
-
+
 //------------------------------------------------------------------------------
 // Get updated Casual Partition (CP) information for BRM for this column at EOJ.
 //------------------------------------------------------------------------------
@@ -1267,7 +1267,7 @@ void ColumnInfo::getCPInfoForBRM( BRMReporter& brmReporter )
 {
     fColExtInf->getCPInfoForBRM(column, brmReporter);
 }
-
+
 //------------------------------------------------------------------------------
 // Get updated HWM information for BRM for this column at EOJ.
 // Returns count of the number of HWM entries added to the BRMReporter.
@@ -1340,7 +1340,7 @@ int ColumnInfo::getHWMInfoForBRM( BRMReporter& brmReporter )
 
     return entriesAdded;
 }
-
+
 //------------------------------------------------------------------------------
 // Setup initial extent we will begin loading at start of import.
 // DBRoot, partition, segment, etc for the starting extent are specified.
@@ -1460,7 +1460,7 @@ int ColumnInfo::setupInitialColumnExtent(
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // Prepare the initial column segment file for import.
 //------------------------------------------------------------------------------
@@ -1521,7 +1521,7 @@ int ColumnInfo::setupInitialColumnFile( HWM oldHwm, HWM hwm )
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // Update the number of bytes in the file, and the free space still remaining.
 //------------------------------------------------------------------------------
@@ -1542,7 +1542,7 @@ bool ColumnInfo::isFileComplete() const
 
     return false;
 }
-
+
 //------------------------------------------------------------------------------
 // Initialize last used auto-increment value from the current "next"
 // auto-increment value taken from the system catalog (or BRM).
@@ -1575,7 +1575,7 @@ int ColumnInfo::finishAutoInc( )
 
     return rc;
 }
-
+
 //------------------------------------------------------------------------------
 // Get current dbroot, partition, segment, and HWM for this column.
 //
@@ -1597,7 +1597,7 @@ void ColumnInfo::getSegFileInfo( DBRootExtentInfo& fileInfo )
     else
         fileInfo.fLocalHwm = 0;
 }
-
+
 //------------------------------------------------------------------------------
 // Open a new or existing Dictionary store file based on the DBRoot,
 // partition, and segment settings in curCol.dataFile.
@@ -1741,7 +1741,7 @@ int ColumnInfo::openDctnryStore( bool bMustExist )
 
     return rc;
 }
-
+
 //------------------------------------------------------------------------------
 // Close the current Dictionary store file.
 //------------------------------------------------------------------------------
@@ -1773,7 +1773,7 @@ int ColumnInfo::closeDctnryStore(bool bAbort)
 
     return rc;
 }
-
+
 //------------------------------------------------------------------------------
 // Update dictionary store file with specified strings, and return the assigned
 // tokens (tokenbuf) to be stored in the corresponding column token file.
@@ -1835,7 +1835,7 @@ int ColumnInfo::updateDctnryStore(char* buf,
 
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // No action necessary for uncompressed dictionary files
 //------------------------------------------------------------------------------
@@ -1856,7 +1856,7 @@ int ColumnInfo::truncateDctnryStore(
 {
     return NO_ERROR;
 }
-
+
 //------------------------------------------------------------------------------
 // utility to convert a Status enumeration to a string
 //------------------------------------------------------------------------------

writeengine/shared/we_fileop.cpp

@@ -593,8 +593,7 @@ int FileOp::extendFile(
             return ERR_FILE_NOT_EXIST;
         }
 
-        pFile = openFile( oid, dbRoot, partition, segment,
+        pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" );//old file
-            segFile, "r+b" );//old file
         if (pFile == 0)
         {
             ostringstream oss;
@@ -613,7 +612,7 @@ int FileOp::extendFile(
         if ( isDebug(DEBUG_1) && getLogger() )
         {
             std::ostringstream oss;
-            oss << "Opening existing column file"  <<
+            oss << "Opening existing column file (extendFile)"  <<
                    ": OID-"    << oid       <<
                    "; DBRoot-" << dbRoot    <<
                    "; part-"   << partition <<
@@ -651,19 +650,55 @@ int FileOp::extendFile(
                 ostringstream oss;
                 oss << "oid: " << oid << " with path " << segFile <<
                     "; new extent fbo " << hwm << "; number of "
-                    "compressed chunks " << ptrCount;
+                    "compressed chunks " << ptrCount <<
+					"; chunkIndex " << chunkIndex;
                 logging::Message::Args args;
                 args.add("compressed");
                 args.add(oss.str());
                 SimpleSysLog::instance()->logMsg(args,
                     logging::LOG_TYPE_ERROR,
                     logging::M0103);
-                return ERR_FILE_NEW_EXTENT_FBO;
+                // Expand the partial extent to full with emptyVal
+                // Since fillCompColumnExtentEmptyChunks() messes with the
+                // file on disk, we need to close it and reopen after or
+                // the cache isn't updated.
+				if ((pFile))
+					closeFile( pFile );
+				pFile = NULL;
+				string failedTask;  // For return error message, if any.
+				rc = FileOp::fillCompColumnExtentEmptyChunks(oid,
+															 width,
+															 emptyVal,
+															 dbRoot,
+															 partition,
+															 segment,
+															 hwm,
+															 segFile,
+															 failedTask);
+                if (rc != NO_ERROR)
+                {
+                    if (getLogger())
+                    {
+                        std::ostringstream oss;
+                        oss << "FileOp::extendFile: error padding partial compressed extent for " <<
+                               "column OID-" << oid       <<
+                               "; DBRoot-"   << dbRoot    <<
+                               "; part-"     << partition <<
+                               "; seg-"      << segment   <<
+                               "; hwm-"      << hwm       <<
+							   " " << failedTask.c_str();
+                        getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL );
+                    }
+                    return rc;
+                }
+				pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" ); // modified file
             }
 
+            // Get the latest file header for the caller. If a partial extent was filled out,
+            // this will be different than when we first read the headers.
             if (hdrs)
             {
-                memcpy(hdrs, hdrsIn, sizeof(hdrsIn) );
+				RETURN_ON_ERROR( readHeaders(pFile, hdrs) );
             }
         }
         else
@@ -693,7 +728,26 @@ int FileOp::extendFile(
                 SimpleSysLog::instance()->logMsg(args,
                     logging::LOG_TYPE_ERROR,
                     logging::M0103);
-                return ERR_FILE_NEW_EXTENT_FBO;
+                // Expand the partial extent to full with emptyVal
+                // This generally won't ever happen, as uncompressed files
+                // are created with full extents.
+                rc = FileOp::expandAbbrevColumnExtent( pFile, dbRoot,
+                    emptyVal, width);
+                if (rc != NO_ERROR)
+                {
+                    if (getLogger())
+                    {
+                        std::ostringstream oss;
+                        oss << "FileOp::extendFile: error padding partial uncompressed extent for " <<
+                               "column OID-" << oid       <<
+                               "; DBRoot-"   << dbRoot    <<
+                               "; part-"     << partition <<
+                               "; seg-"      << segment   <<
+                               "; hwm-"      << hwm;
+                        getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL );
+                    }
+                    return rc;
+                }
             }
         }
     }
@@ -2598,7 +2652,7 @@ int FileOp::expandAbbrevColumnExtent(
     }
 
     // Add blocks to turn the abbreviated extent into a full extent.
-    int rc = initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width,
+    int rc = FileOp::initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width,
         false,   // existing file
         true,    // expand existing extent
         false);  // n/a since not adding new extent

writeengine/shared/we_log.h

@@ -47,7 +47,7 @@ namespace WriteEngine
 {
 const	std::string MSG_LEVEL_STR[] = { 
 	"INFO",
-	"INFO",
+	"INFO2",
 	"WARN",
 	"ERR ",
 	"CRIT" };

writeengine/splitter/we_sdhandler.cpp

@@ -992,7 +992,7 @@ void WESDHandler::onNakResponse(int PmId) {
 // Increase the read timeout for this PM thread so that it won't
 // consume too much CPU
 void WESDHandler::onEodResponse(int PmId) {
-	if (getDebugLvl())
+	if (getDebugLvl()>2)
 		cout << "Received a EOD from " << PmId << endl;
 
 	if (fRef.fCmdArgs.getMode() == 0)

writeengine/wrapper/we_colopcompress.cpp

@@ -188,7 +188,13 @@ int ColumnOpCompress1::expandAbbrevColumnExtent(
    IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width)
 {
    // update the uncompressed initial chunk to full chunk
-   RETURN_ON_ERROR(m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width));
+   int rc = m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width);
+   // ERR_COMP_FILE_NOT_FOUND is acceptable here. It just means that the
+   // file hasn't been loaded into the chunk manager yet. No big deal.
+   if (rc != NO_ERROR && rc != ERR_COMP_FILE_NOT_FOUND)
+   {
+	   return rc;
+   }
 
    // let the base to physically expand extent.
    return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width);