database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Merge branch 'develop-1.1' into 1.1-merge-up

Browse Source
This commit is contained in:
Andrew Hutchings
2017-12-13 09:09:39 +00:00
parent 59d0a45da3 a89438c5fe
commit 17f077012d
11 changed files with 147 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dbcon/mysql/columnstore_info.sql
View File

@ -82,7 +82,7 @@ DROP PROCEDURE IF EXISTS `compression_ratio` //
CREATE PROCEDURE compression_ratio() CREATE PROCEDURE compression_ratio()
BEGIN BEGIN
SELECT CONCAT((SELECT SUM(file_size) FROM information_schema.columnstore_files WHERE compressed_data_size IS NOT NULL) / (SELECT SUM(data_size) FROM information_schema.columnstore_extents) * 100, '%') COMPRESSION_RATIO; SELECT CONCAT((SELECT SUM(data_size) FROM information_schema.columnstore_extents ce left join information_schema.columnstore_columns cc on ce.object_id = cc.object_id where compression_type='Snappy') / (SELECT SUM(compressed_data_size) FROM information_schema.columnstore_files WHERE compressed_data_size IS NOT NULL), ':1') COMPRESSION_RATIO;
END // END //
DELIMITER ; DELIMITER ;

2
oam/install_scripts/myCnf-include-args.text
View File

@ -1,7 +1,7 @@
# List of my.cnf arguments that should be checked and saved during upgrade install # List of my.cnf arguments that should be checked and saved during upgrade install
# #
infinidb_local_query infinidb_local_query
log-bin=mysql-bin log-bin
server-id server-id
max_length_for_sort_data max_length_for_sort_data
tmpdir tmpdir

16
oamapps/postConfigure/mycnfUpgrade.cpp
View File

@ -47,7 +47,7 @@
#include <climits> #include <climits>
#include <cstring> #include <cstring>
#include <glob.h> #include <glob.h>
#include <boost/regex.hpp>
#include "liboamcpp.h" #include "liboamcpp.h"
#include "installdir.h" #include "installdir.h"
@ -118,6 +118,7 @@ int main(int argc, char* argv[])
{ {
includeArg = line; includeArg = line;
boost::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*=");
//see if in my.cnf.rpmsave //see if in my.cnf.rpmsave
ifstream mycnfsavefile (mycnfsaveFile.c_str()); ifstream mycnfsavefile (mycnfsaveFile.c_str());
char line[200]; char line[200];
@ -126,9 +127,8 @@ int main(int argc, char* argv[])
while (mycnfsavefile.getline(line, 200)) while (mycnfsavefile.getline(line, 200))
{ {
oldbuf = line; oldbuf = line;
string::size_type pos = oldbuf.find(includeArg, 0);
if ( pos != string::npos ) if ( boost::regex_search(oldbuf.begin(), oldbuf.end(), icludeArgRegEx) )
{ {
//found in my.cnf.rpmsave, check if this is commented out //found in my.cnf.rpmsave, check if this is commented out
if ( line[0] != '#' ) if ( line[0] != '#' )
@ -144,9 +144,8 @@ int main(int argc, char* argv[])
while (mycnffile.getline(line1, 200)) while (mycnffile.getline(line1, 200))
{ {
newbuf = line1; newbuf = line1;
string::size_type pos = newbuf.find(includeArg, 0);
if ( pos != string::npos ) if ( boost::regex_search(newbuf.begin(), newbuf.end(), icludeArgRegEx) )
{ {
newbuf = oldbuf; newbuf = oldbuf;
cout << "Updated argument: " << includeArg << endl; cout << "Updated argument: " << includeArg << endl;
@ -181,9 +180,9 @@ int main(int argc, char* argv[])
while (mycnffile.getline(line1, 200)) while (mycnffile.getline(line1, 200))
{ {
newbuf = line1; newbuf = line1;
string::size_type pos = newbuf.find("[mysqld]", 0); boost::regex mysqldSectionRegEx("\\[mysqld\\]");
if ( pos != string::npos ) if ( boost::regex_search(newbuf.begin(), newbuf.end(), mysqldSectionRegEx) )
{ {
lines.push_back(newbuf); lines.push_back(newbuf);
newbuf = oldbuf; newbuf = oldbuf;
@ -206,10 +205,9 @@ int main(int argc, char* argv[])
newFile.close(); newFile.close();
close(fd); close(fd);
break;
} }
} }
break;
} }
} }
} }

2
utils/clusterTester/columnstoreClusterTester.sh
View File

@ -978,7 +978,7 @@ checkPackages()
fi fi
fi fi
declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libioa1") declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1")
if [ "$OS" == "debian9" ]; then if [ "$OS" == "debian9" ]; then
if [ ! `which dpkg 2>/dev/null` ] ; then if [ ! `which dpkg 2>/dev/null` ] ; then

32
utils/configcpp/xmlparser.cpp
View File

@ -22,6 +22,7 @@
#include "config.h" #include "config.h"
#include <string> #include <string>
#include <boost/algorithm/string.hpp>
#include <stdexcept> #include <stdexcept>
#include <libxml/xmlmemory.h> #include <libxml/xmlmemory.h>
#include <libxml/parser.h> #include <libxml/parser.h>
@ -45,13 +46,17 @@ const string XMLParser::getConfig(const xmlDocPtr doc, const string& section, co
while (cur1 != NULL) while (cur1 != NULL)
{ {
if ((!xmlStrcmp(cur1->name, (const xmlChar*)section.c_str()))) string cur1name = (const char*)cur1->name;
if ((boost::iequals(cur1name, section)))
{ {
xmlNodePtr cur2 = cur1->xmlChildrenNode; xmlNodePtr cur2 = cur1->xmlChildrenNode;
while (cur2 != NULL) while (cur2 != NULL)
{ {
if ((!xmlStrcmp(cur2->name, (const xmlChar*)name.c_str()))) string cur2name = (const char*)cur2->name;
if ((boost::iequals(cur2name, name)))
{ {
xmlNodePtr cur3 = cur2->xmlChildrenNode; xmlNodePtr cur3 = cur2->xmlChildrenNode;
@ -85,13 +90,17 @@ void XMLParser::getConfig(const xmlDocPtr doc, const string& section, const stri
while (cur1 != NULL) while (cur1 != NULL)
{ {
if ((!xmlStrcmp(cur1->name, (const xmlChar*)section.c_str()))) string cur1name = (const char*)cur1->name;
if ((boost::iequals(cur1name, section)))
{ {
xmlNodePtr cur2 = cur1->xmlChildrenNode; xmlNodePtr cur2 = cur1->xmlChildrenNode;
while (cur2 != NULL) while (cur2 != NULL)
{ {
if ((!xmlStrcmp(cur2->name, (const xmlChar*)name.c_str()))) string cur2name = (const char*)cur2->name;
if ((boost::iequals(cur2name, name)))
{ {
res.clear(); res.clear();
xmlNodePtr cur3 = cur2->xmlChildrenNode; xmlNodePtr cur3 = cur2->xmlChildrenNode;
@ -123,13 +132,17 @@ void XMLParser::setConfig(xmlDocPtr doc, const string& section, const string& na
while (cur1 != NULL) while (cur1 != NULL)
{ {
if (xmlStrcmp(cur1->name, (const xmlChar*)section.c_str()) == 0) string cur1name = (const char*)cur1->name;
if (boost::iequals(cur1name, section))
{ {
cur2 = cur1->xmlChildrenNode; cur2 = cur1->xmlChildrenNode;
while (cur2 != NULL) while (cur2 != NULL)
{ {
if (xmlStrcmp(cur2->name, (const xmlChar*)name.c_str()) == 0) string cur2name = (const char*)cur2->name;
if (boost::iequals(cur2name, name))
{ {
xmlNodePtr cur3 = cur2->xmlChildrenNode; xmlNodePtr cur3 = cur2->xmlChildrenNode;
@ -185,7 +198,9 @@ void XMLParser::delConfig(xmlDocPtr doc, const string& section, const string& na
while (cur1 != NULL) while (cur1 != NULL)
{ {
if ((!xmlStrcmp(cur1->name, (const xmlChar*)section.c_str()))) string cur1name = (const char*)cur1->name;
if ((boost::iequals(cur1name, section)))
{ {
xmlNodePtr cur2 = cur1->xmlChildrenNode; xmlNodePtr cur2 = cur1->xmlChildrenNode;
@ -193,8 +208,9 @@ void XMLParser::delConfig(xmlDocPtr doc, const string& section, const string& na
{ {
xmlNodePtr tmp = cur2; xmlNodePtr tmp = cur2;
cur2 = cur2->next; cur2 = cur2->next;
string tmpname = (const char*)tmp->name;
if ((!xmlStrcmp(tmp->name, (const xmlChar*)name.c_str()))) if ((boost::iequals(tmpname, name)))
{ {
xmlUnlinkNode(tmp); xmlUnlinkNode(tmp);
xmlFreeNode(tmp); xmlFreeNode(tmp);

4
versioning/BRM/extentmap.h
View File

@ -737,8 +737,8 @@ public:
* *
* Function should only be called for System Catalog OIDs, as it assumes * Function should only be called for System Catalog OIDs, as it assumes
* the OID is fully contained on a single DBRoot, returning the first * the OID is fully contained on a single DBRoot, returning the first
* DBRoot found. This only makes since for a System Catalog OID, because * DBRoot found. This only makes sence for a System Catalog
* all other column OIDs can span multiple DBRoots. * OID, because all other column OIDs can span multiple DBRoots.
* *
* @param oid The system catalog OID * @param oid The system catalog OID
* @param dbRoot (out) the DBRoot holding the system catalog OID * @param dbRoot (out) the DBRoot holding the system catalog OID

58
writeengine/bulk/we_columninfo.cpp
View File

@ -123,7 +123,7 @@ unsigned int compactVarBinary(char* charTmpBuf, int fieldLength)
} }
} }
namespace WriteEngine namespace WriteEngine
{ {
@ -222,7 +222,7 @@ ColumnInfo::ColumnInfo(Log* logger,
// fAutoIncMgr = new ColumnAutoIncJob(logger); // fAutoIncMgr = new ColumnAutoIncJob(logger);
} }
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// ColumnInfo destructor // ColumnInfo destructor
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -263,7 +263,7 @@ void ColumnInfo::clearMemory( )
fDictBlocks.clear(); fDictBlocks.clear();
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// If at the start of the job, We have encountered a PM that has no DB file for // If at the start of the job, We have encountered a PM that has no DB file for
// this column, or whose HWM extent is disabled; then this function is called // this column, or whose HWM extent is disabled; then this function is called
@ -295,7 +295,7 @@ void ColumnInfo::setupDelayedFileCreation(
column.compressionType, column.compressionType,
dbRoot, partition, segment ); dbRoot, partition, segment );
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Create a DB file as part of delayed file creation. See setupDelayedFile- // Create a DB file as part of delayed file creation. See setupDelayedFile-
// Creation for an explanation. // Creation for an explanation.
@ -529,7 +529,7 @@ int ColumnInfo::createDelayedFileIfNeeded( const std::string& tableName )
return rc; return rc;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Add an extent for this column. The next segment file in the DBRoot, // Add an extent for this column. The next segment file in the DBRoot,
// partition, segment number rotation will be selected for the extent. // partition, segment number rotation will be selected for the extent.
@ -641,7 +641,7 @@ int ColumnInfo::extendColumn( bool saveLBIDForCP )
return rc; return rc;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Add a new extent to this column, at the specified DBRoot. Partition may be // Add a new extent to this column, at the specified DBRoot. Partition may be
// used if DBRoot is empty. // used if DBRoot is empty.
@ -804,7 +804,7 @@ int ColumnInfo::extendColumnNewExtent(
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Fill out existing partial extent to extent boundary, so that we can resume // Fill out existing partial extent to extent boundary, so that we can resume
// inserting rows on an extent boundary basis. This use case should only take // inserting rows on an extent boundary basis. This use case should only take
@ -916,7 +916,7 @@ int ColumnInfo::extendColumnOldExtent(
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Either add or update the File object, so that it has the updated HWM. // Either add or update the File object, so that it has the updated HWM.
// We will access this info to update the HWM in the ExtentMap at the end // We will access this info to update the HWM in the ExtentMap at the end
@ -970,7 +970,7 @@ void ColumnInfo::addToSegFileList( File& dataFile, HWM hwm )
fSegFileUpdateList.push_back( dataFile ); fSegFileUpdateList.push_back( dataFile );
} }
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Reset file offset data member attributes when we start working on the next // Reset file offset data member attributes when we start working on the next
// extent. // extent.
@ -998,7 +998,7 @@ int ColumnInfo::resetFileOffsetsNewExtent(const char* /*hdr*/)
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Set current size of file in raw (uncompressed) bytes, given the specified // Set current size of file in raw (uncompressed) bytes, given the specified
// hwm. abbrevFlag indicates whether this is a fixed size abbreviated extent. // hwm. abbrevFlag indicates whether this is a fixed size abbreviated extent.
@ -1032,7 +1032,7 @@ void ColumnInfo::setFileSize( HWM hwm, int abbrevFlag )
} }
} }
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// If we are dealing with the first extent in the first segment file for this // If we are dealing with the first extent in the first segment file for this
// column, and the segment file is still equal to 256K rows, then we set the // column, and the segment file is still equal to 256K rows, then we set the
@ -1064,7 +1064,7 @@ void ColumnInfo::setAbbrevExtentCheck( )
} }
} }
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// If this is an abbreviated extent, we expand the extent to a full extent on // If this is an abbreviated extent, we expand the extent to a full extent on
// disk, by initializing the necessary number of remaining blocks. // disk, by initializing the necessary number of remaining blocks.
@ -1144,7 +1144,7 @@ int ColumnInfo::expandAbbrevExtent( bool bRetainFilePos )
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Close the current Column file. // Close the current Column file.
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -1158,7 +1158,7 @@ int ColumnInfo::closeColumnFile(bool /*bCompletingExtent*/, bool /*bAbort*/)
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Initialize fLastInputRowInCurrentExtent used in detecting when a Read Buffer // Initialize fLastInputRowInCurrentExtent used in detecting when a Read Buffer
// is crossing an extent boundary, so that we can accurately track the min/max // is crossing an extent boundary, so that we can accurately track the min/max
@ -1207,7 +1207,7 @@ void ColumnInfo::lastInputRowInExtentInit( bool bIsNewExtent )
bIsNewExtent ); bIsNewExtent );
} }
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Increment fLastRIDInExtent to the end of the next extent. // Increment fLastRIDInExtent to the end of the next extent.
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -1215,7 +1215,7 @@ void ColumnInfo::lastInputRowInExtentInc( )
{ {
fLastInputRowInCurrentExtent += fRowsPerExtent; fLastInputRowInCurrentExtent += fRowsPerExtent;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Parsing is complete for this column. Flush pending data. Close the current // Parsing is complete for this column. Flush pending data. Close the current
// segment file, and corresponding dictionary store file (if applicable). Also // segment file, and corresponding dictionary store file (if applicable). Also
@ -1281,7 +1281,7 @@ int ColumnInfo::finishParsing( )
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Store updated column information in BRMReporter for this column at EOJ; // Store updated column information in BRMReporter for this column at EOJ;
// so that Extent Map CP information and HWM's can be updated. // so that Extent Map CP information and HWM's can be updated.
@ -1306,7 +1306,7 @@ void ColumnInfo::getBRMUpdateInfo( BRMReporter& brmReporter )
if (entriesAdded > 0) if (entriesAdded > 0)
getCPInfoForBRM( brmReporter ); getCPInfoForBRM( brmReporter );
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Get updated Casual Partition (CP) information for BRM for this column at EOJ. // Get updated Casual Partition (CP) information for BRM for this column at EOJ.
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -1314,7 +1314,7 @@ void ColumnInfo::getCPInfoForBRM( BRMReporter& brmReporter )
{ {
fColExtInf->getCPInfoForBRM(column, brmReporter); fColExtInf->getCPInfoForBRM(column, brmReporter);
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Get updated HWM information for BRM for this column at EOJ. // Get updated HWM information for BRM for this column at EOJ.
// Returns count of the number of HWM entries added to the BRMReporter. // Returns count of the number of HWM entries added to the BRMReporter.
@ -1388,7 +1388,7 @@ int ColumnInfo::getHWMInfoForBRM( BRMReporter& brmReporter )
return entriesAdded; return entriesAdded;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Setup initial extent we will begin loading at start of import. // Setup initial extent we will begin loading at start of import.
// DBRoot, partition, segment, etc for the starting extent are specified. // DBRoot, partition, segment, etc for the starting extent are specified.
@ -1516,7 +1516,7 @@ int ColumnInfo::setupInitialColumnExtent(
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Prepare the initial column segment file for import. // Prepare the initial column segment file for import.
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -1579,7 +1579,7 @@ int ColumnInfo::setupInitialColumnFile( HWM oldHwm, HWM hwm )
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Update the number of bytes in the file, and the free space still remaining. // Update the number of bytes in the file, and the free space still remaining.
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -1600,7 +1600,7 @@ bool ColumnInfo::isFileComplete() const
return false; return false;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Initialize last used auto-increment value from the current "next" // Initialize last used auto-increment value from the current "next"
// auto-increment value taken from the system catalog (or BRM). // auto-increment value taken from the system catalog (or BRM).
@ -1633,7 +1633,7 @@ int ColumnInfo::finishAutoInc( )
return rc; return rc;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Get current dbroot, partition, segment, and HWM for this column. // Get current dbroot, partition, segment, and HWM for this column.
// //
@ -1656,7 +1656,7 @@ void ColumnInfo::getSegFileInfo( DBRootExtentInfo& fileInfo )
else else
fileInfo.fLocalHwm = 0; fileInfo.fLocalHwm = 0;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Open a new or existing Dictionary store file based on the DBRoot, // Open a new or existing Dictionary store file based on the DBRoot,
// partition, and segment settings in curCol.dataFile. // partition, and segment settings in curCol.dataFile.
@ -1805,7 +1805,7 @@ int ColumnInfo::openDctnryStore( bool bMustExist )
return rc; return rc;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Close the current Dictionary store file. // Close the current Dictionary store file.
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -1837,7 +1837,7 @@ int ColumnInfo::closeDctnryStore(bool bAbort)
return rc; return rc;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Update dictionary store file with specified strings, and return the assigned // Update dictionary store file with specified strings, and return the assigned
// tokens (tokenbuf) to be stored in the corresponding column token file. // tokens (tokenbuf) to be stored in the corresponding column token file.
@ -1903,7 +1903,7 @@ int ColumnInfo::updateDctnryStore(char* buf,
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// No action necessary for uncompressed dictionary files // No action necessary for uncompressed dictionary files
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -1924,7 +1924,7 @@ const
{ {
return NO_ERROR; return NO_ERROR;
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// utility to convert a Status enumeration to a string // utility to convert a Status enumeration to a string
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------

83
writeengine/shared/we_fileop.cpp
View File

@ -605,8 +605,7 @@ int FileOp::extendFile(
return ERR_FILE_NOT_EXIST; return ERR_FILE_NOT_EXIST;
} }
pFile = openFile( oid, dbRoot, partition, segment, pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" );//old file
segFile, "r+b" );//old file
if (pFile == 0) if (pFile == 0)
{ {
@ -626,7 +625,7 @@ int FileOp::extendFile(
if ( isDebug(DEBUG_1) && getLogger() ) if ( isDebug(DEBUG_1) && getLogger() )
{ {
std::ostringstream oss; std::ostringstream oss;
oss << "Opening existing column file" << oss << "Opening existing column file (extendFile)" <<
": OID-" << oid << ": OID-" << oid <<
"; DBRoot-" << dbRoot << "; DBRoot-" << dbRoot <<
"; part-" << partition << "; part-" << partition <<
@ -664,19 +663,60 @@ int FileOp::extendFile(
ostringstream oss; ostringstream oss;
oss << "oid: " << oid << " with path " << segFile << oss << "oid: " << oid << " with path " << segFile <<
"; new extent fbo " << hwm << "; number of " "; new extent fbo " << hwm << "; number of "
"compressed chunks " << ptrCount; "compressed chunks " << ptrCount <<
"; chunkIndex " << chunkIndex;
logging::Message::Args args; logging::Message::Args args;
args.add("compressed"); args.add("compressed");
args.add(oss.str()); args.add(oss.str());
SimpleSysLog::instance()->logMsg(args, SimpleSysLog::instance()->logMsg(args,
logging::LOG_TYPE_ERROR, logging::LOG_TYPE_ERROR,
logging::M0103); logging::M0103);
return ERR_FILE_NEW_EXTENT_FBO;
// Expand the partial extent to full with emptyVal
// Since fillCompColumnExtentEmptyChunks() messes with the
// file on disk, we need to close it and reopen after or
// the cache isn't updated.
if ((pFile))
closeFile( pFile );
pFile = NULL;
string failedTask; // For return error message, if any.
rc = FileOp::fillCompColumnExtentEmptyChunks(oid,
width,
emptyVal,
dbRoot,
partition,
segment,
hwm,
segFile,
failedTask);
if (rc != NO_ERROR)
{
if (getLogger())
{
std::ostringstream oss;
oss << "FileOp::extendFile: error padding partial compressed extent for " <<
"column OID-" << oid <<
"; DBRoot-" << dbRoot <<
"; part-" << partition <<
"; seg-" << segment <<
"; hwm-" << hwm <<
" " << failedTask.c_str();
getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL );
}
return rc;
}
pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" ); // modified file
} }
// Get the latest file header for the caller. If a partial extent was filled out,
// this will be different than when we first read the headers.
if (hdrs) if (hdrs)
{ {
memcpy(hdrs, hdrsIn, sizeof(hdrsIn) ); RETURN_ON_ERROR( readHeaders(pFile, hdrs) );
} }
} }
else else
@ -706,7 +746,28 @@ int FileOp::extendFile(
SimpleSysLog::instance()->logMsg(args, SimpleSysLog::instance()->logMsg(args,
logging::LOG_TYPE_ERROR, logging::LOG_TYPE_ERROR,
logging::M0103); logging::M0103);
return ERR_FILE_NEW_EXTENT_FBO; // Expand the partial extent to full with emptyVal
// This generally won't ever happen, as uncompressed files
// are created with full extents.
rc = FileOp::expandAbbrevColumnExtent( pFile, dbRoot,
emptyVal, width);
if (rc != NO_ERROR)
{
if (getLogger())
{
std::ostringstream oss;
oss << "FileOp::extendFile: error padding partial uncompressed extent for " <<
"column OID-" << oid <<
"; DBRoot-" << dbRoot <<
"; part-" << partition <<
"; seg-" << segment <<
"; hwm-" << hwm;
getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL );
}
return rc;
}
} }
} }
} }
@ -2726,10 +2787,10 @@ int FileOp::expandAbbrevColumnExtent(
} }
// Add blocks to turn the abbreviated extent into a full extent. // Add blocks to turn the abbreviated extent into a full extent.
int rc = initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width, int rc = FileOp::initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width,
false, // existing file false, // existing file
true, // expand existing extent true, // expand existing extent
false); // n/a since not adding new extent false); // n/a since not adding new extent
return rc; return rc;
} }

2
writeengine/shared/we_log.h
View File

@ -48,7 +48,7 @@ namespace WriteEngine
const std::string MSG_LEVEL_STR[] = const std::string MSG_LEVEL_STR[] =
{ {
"INFO", "INFO",
"INFO", "INFO2",
"WARN", "WARN",
"ERR ", "ERR ",
"CRIT" "CRIT"

2
writeengine/splitter/we_sdhandler.cpp
View File

@ -1110,7 +1110,7 @@ void WESDHandler::onNakResponse(int PmId)
// consume too much CPU // consume too much CPU
void WESDHandler::onEodResponse(int PmId) void WESDHandler::onEodResponse(int PmId)
{ {
if (getDebugLvl()) if (getDebugLvl() > 2)
cout << "Received a EOD from " << PmId << endl; cout << "Received a EOD from " << PmId << endl;
if (fRef.fCmdArgs.getMode() == 0) if (fRef.fCmdArgs.getMode() == 0)

9
writeengine/wrapper/we_colopcompress.cpp
View File

@ -194,7 +194,14 @@ int ColumnOpCompress1::expandAbbrevColumnExtent(
IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width) IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width)
{ {
// update the uncompressed initial chunk to full chunk // update the uncompressed initial chunk to full chunk
RETURN_ON_ERROR(m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width)); int rc = m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width);
// ERR_COMP_FILE_NOT_FOUND is acceptable here. It just means that the
// file hasn't been loaded into the chunk manager yet. No big deal.
if (rc != NO_ERROR && rc != ERR_COMP_FILE_NOT_FOUND)
{
return rc;
}
// let the base to physically expand extent. // let the base to physically expand extent.
return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width); return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width);