database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-498 Add DBRootX.PreallocSpace setting in the XML. Dict files extents now contain a correct number of blocks available.

Browse Source
This commit is contained in:
Roman Nozdrin
2018-12-20 06:52:05 +03:00
parent 29becc2971
commit cbdcdb9f10
8 changed files with 131 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
oam/etc/Columnstore.xml
View File

@ -102,162 +102,130 @@
<PMS1> <PMS1>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS1> </PMS1>
<PMS2> <PMS2>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS2> </PMS2>
<PMS3> <PMS3>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS3> </PMS3>
<PMS4> <PMS4>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS4> </PMS4>
<PMS5> <PMS5>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS5> </PMS5>
<PMS6> <PMS6>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS6> </PMS6>
<PMS7> <PMS7>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS7> </PMS7>
<PMS8> <PMS8>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS8> </PMS8>
<PMS9> <PMS9>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS9> </PMS9>
<PMS10> <PMS10>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS10> </PMS10>
<PMS11> <PMS11>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS11> </PMS11>
<PMS12> <PMS12>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS12> </PMS12>
<PMS13> <PMS13>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS13> </PMS13>
<PMS14> <PMS14>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS14> </PMS14>
<PMS15> <PMS15>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS15> </PMS15>
<PMS16> <PMS16>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS16> </PMS16>
<PMS17> <PMS17>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS17> </PMS17>
<PMS18> <PMS18>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS18> </PMS18>
<PMS19> <PMS19>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS19> </PMS19>
<PMS20> <PMS20>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS20> </PMS20>
<PMS21> <PMS21>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS21> </PMS21>
<PMS22> <PMS22>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS22> </PMS22>
<PMS23> <PMS23>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS23> </PMS23>
<PMS24> <PMS24>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS24> </PMS24>
<PMS25> <PMS25>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS25> </PMS25>
<PMS26> <PMS26>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS26> </PMS26>
<PMS27> <PMS27>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS27> </PMS27>
<PMS28> <PMS28>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS28> </PMS28>
<PMS29> <PMS29>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS29> </PMS29>
<PMS30> <PMS30>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS30> </PMS30>
<PMS31> <PMS31>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS31> </PMS31>
<PMS32> <PMS32>
<IPAddr>0.0.0.0</IPAddr> <IPAddr>0.0.0.0</IPAddr>
<Port>8620</Port> <Port>8620</Port>
<PreallocSpace>ON</PreallocSpace>
</PMS32> </PMS32>
<SystemConfig> <SystemConfig>
<SystemLang>C</SystemLang> <SystemLang>C</SystemLang>

94
utils/idbdatafile/IDBPolicy.cpp
View File

@ -18,14 +18,13 @@
#include <unistd.h> #include <unistd.h>
#include <iostream> #include <iostream>
#include <sstream> #include <sstream>
#include <string>
#include <boost/filesystem/path.hpp> #include <boost/filesystem/path.hpp>
#include <boost/filesystem/convenience.hpp> #include <boost/filesystem/convenience.hpp>
#include <boost/algorithm/string.hpp> // to_upper
#include <boost/thread/thread.hpp> #include <boost/thread/thread.hpp>
#include "configcpp.h" // for Config #include "configcpp.h" // for Config
#include "oamcache.h" #include "oamcache.h"
#include "liboamcpp.h"
#include "IDBPolicy.h" #include "IDBPolicy.h"
#include "PosixFileSystem.h" #include "PosixFileSystem.h"
//#include "HdfsFileSystem.h" //#include "HdfsFileSystem.h"
@ -50,7 +49,7 @@ int64_t IDBPolicy::s_hdfsRdwrBufferMaxSize = 0;
std::string IDBPolicy::s_hdfsRdwrScratch; std::string IDBPolicy::s_hdfsRdwrScratch;
bool IDBPolicy::s_configed = false; bool IDBPolicy::s_configed = false;
boost::mutex IDBPolicy::s_mutex; boost::mutex IDBPolicy::s_mutex;
bool IDBPolicy::s_PreallocSpace = true; std::vector<uint16_t> IDBPolicy::s_PreallocSpace;
void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& hdfsRdwrScratch, int64_t hdfsRdwrBufferMaxSize ) void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& hdfsRdwrScratch, int64_t hdfsRdwrBufferMaxSize )
{ {
@ -184,10 +183,12 @@ void IDBPolicy::configIDBPolicy()
bool idblog = false; bool idblog = false;
string idblogstr = cf->getConfig("SystemConfig", "DataFileLog"); string idblogstr = cf->getConfig("SystemConfig", "DataFileLog");
if ( idblogstr.length() != 0 ) // Must be faster.
if ( idblogstr.size() == 2
&& ( idblogstr[0] == 'O' || idblogstr[0] == 'o' )
&& ( idblogstr[1] == 'N' || idblogstr[1] == 'n' ))
{ {
boost::to_upper(idblogstr); idblog = true;
idblog = ( idblogstr == "ON" );
} }
//-------------------------------------------------------------------------- //--------------------------------------------------------------------------
@ -227,26 +228,73 @@ void IDBPolicy::configIDBPolicy()
string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch");
string hdfsRdwrScratch = tmpDir + scratch; string hdfsRdwrScratch = tmpDir + scratch;
// MCOL-498. Set the PMSX.PreallocSpace knob, where X is a PM number, // MCOL-498. Use DBRootX.PreallocSpace to disable
// to disable file space preallocation. The feature is used in the FileOp code // dbroots file space preallocation.
// and is enabled by default for a backward compatibility. // The feature is used in the FileOp code and enabled by default.
oam::OamCache* oamcache = oam::OamCache::makeOamCache(); char configSectionPref[] = "DBRoot";
int PMId = oamcache->getLocalPMId(); int confSectionLen = sizeof(configSectionPref)+oam::MAX_MODULE_ID_SIZE;
char configSectionPref[] = "PMS"; char configSection[confSectionLen];
char configSection[sizeof(configSectionPref)+oam::MAX_MODULE_ID_SIZE];
::memset(configSection, 0, sizeof(configSection));
sprintf(configSection, "%s%d", configSectionPref, PMId);
string PreallocSpace = cf->getConfig(configSection, "PreallocSpace");
if ( PreallocSpace.length() != 0 )
{
boost::to_upper(PreallocSpace);
s_PreallocSpace = ( PreallocSpace != "OFF" );
}
IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize );
s_configed = true; s_configed = true;
oam::OamCache* oamcache = oam::OamCache::makeOamCache();
int PMId = oamcache->getLocalPMId();
oam::OamCache::PMDbrootsMap_t pmDbrootsMap;
pmDbrootsMap.reset(new oam::OamCache::PMDbrootsMap_t::element_type());
oam::systemStorageInfo_t t;
oam::Oam oamInst;
t = oamInst.getStorageConfig();
oam::DeviceDBRootList moduledbrootlist = boost::get<2>(t);
oam::DeviceDBRootList::iterator pt = moduledbrootlist.begin();
while ( pt != moduledbrootlist.end() && (*pt).DeviceID != PMId)
{
pt++;
continue;
}
// CS could return here b/c it initialised this singleton and
// there is no DBRootX sections in XML.
if (pt == moduledbrootlist.end())
{
return;
}
oam::DBRootConfigList &dbRootVec = (*pt).dbrootConfigList;
s_PreallocSpace.reserve(dbRootVec.size()>>1);
{
int rc;
oam::DBRootConfigList::iterator dbRootIter = dbRootVec.begin();
for(; dbRootIter != dbRootVec.end(); dbRootIter++)
{
::memset(configSection + sizeof(configSectionPref), 0, oam::MAX_MODULE_ID_SIZE);
rc = snprintf(configSection, confSectionLen, "%s%d", configSectionPref, *dbRootIter);
// gcc 8.2 warnings
if ( rc < 0 || rc >= confSectionLen)
{
ostringstream oss;
oss << "IDBPolicy::configIDBPolicy: failed to parse DBRootX section.";
throw runtime_error(oss.str());
}
string setting = cf->getConfig(configSection, "PreallocSpace");
if ( setting.length() != 0 )
{
if ( setting.size() == 3
&& ( setting[0] == 'O' || setting[0] == 'o' )
&& ( setting[1] == 'F' || setting[1] == 'f' )
&& ( setting[2] == 'F' || setting[2] == 'f' )
)
{
// int into uint16_t implicit conversion
s_PreallocSpace.push_back(*dbRootIter);
}
}
}
}
} }
} }

14
utils/idbdatafile/IDBPolicy.h
View File

@ -19,6 +19,7 @@
#define IDBPOLICY_H_ #define IDBPOLICY_H_
#include <string> #include <string>
#include <vector>
#include <stdint.h> #include <stdint.h>
#include <boost/thread/mutex.hpp> #include <boost/thread/mutex.hpp>
@ -81,9 +82,9 @@ public:
static bool useHdfs(); static bool useHdfs();
/** /**
* Accessor method that returns whether or not HDFS is enabled * Checks for disk space preallocation feature status for a dbroot
*/ */
static bool PreallocSpace(); static bool PreallocSpace(uint16_t dbRoot);
/** /**
* Accessor method that returns whether to use HDFS memory buffers * Accessor method that returns whether to use HDFS memory buffers
@ -139,7 +140,7 @@ private:
static bool isLocalFile( const std::string& path ); static bool isLocalFile( const std::string& path );
static bool s_usehdfs; static bool s_usehdfs;
static bool s_PreallocSpace; static std::vector<uint16_t> s_PreallocSpace;
static bool s_bUseRdwrMemBuffer; static bool s_bUseRdwrMemBuffer;
static std::string s_hdfsRdwrScratch; static std::string s_hdfsRdwrScratch;
static int64_t s_hdfsRdwrBufferMaxSize; static int64_t s_hdfsRdwrBufferMaxSize;
@ -159,10 +160,13 @@ bool IDBPolicy::useHdfs()
return s_usehdfs; return s_usehdfs;
} }
// MCOL-498 Looking for dbRoot in the List set in configIDBPolicy.
inline inline
bool IDBPolicy::PreallocSpace() bool IDBPolicy::PreallocSpace(uint16_t dbRoot)
{ {
return s_PreallocSpace; std::vector<uint16_t>::iterator dbRootIter =
find(s_PreallocSpace.begin(), s_PreallocSpace.end(), dbRoot);
return dbRootIter != s_PreallocSpace.end();
} }
inline inline

4
writeengine/dictionary/we_dctnry.cpp
View File

@ -259,12 +259,14 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth,
if ( m_dFile != NULL ) if ( m_dFile != NULL )
{ {
bool optimizePrealloc = ( flag ) ? false : true;
rc = FileOp::initDctnryExtent( m_dFile, rc = FileOp::initDctnryExtent( m_dFile,
m_dbRoot, m_dbRoot,
totalSize, totalSize,
m_dctnryHeader2, m_dctnryHeader2,
m_totalHdrBytes, m_totalHdrBytes,
false ); false,
optimizePrealloc );
if (rc != NO_ERROR) if (rc != NO_ERROR)
{ {

4
writeengine/server/we_dmlcommandproc.cpp
View File

@ -102,13 +102,13 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
bs >> tmp32; bs >> tmp32;
uint32_t dbroot = tmp32; uint32_t dbroot = tmp32;
cout << "processSingleInsert received bytestream length " << bs.length() << endl; //cout << "processSingleInsert received bytestream length " << bs.length() << endl;
messageqcpp::ByteStream::byte packageType; messageqcpp::ByteStream::byte packageType;
bs >> packageType; bs >> packageType;
insertPkg.read( bs); insertPkg.read( bs);
uint32_t sessionId = insertPkg.get_SessionID(); uint32_t sessionId = insertPkg.get_SessionID();
cout << " processSingleInsert for session " << sessionId << endl; //cout << " processSingleInsert for session " << sessionId << endl;
DMLTable* tablePtr = insertPkg.get_Table(); DMLTable* tablePtr = insertPkg.get_Table();
RowList rows = tablePtr->get_RowList(); RowList rows = tablePtr->get_RowList();

3
writeengine/shared/we_define.h
View File

@ -45,7 +45,8 @@ const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file
const int BYTE_PER_BLOCK = 8192; // Num bytes per data block const int BYTE_PER_BLOCK = 8192; // Num bytes per data block
const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block
const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block
const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; // Used for initial number of blocks calculation
const int MAX_INITIAL_EXTENT_BLOCKS_TO_DISK = 256; // Number of blocks in abbrev extent for 8byte col.
// Num rows reserved to disk for 'initial' extent // Num rows reserved to disk for 'initial' extent
const int FILE_NAME_SIZE = 200; // Max size of file name const int FILE_NAME_SIZE = 200; // Max size of file name
const long long MAX_ALLOW_ERROR_COUNT = 100000; //Max allowable error count const long long MAX_ALLOW_ERROR_COUNT = 100000; //Max allowable error count

65
writeengine/shared/we_fileop.cpp
View File

@ -1046,7 +1046,7 @@ int FileOp::initColumnExtent(
// @bug5769 Don't initialize extents or truncate db files on HDFS // @bug5769 Don't initialize extents or truncate db files on HDFS
// MCOL-498 We don't need sequential segment files if a PM uses SSD either. // MCOL-498 We don't need sequential segment files if a PM uses SSD either.
if (idbdatafile::IDBPolicy::useHdfs() || !idbdatafile::IDBPolicy::PreallocSpace()) if (idbdatafile::IDBPolicy::useHdfs())
{ {
//@Bug 3219. update the compression header after the extent is expanded. //@Bug 3219. update the compression header after the extent is expanded.
if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) if ((!bNewFile) && (m_compressionType) && (bExpandExtent))
@ -1101,10 +1101,19 @@ int FileOp::initColumnExtent(
else else
Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT);
#endif #endif
// MCOL-498 Skip the huge preallocations if the option is set
// for the dbroot
if ( bOptExtension )
{
bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot))
? bOptExtension : false;
}
int savedErrno = 0; int savedErrno = 0;
// MCOL-498 Try to preallocate the space, fallback to write if fallocate has failed // MCOL-498 fallocate the abbreviated extent,
if ( !bOptExtension || ( nBlocks < 300 && pFile->fallocate(0, currFileSize, writeSize) )) // fallback to sequential write if fallocate failed
if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK
&& pFile->fallocate(0, currFileSize, writeSize) )
)
{ {
savedErrno = errno; savedErrno = errno;
// Log the failed fallocate() call result // Log the failed fallocate() call result
@ -1817,7 +1826,7 @@ int FileOp::initDctnryExtent(
off64_t currFileSize = pFile->size(); off64_t currFileSize = pFile->size();
// @bug5769 Don't initialize extents or truncate db files on HDFS // @bug5769 Don't initialize extents or truncate db files on HDFS
// MCOL-498 We don't need sequential segment files if a PM uses SSD either. // MCOL-498 We don't need sequential segment files if a PM uses SSD either.
if (idbdatafile::IDBPolicy::useHdfs() || !idbdatafile::IDBPolicy::PreallocSpace()) if (idbdatafile::IDBPolicy::useHdfs())
{ {
if (m_compressionType) if (m_compressionType)
updateDctnryExtent(pFile, nBlocks); updateDctnryExtent(pFile, nBlocks);
@ -1867,12 +1876,21 @@ int FileOp::initDctnryExtent(
else else
Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT);
#endif #endif
int savedErrno = 0; // MCOL-498 Skip the huge preallocations if the option is set
// MCOL-498 Try to preallocate the space, fallback to write if fallocate // for the dbroot
// has failed if ( bOptExtension )
//if (!bOptExtension || pFile->fallocate(0, currFileSize, writeSize))
{ {
// Log the failed fallocate() call result bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot))
? bOptExtension : false;
}
int savedErrno = 0;
// MCOL-498 fallocate the abbreviated extent,
// fallback to sequential write if fallocate failed
if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK
&& pFile->fallocate(0, currFileSize, writeSize) )
)
{
// MCOL-498 Log the failed fallocate() call result
if ( bOptExtension ) if ( bOptExtension )
{ {
std::ostringstream oss; std::ostringstream oss;
@ -1935,23 +1953,22 @@ int FileOp::initDctnryExtent(
return ERR_FILE_WRITE; return ERR_FILE_WRITE;
} }
} }
} // CS doesn't account flush timings.
if (m_compressionType)
updateDctnryExtent(pFile, nBlocks);
// Synchronize to avoid write buffer pile up too much, which could cause
// controllernode to timeout later when it needs to save a snapshot.
pFile->flush();
#ifdef PROFILE #ifdef PROFILE
if (bExpandExtent)
if (bExpandExtent) Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT);
Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); else
else Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT);
Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT);
#endif #endif
} }
} // preallocation fallback end
// MCOL-498 CS has to set a number of blocs in the chunk header
if ( m_compressionType )
{
updateDctnryExtent(pFile, nBlocks);
}
pFile->flush();
} }
return NO_ERROR; return NO_ERROR;

8
writeengine/wrapper/we_colop.cpp
View File

@ -1527,7 +1527,7 @@ void ColumnOp::setColParam(Column& column,
* rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted * rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted
* valArray - the array of row values * valArray - the array of row values
* oldValArray - the array of old value * oldValArray - the array of old value
* bDelete - yet * bDelete - yet. The flag must be useless.
* RETURN: * RETURN:
* NO_ERROR if success, other number otherwise * NO_ERROR if success, other number otherwise
***********************************************************/ ***********************************************************/
@ -1571,7 +1571,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
} }
// MCOL-498 CS hasn't touched any block yet, // MCOL-498 CS hasn't touched any block yet,
// but the row fill be the first in the block. // but the row filled will be the first in the block.
fistRowInBlock = ( !(curRowId % (rowsInBlock)) ) ? true : false; fistRowInBlock = ( !(curRowId % (rowsInBlock)) ) ? true : false;
if( fistRowInBlock && !bDelete ) if( fistRowInBlock && !bDelete )
fillUpWEmptyVals = true; fillUpWEmptyVals = true;
@ -1708,8 +1708,6 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
if ( writeSize ) if ( writeSize )
setEmptyBuf( dataBuf + dataBio + curCol.colWidth, writeSize, setEmptyBuf( dataBuf + dataBio + curCol.colWidth, writeSize,
emptyVal, curCol.colWidth ); emptyVal, curCol.colWidth );
//fillUpWEmptyVals = false;
//fistRowInBlock = false;
} }
rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo);
@ -1726,7 +1724,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
if( !fillUpWEmptyVals ) if( !fillUpWEmptyVals )
emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth);
// MCOL-498 Skip if this is the last block in an extent. // MCOL-498 Skip if this is the last block in an extent.
if ( curDataFbo != MAX_NBLOCKS - 1) if ( curDataFbo % MAX_NBLOCKS != MAX_NBLOCKS - 1 )
{ {
rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo);
if ( rc != NO_ERROR) if ( rc != NO_ERROR)