database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-27 21:01:50 +03:00
Code Activity

MCOL-4685: Eliminate some irrelevant settings (uncompressed data and extents per file).

Browse Source 
This patch:
1. Removes the option to declare uncompressed columns (set columnstore_compression_type = 0).
2. Ignores [COMMENT '[compression=0] option at table or column level (no error messages, just disregard).
3. Removes the option to set more than 2 extents per file (ExtentsPreSegmentFile).
4. Updates rebuildEM tool to support up to 10 dictionary extent per dictionary segment file.
5. Adds check for `DBRootStorageType` for rebuildEM tool.
6. Renamed rebuildEM to mcsRebuildEM.
This commit is contained in:
Denis Khalikov
2021-05-05 15:02:08 +03:00
parent ddf19a9cc3
commit 606194e6e4
23 changed files with 170 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
dbcon/joblist/columncommand-jl.cpp
View File

@ -74,17 +74,13 @@ ColumnCommandJL::ColumnCommandJL(const pColScanStep& scan, vector<BRM::LBID_t> l
// @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use in RID calculation.
fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION;
// MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
config::Config* cf = config::Config::makeConfig();
string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition");
if ( fpc.length() != 0 )
fFilesPerColumnPartition = cf->uFromText(fpc);
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if ( epsf.length() != 0 )
fExtentsPerSegmentFile = cf->uFromText(epsf);
}
ColumnCommandJL::ColumnCommandJL(const pColStep& step)
@ -124,17 +120,13 @@ ColumnCommandJL::ColumnCommandJL(const pColStep& step)
// @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use in RID calculation.
fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION;
// MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
config::Config* cf = config::Config::makeConfig();
string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition");
if ( fpc.length() != 0 )
fFilesPerColumnPartition = cf->uFromText(fpc);
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if ( epsf.length() != 0 )
fExtentsPerSegmentFile = cf->uFromText(epsf);
}
ColumnCommandJL::~ColumnCommandJL()

5
dbcon/joblist/columncommand-jl.h
View File

@ -123,7 +123,10 @@ private:
uint32_t dbroot;
static const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 32;
static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 4;
public:
// MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile)
static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
};
}

4
dbcon/joblist/tuple-bps.cpp
View File

@ -180,10 +180,8 @@ void TupleBPS::initializeConfigParms()
fNumThreads = 0;
config::Config* cf = config::Config::makeConfig();
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if ( epsf.length() != 0 )
fExtentsPerSegFile = cf->uFromText(epsf);
fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE;
if (fRequestSize >= fMaxOutstandingRequests)
fRequestSize = 1;

37
dbcon/mysql/ha_mcs_ddl.cpp
View File

@ -189,6 +189,11 @@ int parseCompressionComment ( std::string comment )
else
compressiontype = MAX_INT;
// MCOL-4685: ignore [COMMENT '[compression=0] option at table or column level (no error
// messages, just disregard);
if (compressiontype == 0)
compressiontype = 2;
return compressiontype;
}
@ -2321,7 +2326,10 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
int compressiontype = get_compression_type(thd);
if (compressiontype == 1) compressiontype = 2;
// MCOL-4685:
// remove the option to declare uncompressed columns (set infinidb_compression_type = 0).
if (compressiontype == 1 || compressiontype == 0)
compressiontype = 2;
//string tablecomment;
if (table_arg->s->comment.length > 0 )
@ -2330,8 +2338,14 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
compressiontype = parseCompressionComment( tablecomment );
}
if ( compressiontype == MAX_INT )
if (compressiontype == MAX_INT)
{
compressiontype = get_compression_type(thd);
// MCOL-4685:
// remove the option to declare uncompressed columns (set infinidb_compression_type = 0).
if (compressiontype == 0)
compressiontype = 2;
}
else if ( compressiontype < 0 )
{
string emsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE);
@ -2644,17 +2658,20 @@ extern "C"
if ( thd->db.length )
db = thd->db.str;
// MCOL-4685:
// remove the option to declare uncompressed columns (set infinidb_compression_type = 0).
int compressiontype = get_compression_type(thd);
if (compressiontype == 1) compressiontype = 2;
if ( compressiontype == MAX_INT )
compressiontype = get_compression_type(thd);
//hdfs
if ((compressiontype == 0) && (useHdfs))
{
if (compressiontype == 1 || compressiontype == 0)
compressiontype = 2;
if (compressiontype == MAX_INT)
{
// MCOL-4685: remove the option to declare uncompressed columns (set
// infinidb_compression_type = 0).
compressiontype = get_compression_type(thd);
if (compressiontype == 0)
compressiontype = 2;
}
if (compressiontype == 1) compressiontype = 2;

2
dbcon/mysql/ha_mcs_sysvars.cpp
View File

@ -21,7 +21,7 @@
#include "ha_mcs_sysvars.h"
const char* mcs_compression_type_names[] = {
"NO_COMPRESSION",
"UNUSED",
"SNAPPY",
NullS
};

2
debian/mariadb-plugin-columnstore.install vendored
View File

@ -3,7 +3,7 @@ etc/columnstore/ErrorMessage.txt
etc/columnstore/MessageFile.txt
etc/columnstore/storagemanager.cnf
etc/mysql/mariadb.conf.d/columnstore.cnf
usr/bin/rebuildEM
usr/bin/mcsRebuildEM
usr/bin/DDLProc
usr/bin/DMLProc
usr/bin/ExeMgr

10
mysql-test/columnstore/basic/r/mcs229_data_compression_type.result
View File

@ -39,11 +39,11 @@ t2 CREATE TABLE `t2` (
) ENGINE=Columnstore DEFAULT CHARSET=latin1
SELECT `schema`, tablename, columnname, compressiontype FROM calpontsys.syscolumn WHERE `schema`='mcs229_db' ORDER BY 2;
schema tablename columnname compressiontype
mcs229_db t1 t1_int 0
mcs229_db t1 t1_text 0
mcs229_db t1 t1_date 0
mcs229_db t1 t1_time 0
mcs229_db t1 t1_decimal 0
mcs229_db t1 t1_int 2
mcs229_db t1 t1_text 2
mcs229_db t1 t1_date 2
mcs229_db t1 t1_time 2
mcs229_db t1 t1_decimal 2
mcs229_db t2 t2_date 2
mcs229_db t2 t2_int 2
mcs229_db t2 t2_text 2

1
oam/etc/Columnstore.xml
View File

@ -478,7 +478,6 @@
it cannot be changed!. Extent size is 8M rows.
-->
<FilesPerColumnPartition>4</FilesPerColumnPartition> <!-- should be multiple of DBRootCount -->
<ExtentsPerSegmentFile>2</ExtentsPerSegmentFile>
<BRM_UID>0x0</BRM_UID>
</ExtentMap>
<HashJoin>

1
oam/etc/Columnstore.xml.singleserver
View File

@ -471,7 +471,6 @@
it cannot be changed!. Valid values for ExtentRows are: 64M, 8M and 1M. The default is 8M.
-->
<FilesPerColumnPartition>4</FilesPerColumnPartition> <!-- should be multiple of DBRootCount -->
<ExtentsPerSegmentFile>2</ExtentsPerSegmentFile>
<BRM_UID>0x0</BRM_UID>
</ExtentMap>
<HashJoin>

6
tools/rebuildEM/CMakeLists.txt
View File

@ -1,6 +1,6 @@
include_directories(${ENGINE_COMMON_INCLUDES})
set(rebuildEM_SRCS main.cpp rebuildEM.cpp)
add_executable(rebuildEM ${rebuildEM_SRCS})
target_link_libraries(rebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem)
install(TARGETS rebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
add_executable(mcsRebuildEM ${rebuildEM_SRCS})
target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem boost_system)
install(TARGETS mcsRebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

27
tools/rebuildEM/main.cpp
View File

@ -19,6 +19,7 @@
#include <string>
#include <ftw.h>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string/case_conv.hpp>
#include "configcpp.h"
#include "rebuildEM.h"
@ -84,10 +85,36 @@ int main(int argc, char** argv)
return 0;
}
// MCOL-4685
std::cout << "The launch of mcsRebuildEM tool must be sanctioned by MariaDB support. "
<< std::endl;
std::cout << "Requirement: all DBRoots must be on this node. " << std::endl;
std::cout << "Note: that the launch can break the cluster." << std::endl;
std::cout << "Do you want to continue Y/N? " << std::endl;
std::string confirmation;
cin >> confirmation;
if (confirmation.size() == 0)
return 0;
boost::algorithm::to_lower(confirmation);
if (!(confirmation == "y" || confirmation == "yes"))
return 0;
auto* config = config::Config::makeConfig();
// Check for storage type.
const auto DBRootStorageType = config->getConfig("Installation", "DBRootStorageType");
if (DBRootStorageType != "internal")
{
std::cout << "Only internal DBRootStorageType is supported, provided: " << DBRootStorageType
<< std::endl;
return 0;
}
const auto BRMSavesEM =
config->getConfig("SystemConfig", "DBRMRoot") + "_em";
// Check for `BRM_saves_em` file presents.
// TODO: Should we add force option to remove file?
if (boost::filesystem::exists(BRMSavesEM))
{
std::cout << BRMSavesEM << " file exists. " << std::endl;

70
tools/rebuildEM/rebuildEM.cpp
View File

@ -132,7 +132,7 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
auto colDataType = compressor.getColDataType(fileHeader);
auto colWidth = compressor.getColumnWidth(fileHeader);
auto blockCount = compressor.getBlockCount(fileHeader);
auto lbid = compressor.getLBID0(fileHeader);
auto lbidCount = compressor.getLBIDCount(fileHeader);
if (colDataType == execplan::CalpontSystemCatalog::UNDEFINED)
{
@ -146,7 +146,6 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
auto isDict = isDictFile(colDataType, colWidth);
if (isDict)
colWidth = 8;
uint64_t hwm = 0;
if (doVerbose())
{
@ -154,48 +153,55 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
std::cout << "Block count: " << blockCount << std::endl;
}
rc =
searchHWMInSegmentFile(oid, getDBRoot(), partition, segment,
colDataType, colWidth, blockCount, isDict, hwm);
uint64_t hwm = 0;
rc = searchHWMInSegmentFile(oid, getDBRoot(), partition, segment, colDataType, colWidth,
blockCount, isDict, hwm);
if (rc != 0)
{
return rc;
}
const uint32_t extentMaxBlockCount =
getEM().getExtentRows() * colWidth / BLOCK_SIZE;
// We found multiple extents per one segment file.
if (hwm >= extentMaxBlockCount)
{
auto lbid = compressor.getLBID1(fileHeader);
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid,
hwm, isDict);
extentMap.push_back(fileId);
// Update HWM.
hwm = extentMaxBlockCount - 1;
if (doVerbose())
{
std::cout << "Found multiple extents per segment file "
<< std::endl;
std::cout << "FileId is collected " << fileId << std::endl;
}
}
if (doVerbose())
{
std::cout << "HWM is: " << hwm << std::endl;
}
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm,
isDict);
extentMap.push_back(fileId);
if (doVerbose())
const uint32_t extentMaxBlockCount = getEM().getExtentRows() * colWidth / BLOCK_SIZE;
// We found multiple extents per one segment file.
if (hwm >= extentMaxBlockCount)
{
std::cout << "FileId is collected " << fileId << std::endl;
for (uint32_t lbidIndex = 0; lbidIndex < lbidCount - 1; ++lbidIndex)
{
auto lbid = compressor.getLBIDByIndex(fileHeader, lbidIndex);
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, /*hwm*/ 0, isDict);
extentMap.push_back(fileId);
}
// Last one has an actual HWM.
auto lbid = compressor.getLBIDByIndex(fileHeader, lbidCount - 1);
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict);
extentMap.push_back(fileId);
if (doVerbose())
{
std::cout << "Found multiple extents per segment file "
<< std::endl;
std::cout << "FileId is collected " << fileId << std::endl;
}
}
else
{
// One extent per segment file.
auto lbid = compressor.getLBIDByIndex(fileHeader, 0);
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict);
extentMap.push_back(fileId);
if (doVerbose())
{
std::cout << "FileId is collected " << fileId << std::endl;
}
}
return 0;
}

47
utils/compress/idbcompress.cpp
View File

@ -63,6 +63,9 @@ const uint8_t CHUNK_MAGIC2 = 0xfe;
*/
const uint8_t CHUNK_MAGIC3 = 0xfd;
// The max number of lbids to be stored in segment file.
const uint32_t LBID_MAX_SIZE = 10;
struct CompressedDBFileHeader
{
uint64_t fMagicNumber;
@ -72,8 +75,8 @@ struct CompressedDBFileHeader
uint64_t fBlockCount;
uint64_t fColumnWidth;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
uint64_t fLBID0;
uint64_t fLBID1;
uint64_t fLBIDCount;
uint64_t fLBIDS[LBID_MAX_SIZE];
};
// Make the header to be 4K, regardless number of fields being defined/used in header.
@ -96,8 +99,8 @@ void initCompressedDBFileHeader(
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = columnWidth;
hdr->fHeader.fColDataType = colDataType;
hdr->fHeader.fLBID0 = 0;
hdr->fHeader.fLBID1 = 0;
hdr->fHeader.fLBIDCount = 0;
std::memset(hdr->fHeader.fLBIDS, 0, sizeof(hdr->fHeader.fLBIDS));
}
} // namespace
@ -432,37 +435,31 @@ uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const
}
//------------------------------------------------------------------------------
// Get start LBID
// Get LBID by index
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getLBID0(const void* hdrBuf) const
uint64_t IDBCompressInterface::getLBIDByIndex(const void* hdrBuf, uint64_t index) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID0);
if (index < LBID_MAX_SIZE)
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBIDS[index]);
return 0;
}
//------------------------------------------------------------------------------
// Set start LBID
// Set LBID by index
//------------------------------------------------------------------------------
void IDBCompressInterface::setLBID0(void* hdrBuf, uint64_t lbid) const
void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index) const
{
if (lbid)
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID0 = lbid;
if (lbid && index < LBID_MAX_SIZE)
{
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDS[index] = lbid;
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDCount =
std::max(index + 1, reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDCount);
}
}
//------------------------------------------------------------------------------
// Get start LBID
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getLBID1(const void* hdrBuf) const
uint64_t IDBCompressInterface::getLBIDCount(void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID1);
}
//------------------------------------------------------------------------------
// Set start LBID
//------------------------------------------------------------------------------
void IDBCompressInterface::setLBID1(void* hdrBuf, uint64_t lbid) const
{
if (lbid)
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID1 = lbid;
return reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBIDCount;
}
//------------------------------------------------------------------------------

29
utils/compress/idbcompress.h
View File

@ -234,24 +234,19 @@ public:
EXPORT uint64_t getColumnWidth(const void* hdrBuf) const;
/**
* getLBID
* getLBIDByIndex
*/
EXPORT uint64_t getLBID0(const void* hdrBuf) const;
EXPORT uint64_t getLBIDByIndex(const void* hdrBuf, uint64_t index) const;
/**
* setBID
* setLBIDByIndex
*/
EXPORT void setLBID0(void* hdrBuf, uint64_t lbid) const;
EXPORT void setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index) const;
/**
* getLBID
* getLBIDCount
*/
EXPORT uint64_t getLBID1(const void* hdrBuf) const;
/**
* setBID
*/
EXPORT void setLBID1(void* hdrBuf, uint64_t lbid) const;
EXPORT uint64_t getLBIDCount(void* hdrBuf) const;
/**
* Mutator methods for the user padding bytes
@ -367,15 +362,17 @@ IDBCompressInterface::getColDataType(const void* hdrBuf) const
{
return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
}
inline uint64_t getColumnWidth(const void* hdrBuf) const { return 0; }
inline uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const { return 0; }
inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize)
{
return uncompSize;
}
inline uint64_t getLBID0(const void* hdrBuf) const { return 0; }
void setLBID0(void* hdrBuf, uint64_t lbid) const {}
inline uint64_t getLBID1(const void* hdrBuf) const { return 0; }
void setLBID1(void* hdrBuf, uint64_t lbid) const {}
inline uint64_t IDBCompressInterface::getLBIDByIndex(const void* hdrBuf, uint32_t index) const
{
return 0;
}
void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint32_t index) const {}
void IDBCompressInterface::getLBIDCount(void* hdrBuf) const {}
inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen)
{
return false;

10
versioning/BRM/extentmap.cpp
View File

@ -59,6 +59,7 @@ namespace bi = boost::interprocess;
#include "oamcache.h"
#include "IDBDataFile.h"
#include "IDBPolicy.h"
#include "columncommand-jl.h"
#ifdef BRM_INFO
#include "tracer.h"
#include "configcpp.h"
@ -96,7 +97,7 @@ namespace
unsigned ExtentSize = 0; // dmc-need to deprecate
unsigned ExtentRows = 0;
unsigned filesPerColumnPartition = 0;
unsigned extentsPerSegmentFile = 0;
unsigned extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE;
// Increment CP sequence (version) number, and wrap-around when applicable
inline void incSeqNum(int32_t& seqNum)
@ -6033,11 +6034,8 @@ void ExtentMap::checkReloadConfig()
//--------------------------------------------------------------------------
// Initialize extents per segment file
//--------------------------------------------------------------------------
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
extentsPerSegmentFile = cf->uFromText(epsf);
if (extentsPerSegmentFile == 0)
extentsPerSegmentFile = 2;
// MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE;
}
//------------------------------------------------------------------------------

13
writeengine/bulk/we_colbufcompressed.cpp
View File

@ -584,21 +584,18 @@ int ColumnBufferCompressed::saveCompressionHeaders( )
char hdrBuf[IDBCompressInterface::HDR_BUF_LEN * 2];
RETURN_ON_ERROR(fColInfo->colOp->readHeaders(fFile, hdrBuf));
auto lbid = fCompressor->getLBID0(hdrBuf);
auto lbid = fCompressor->getLBIDByIndex(hdrBuf, 0);
fCompressor->initHdr(hdrBuf, fColInfo->column.width,
fColInfo->column.dataType,
fColInfo->column.compressionType);
fCompressor->setBlockCount(hdrBuf,
(fColInfo->getFileSize() / BYTE_PER_BLOCK) );
// If lbid written in the header is not 0 - we are running for the next extent for column
// segment file.
if (lbid)
{
fCompressor->setLBID0(hdrBuf, lbid);
fCompressor->setLBID1(hdrBuf, fColInfo->getLastUpdatedLBID());
}
fCompressor->setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 1);
else
{
fCompressor->setLBID0(hdrBuf, fColInfo->getLastUpdatedLBID());
}
fCompressor->setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 0);
std::vector<uint64_t> ptrs;

6
writeengine/server/we_ddlcommandproc.cpp
View File

@ -59,10 +59,8 @@ WE_DDLCommandProc::WE_DDLCommandProc()
if (fpc.length() != 0)
filesPerColumnPartition = cf->uFromText(fpc);
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if (epsf.length() != 0)
extentsPerSegmentFile = cf->uFromText(epsf);
// MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
string dbct = cf->getConfig("SystemConfig", "DBRootCount");

2
writeengine/server/we_ddlcommandproc.h
View File

@ -112,7 +112,7 @@ private:
WriteEngineWrapper fWEWrapper;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
};
}
#undef EXPORT

6
writeengine/server/we_dmlcommandproc.cpp
View File

@ -70,10 +70,8 @@ WE_DMLCommandProc::WE_DMLCommandProc()
if (fpc.length() != 0)
filesPerColumnPartition = cf->uFromText(fpc);
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if (epsf.length() != 0)
extentsPerSegmentFile = cf->uFromText(epsf);
// MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
string dbct = cf->getConfig("SystemConfig", "DBRootCount");

1
writeengine/server/we_dmlcommandproc.h
View File

@ -132,6 +132,7 @@ private:
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
Log fLog;
static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
};
}

10
writeengine/shared/we_chunkmanager.cpp
View File

@ -466,7 +466,7 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
fileData->fFileHeader.fPtrSection,
/*colWidth=*/0, fileData->fColDataType,
fFileOp->compressionType(), hdrSize);
fCompressor.setLBID0(fileData->fFileHeader.fControlData, lbid);
fCompressor.setLBIDByIndex(fileData->fFileHeader.fControlData, lbid, 0);
if (writeHeader(fileData, __LINE__) != NO_ERROR)
{
@ -1423,7 +1423,7 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int6
int rc = NO_ERROR;
char* hdr = pFileData->fFileHeader.fControlData;
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
fCompressor.setLBID1(hdr, lbid);
fCompressor.setLBIDByIndex(hdr, lbid, 1);
ChunkData* chunkData = (pFileData)->findChunk(0);
if (chunkData != NULL)
@ -1514,7 +1514,11 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount,
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
if (currentBlockCount)
fCompressor.setLBID1(hdr, lbid);
{
// Append to the end.
uint64_t lbidCount = fCompressor.getLBIDCount(hdr);
fCompressor.setLBIDByIndex(hdr, lbid, lbidCount);
}
return rc;
}

4
writeengine/shared/we_config.cpp
View File

@ -153,10 +153,6 @@ void Config::checkReload( )
// Initialize extents per segment file
//--------------------------------------------------------------------------
m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
if ( epsf.length() != 0 )
m_ExtentsPerSegmentFile = cf->uFromText(epsf);
//--------------------------------------------------------------------------
// Initialize bulk load process priority

8
writeengine/shared/we_fileop.cpp
View File

@ -818,7 +818,7 @@ int FileOp::extendFile(
{
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, startLbid);
compressor.setLBIDByIndex(hdrs, startLbid, 0);
}
}
@ -978,7 +978,7 @@ int FileOp::addExtentExactFile(
{
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, startLbid);
compressor.setLBIDByIndex(hdrs, startLbid, 0);
}
}
@ -1067,7 +1067,7 @@ int FileOp::initColumnExtent(
char hdrs[IDBCompressInterface::HDR_BUF_LEN * 2];
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, lbid);
compressor.setLBIDByIndex(hdrs, lbid, 0);
if (bAbbrevExtent)
compressor.setBlockCount(hdrs, nBlocks);
@ -1349,7 +1349,7 @@ int FileOp::writeInitialCompColumnChunk(
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setBlockCount(hdrs, nBlocksAllocated);
compressor.setLBID0(hdrs, startLBID);
compressor.setLBIDByIndex(hdrs, startLBID, 0);
// Store compression pointers in the header
std::vector<uint64_t> ptrs;