From 606194e6e432e89bf9b85aeb1536282d497716c8 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Wed, 5 May 2021 15:02:08 +0300 Subject: [PATCH] MCOL-4685: Eliminate some irrelevant settings (uncompressed data and extents per file). This patch: 1. Removes the option to declare uncompressed columns (set columnstore_compression_type = 0). 2. Ignores [COMMENT '[compression=0] option at table or column level (no error messages, just disregard). 3. Removes the option to set more than 2 extents per file (ExtentsPreSegmentFile). 4. Updates rebuildEM tool to support up to 10 dictionary extent per dictionary segment file. 5. Adds check for `DBRootStorageType` for rebuildEM tool. 6. Renamed rebuildEM to mcsRebuildEM. --- dbcon/joblist/columncommand-jl.cpp | 12 +--- dbcon/joblist/columncommand-jl.h | 5 +- dbcon/joblist/tuple-bps.cpp | 4 +- dbcon/mysql/ha_mcs_ddl.cpp | 37 +++++++--- dbcon/mysql/ha_mcs_sysvars.cpp | 2 +- debian/mariadb-plugin-columnstore.install | 2 +- .../r/mcs229_data_compression_type.result | 10 +-- oam/etc/Columnstore.xml | 1 - oam/etc/Columnstore.xml.singleserver | 1 - tools/rebuildEM/CMakeLists.txt | 6 +- tools/rebuildEM/main.cpp | 27 +++++++ tools/rebuildEM/rebuildEM.cpp | 70 ++++++++++--------- utils/compress/idbcompress.cpp | 47 ++++++------- utils/compress/idbcompress.h | 29 ++++---- versioning/BRM/extentmap.cpp | 10 ++- writeengine/bulk/we_colbufcompressed.cpp | 13 ++-- writeengine/server/we_ddlcommandproc.cpp | 6 +- writeengine/server/we_ddlcommandproc.h | 2 +- writeengine/server/we_dmlcommandproc.cpp | 6 +- writeengine/server/we_dmlcommandproc.h | 1 + writeengine/shared/we_chunkmanager.cpp | 10 ++- writeengine/shared/we_config.cpp | 4 -- writeengine/shared/we_fileop.cpp | 8 +-- 23 files changed, 170 insertions(+), 143 deletions(-) diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 10d4ea21e..1910e7e7a 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -74,17 +74,13 @@ ColumnCommandJL::ColumnCommandJL(const pColScanStep& scan, vector l // @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use in RID calculation. fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; + // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; config::Config* cf = config::Config::makeConfig(); string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); if ( fpc.length() != 0 ) fFilesPerColumnPartition = cf->uFromText(fpc); - - string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - - if ( epsf.length() != 0 ) - fExtentsPerSegmentFile = cf->uFromText(epsf); } ColumnCommandJL::ColumnCommandJL(const pColStep& step) @@ -124,17 +120,13 @@ ColumnCommandJL::ColumnCommandJL(const pColStep& step) // @Bug 2889. Drop partition enhancement. Read FilesPerColumnPartition and ExtentsPerSegmentFile for use in RID calculation. fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION; + // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; config::Config* cf = config::Config::makeConfig(); string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition"); if ( fpc.length() != 0 ) fFilesPerColumnPartition = cf->uFromText(fpc); - - string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - - if ( epsf.length() != 0 ) - fExtentsPerSegmentFile = cf->uFromText(epsf); } ColumnCommandJL::~ColumnCommandJL() diff --git a/dbcon/joblist/columncommand-jl.h b/dbcon/joblist/columncommand-jl.h index 826a8279c..9206f484d 100644 --- a/dbcon/joblist/columncommand-jl.h +++ b/dbcon/joblist/columncommand-jl.h @@ -123,7 +123,10 @@ private: uint32_t dbroot; static const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 32; - static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 4; + + public: + // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile) + static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; }; } diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index ec6ce6b4b..65ea7d9cb 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -180,10 +180,8 @@ void TupleBPS::initializeConfigParms() fNumThreads = 0; config::Config* cf = config::Config::makeConfig(); - string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - if ( epsf.length() != 0 ) - fExtentsPerSegFile = cf->uFromText(epsf); + fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE; if (fRequestSize >= fMaxOutstandingRequests) fRequestSize = 1; diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index e5738e993..339b43750 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -189,6 +189,11 @@ int parseCompressionComment ( std::string comment ) else compressiontype = MAX_INT; + // MCOL-4685: ignore [COMMENT '[compression=0] option at table or column level (no error + // messages, just disregard); + if (compressiontype == 0) + compressiontype = 2; + return compressiontype; } @@ -2321,7 +2326,10 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea int compressiontype = get_compression_type(thd); - if (compressiontype == 1) compressiontype = 2; + // MCOL-4685: + // remove the option to declare uncompressed columns (set infinidb_compression_type = 0). + if (compressiontype == 1 || compressiontype == 0) + compressiontype = 2; //string tablecomment; if (table_arg->s->comment.length > 0 ) @@ -2330,8 +2338,14 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea compressiontype = parseCompressionComment( tablecomment ); } - if ( compressiontype == MAX_INT ) + if (compressiontype == MAX_INT) + { compressiontype = get_compression_type(thd); + // MCOL-4685: + // remove the option to declare uncompressed columns (set infinidb_compression_type = 0). + if (compressiontype == 0) + compressiontype = 2; + } else if ( compressiontype < 0 ) { string emsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE); @@ -2644,17 +2658,20 @@ extern "C" if ( thd->db.length ) db = thd->db.str; + // MCOL-4685: + // remove the option to declare uncompressed columns (set infinidb_compression_type = 0). int compressiontype = get_compression_type(thd); - if (compressiontype == 1) compressiontype = 2; - - if ( compressiontype == MAX_INT ) - compressiontype = get_compression_type(thd); - - //hdfs - if ((compressiontype == 0) && (useHdfs)) - { + if (compressiontype == 1 || compressiontype == 0) compressiontype = 2; + + if (compressiontype == MAX_INT) + { + // MCOL-4685: remove the option to declare uncompressed columns (set + // infinidb_compression_type = 0). + compressiontype = get_compression_type(thd); + if (compressiontype == 0) + compressiontype = 2; } if (compressiontype == 1) compressiontype = 2; diff --git a/dbcon/mysql/ha_mcs_sysvars.cpp b/dbcon/mysql/ha_mcs_sysvars.cpp index fc037005b..a0a925e40 100644 --- a/dbcon/mysql/ha_mcs_sysvars.cpp +++ b/dbcon/mysql/ha_mcs_sysvars.cpp @@ -21,7 +21,7 @@ #include "ha_mcs_sysvars.h" const char* mcs_compression_type_names[] = { - "NO_COMPRESSION", + "UNUSED", "SNAPPY", NullS }; diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index bd0c32dfd..eca3098d7 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -3,7 +3,7 @@ etc/columnstore/ErrorMessage.txt etc/columnstore/MessageFile.txt etc/columnstore/storagemanager.cnf etc/mysql/mariadb.conf.d/columnstore.cnf -usr/bin/rebuildEM +usr/bin/mcsRebuildEM usr/bin/DDLProc usr/bin/DMLProc usr/bin/ExeMgr diff --git a/mysql-test/columnstore/basic/r/mcs229_data_compression_type.result b/mysql-test/columnstore/basic/r/mcs229_data_compression_type.result index ae9e34b56..b00dc3754 100644 --- a/mysql-test/columnstore/basic/r/mcs229_data_compression_type.result +++ b/mysql-test/columnstore/basic/r/mcs229_data_compression_type.result @@ -39,11 +39,11 @@ t2 CREATE TABLE `t2` ( ) ENGINE=Columnstore DEFAULT CHARSET=latin1 SELECT `schema`, tablename, columnname, compressiontype FROM calpontsys.syscolumn WHERE `schema`='mcs229_db' ORDER BY 2; schema tablename columnname compressiontype -mcs229_db t1 t1_int 0 -mcs229_db t1 t1_text 0 -mcs229_db t1 t1_date 0 -mcs229_db t1 t1_time 0 -mcs229_db t1 t1_decimal 0 +mcs229_db t1 t1_int 2 +mcs229_db t1 t1_text 2 +mcs229_db t1 t1_date 2 +mcs229_db t1 t1_time 2 +mcs229_db t1 t1_decimal 2 mcs229_db t2 t2_date 2 mcs229_db t2 t2_int 2 mcs229_db t2 t2_text 2 diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 582dec5d8..14734d9d4 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -478,7 +478,6 @@ it cannot be changed!. Extent size is 8M rows. --> 4 - 2 0x0 diff --git a/oam/etc/Columnstore.xml.singleserver b/oam/etc/Columnstore.xml.singleserver index d8c7e2e05..1dbd870d6 100644 --- a/oam/etc/Columnstore.xml.singleserver +++ b/oam/etc/Columnstore.xml.singleserver @@ -471,7 +471,6 @@ it cannot be changed!. Valid values for ExtentRows are: 64M, 8M and 1M. The default is 8M. --> 4 - 2 0x0 diff --git a/tools/rebuildEM/CMakeLists.txt b/tools/rebuildEM/CMakeLists.txt index 3e5feeb9a..54bd14e6b 100644 --- a/tools/rebuildEM/CMakeLists.txt +++ b/tools/rebuildEM/CMakeLists.txt @@ -1,6 +1,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rebuildEM_SRCS main.cpp rebuildEM.cpp) -add_executable(rebuildEM ${rebuildEM_SRCS}) -target_link_libraries(rebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem) -install(TARGETS rebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +add_executable(mcsRebuildEM ${rebuildEM_SRCS}) +target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem boost_system) +install(TARGETS mcsRebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/tools/rebuildEM/main.cpp b/tools/rebuildEM/main.cpp index a3d876705..2dc187fc5 100644 --- a/tools/rebuildEM/main.cpp +++ b/tools/rebuildEM/main.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include "configcpp.h" #include "rebuildEM.h" @@ -84,10 +85,36 @@ int main(int argc, char** argv) return 0; } + // MCOL-4685 + std::cout << "The launch of mcsRebuildEM tool must be sanctioned by MariaDB support. " + << std::endl; + std::cout << "Requirement: all DBRoots must be on this node. " << std::endl; + std::cout << "Note: that the launch can break the cluster." << std::endl; + std::cout << "Do you want to continue Y/N? " << std::endl; + std::string confirmation; + cin >> confirmation; + if (confirmation.size() == 0) + return 0; + + boost::algorithm::to_lower(confirmation); + if (!(confirmation == "y" || confirmation == "yes")) + return 0; + auto* config = config::Config::makeConfig(); + + // Check for storage type. + const auto DBRootStorageType = config->getConfig("Installation", "DBRootStorageType"); + if (DBRootStorageType != "internal") + { + std::cout << "Only internal DBRootStorageType is supported, provided: " << DBRootStorageType + << std::endl; + return 0; + } + const auto BRMSavesEM = config->getConfig("SystemConfig", "DBRMRoot") + "_em"; // Check for `BRM_saves_em` file presents. + // TODO: Should we add force option to remove file? if (boost::filesystem::exists(BRMSavesEM)) { std::cout << BRMSavesEM << " file exists. " << std::endl; diff --git a/tools/rebuildEM/rebuildEM.cpp b/tools/rebuildEM/rebuildEM.cpp index 7ba7247d5..2895d40b4 100644 --- a/tools/rebuildEM/rebuildEM.cpp +++ b/tools/rebuildEM/rebuildEM.cpp @@ -132,7 +132,7 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName) auto colDataType = compressor.getColDataType(fileHeader); auto colWidth = compressor.getColumnWidth(fileHeader); auto blockCount = compressor.getBlockCount(fileHeader); - auto lbid = compressor.getLBID0(fileHeader); + auto lbidCount = compressor.getLBIDCount(fileHeader); if (colDataType == execplan::CalpontSystemCatalog::UNDEFINED) { @@ -146,7 +146,6 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName) auto isDict = isDictFile(colDataType, colWidth); if (isDict) colWidth = 8; - uint64_t hwm = 0; if (doVerbose()) { @@ -154,48 +153,55 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName) std::cout << "Block count: " << blockCount << std::endl; } - rc = - searchHWMInSegmentFile(oid, getDBRoot(), partition, segment, - colDataType, colWidth, blockCount, isDict, hwm); + uint64_t hwm = 0; + rc = searchHWMInSegmentFile(oid, getDBRoot(), partition, segment, colDataType, colWidth, + blockCount, isDict, hwm); if (rc != 0) { return rc; } - const uint32_t extentMaxBlockCount = - getEM().getExtentRows() * colWidth / BLOCK_SIZE; - - // We found multiple extents per one segment file. - if (hwm >= extentMaxBlockCount) - { - auto lbid = compressor.getLBID1(fileHeader); - FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, - hwm, isDict); - extentMap.push_back(fileId); - - // Update HWM. - hwm = extentMaxBlockCount - 1; - if (doVerbose()) - { - std::cout << "Found multiple extents per segment file " - << std::endl; - std::cout << "FileId is collected " << fileId << std::endl; - } - } - if (doVerbose()) { std::cout << "HWM is: " << hwm << std::endl; } - FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, - isDict); - extentMap.push_back(fileId); - - if (doVerbose()) + const uint32_t extentMaxBlockCount = getEM().getExtentRows() * colWidth / BLOCK_SIZE; + // We found multiple extents per one segment file. + if (hwm >= extentMaxBlockCount) { - std::cout << "FileId is collected " << fileId << std::endl; + for (uint32_t lbidIndex = 0; lbidIndex < lbidCount - 1; ++lbidIndex) + { + auto lbid = compressor.getLBIDByIndex(fileHeader, lbidIndex); + FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, /*hwm*/ 0, isDict); + extentMap.push_back(fileId); + } + + // Last one has an actual HWM. + auto lbid = compressor.getLBIDByIndex(fileHeader, lbidCount - 1); + FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict); + extentMap.push_back(fileId); + + if (doVerbose()) + { + std::cout << "Found multiple extents per segment file " + << std::endl; + std::cout << "FileId is collected " << fileId << std::endl; + } } + else + { + // One extent per segment file. + auto lbid = compressor.getLBIDByIndex(fileHeader, 0); + FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict); + extentMap.push_back(fileId); + + if (doVerbose()) + { + std::cout << "FileId is collected " << fileId << std::endl; + } + } + return 0; } diff --git a/utils/compress/idbcompress.cpp b/utils/compress/idbcompress.cpp index 993a0c224..a434f19f5 100644 --- a/utils/compress/idbcompress.cpp +++ b/utils/compress/idbcompress.cpp @@ -63,6 +63,9 @@ const uint8_t CHUNK_MAGIC2 = 0xfe; */ const uint8_t CHUNK_MAGIC3 = 0xfd; +// The max number of lbids to be stored in segment file. +const uint32_t LBID_MAX_SIZE = 10; + struct CompressedDBFileHeader { uint64_t fMagicNumber; @@ -72,8 +75,8 @@ struct CompressedDBFileHeader uint64_t fBlockCount; uint64_t fColumnWidth; execplan::CalpontSystemCatalog::ColDataType fColDataType; - uint64_t fLBID0; - uint64_t fLBID1; + uint64_t fLBIDCount; + uint64_t fLBIDS[LBID_MAX_SIZE]; }; // Make the header to be 4K, regardless number of fields being defined/used in header. @@ -96,8 +99,8 @@ void initCompressedDBFileHeader( hdr->fHeader.fHeaderSize = hdrSize; hdr->fHeader.fColumnWidth = columnWidth; hdr->fHeader.fColDataType = colDataType; - hdr->fHeader.fLBID0 = 0; - hdr->fHeader.fLBID1 = 0; + hdr->fHeader.fLBIDCount = 0; + std::memset(hdr->fHeader.fLBIDS, 0, sizeof(hdr->fHeader.fLBIDS)); } } // namespace @@ -432,37 +435,31 @@ uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const } //------------------------------------------------------------------------------ -// Get start LBID +// Get LBID by index //------------------------------------------------------------------------------ -uint64_t IDBCompressInterface::getLBID0(const void* hdrBuf) const +uint64_t IDBCompressInterface::getLBIDByIndex(const void* hdrBuf, uint64_t index) const { - return (reinterpret_cast(hdrBuf)->fLBID0); + if (index < LBID_MAX_SIZE) + return (reinterpret_cast(hdrBuf)->fLBIDS[index]); + return 0; } //------------------------------------------------------------------------------ -// Set start LBID +// Set LBID by index //------------------------------------------------------------------------------ -void IDBCompressInterface::setLBID0(void* hdrBuf, uint64_t lbid) const +void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index) const { - if (lbid) - reinterpret_cast(hdrBuf)->fLBID0 = lbid; + if (lbid && index < LBID_MAX_SIZE) + { + reinterpret_cast(hdrBuf)->fLBIDS[index] = lbid; + reinterpret_cast(hdrBuf)->fLBIDCount = + std::max(index + 1, reinterpret_cast(hdrBuf)->fLBIDCount); + } } -//------------------------------------------------------------------------------ -// Get start LBID -//------------------------------------------------------------------------------ -uint64_t IDBCompressInterface::getLBID1(const void* hdrBuf) const +uint64_t IDBCompressInterface::getLBIDCount(void* hdrBuf) const { - return (reinterpret_cast(hdrBuf)->fLBID1); -} - -//------------------------------------------------------------------------------ -// Set start LBID -//------------------------------------------------------------------------------ -void IDBCompressInterface::setLBID1(void* hdrBuf, uint64_t lbid) const -{ - if (lbid) - reinterpret_cast(hdrBuf)->fLBID1 = lbid; + return reinterpret_cast(hdrBuf)->fLBIDCount; } //------------------------------------------------------------------------------ diff --git a/utils/compress/idbcompress.h b/utils/compress/idbcompress.h index 196254ffe..03d327a41 100644 --- a/utils/compress/idbcompress.h +++ b/utils/compress/idbcompress.h @@ -234,24 +234,19 @@ public: EXPORT uint64_t getColumnWidth(const void* hdrBuf) const; /** - * getLBID + * getLBIDByIndex */ - EXPORT uint64_t getLBID0(const void* hdrBuf) const; + EXPORT uint64_t getLBIDByIndex(const void* hdrBuf, uint64_t index) const; /** - * setBID + * setLBIDByIndex */ - EXPORT void setLBID0(void* hdrBuf, uint64_t lbid) const; + EXPORT void setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index) const; /** - * getLBID + * getLBIDCount */ - EXPORT uint64_t getLBID1(const void* hdrBuf) const; - - /** - * setBID - */ - EXPORT void setLBID1(void* hdrBuf, uint64_t lbid) const; + EXPORT uint64_t getLBIDCount(void* hdrBuf) const; /** * Mutator methods for the user padding bytes @@ -367,15 +362,17 @@ IDBCompressInterface::getColDataType(const void* hdrBuf) const { return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED; } -inline uint64_t getColumnWidth(const void* hdrBuf) const { return 0; } +inline uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const { return 0; } inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize) { return uncompSize; } -inline uint64_t getLBID0(const void* hdrBuf) const { return 0; } -void setLBID0(void* hdrBuf, uint64_t lbid) const {} -inline uint64_t getLBID1(const void* hdrBuf) const { return 0; } -void setLBID1(void* hdrBuf, uint64_t lbid) const {} +inline uint64_t IDBCompressInterface::getLBIDByIndex(const void* hdrBuf, uint32_t index) const +{ + return 0; +} +void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint32_t index) const {} +void IDBCompressInterface::getLBIDCount(void* hdrBuf) const {} inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen) { return false; diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index df4b27b79..aba3e3211 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -59,6 +59,7 @@ namespace bi = boost::interprocess; #include "oamcache.h" #include "IDBDataFile.h" #include "IDBPolicy.h" +#include "columncommand-jl.h" #ifdef BRM_INFO #include "tracer.h" #include "configcpp.h" @@ -96,7 +97,7 @@ namespace unsigned ExtentSize = 0; // dmc-need to deprecate unsigned ExtentRows = 0; unsigned filesPerColumnPartition = 0; -unsigned extentsPerSegmentFile = 0; +unsigned extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE; // Increment CP sequence (version) number, and wrap-around when applicable inline void incSeqNum(int32_t& seqNum) @@ -6033,11 +6034,8 @@ void ExtentMap::checkReloadConfig() //-------------------------------------------------------------------------- // Initialize extents per segment file //-------------------------------------------------------------------------- - string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - extentsPerSegmentFile = cf->uFromText(epsf); - - if (extentsPerSegmentFile == 0) - extentsPerSegmentFile = 2; + // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE; } //------------------------------------------------------------------------------ diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index ef1c66ed9..c64bf9257 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -584,21 +584,18 @@ int ColumnBufferCompressed::saveCompressionHeaders( ) char hdrBuf[IDBCompressInterface::HDR_BUF_LEN * 2]; RETURN_ON_ERROR(fColInfo->colOp->readHeaders(fFile, hdrBuf)); - auto lbid = fCompressor->getLBID0(hdrBuf); + auto lbid = fCompressor->getLBIDByIndex(hdrBuf, 0); fCompressor->initHdr(hdrBuf, fColInfo->column.width, fColInfo->column.dataType, fColInfo->column.compressionType); fCompressor->setBlockCount(hdrBuf, (fColInfo->getFileSize() / BYTE_PER_BLOCK) ); + // If lbid written in the header is not 0 - we are running for the next extent for column + // segment file. if (lbid) - { - fCompressor->setLBID0(hdrBuf, lbid); - fCompressor->setLBID1(hdrBuf, fColInfo->getLastUpdatedLBID()); - } + fCompressor->setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 1); else - { - fCompressor->setLBID0(hdrBuf, fColInfo->getLastUpdatedLBID()); - } + fCompressor->setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 0); std::vector ptrs; diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index f78e6ebd3..6e955c853 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -59,10 +59,8 @@ WE_DDLCommandProc::WE_DDLCommandProc() if (fpc.length() != 0) filesPerColumnPartition = cf->uFromText(fpc); - string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - - if (epsf.length() != 0) - extentsPerSegmentFile = cf->uFromText(epsf); + // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; string dbct = cf->getConfig("SystemConfig", "DBRootCount"); diff --git a/writeengine/server/we_ddlcommandproc.h b/writeengine/server/we_ddlcommandproc.h index e5568130f..786fbcafa 100644 --- a/writeengine/server/we_ddlcommandproc.h +++ b/writeengine/server/we_ddlcommandproc.h @@ -112,7 +112,7 @@ private: WriteEngineWrapper fWEWrapper; BRM::DBRM fDbrm; unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt; - + static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; }; } #undef EXPORT diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index f94454ad6..7f0d57f73 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -70,10 +70,8 @@ WE_DMLCommandProc::WE_DMLCommandProc() if (fpc.length() != 0) filesPerColumnPartition = cf->uFromText(fpc); - string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - - if (epsf.length() != 0) - extentsPerSegmentFile = cf->uFromText(epsf); + // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile). + extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; string dbct = cf->getConfig("SystemConfig", "DBRootCount"); diff --git a/writeengine/server/we_dmlcommandproc.h b/writeengine/server/we_dmlcommandproc.h index e80597d68..4265aa59c 100644 --- a/writeengine/server/we_dmlcommandproc.h +++ b/writeengine/server/we_dmlcommandproc.h @@ -132,6 +132,7 @@ private: BRM::DBRM fDbrm; unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt; Log fLog; + static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2; }; } diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index bdcdd7f3e..8ee736c73 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -466,7 +466,7 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid, fileData->fFileHeader.fPtrSection, /*colWidth=*/0, fileData->fColDataType, fFileOp->compressionType(), hdrSize); - fCompressor.setLBID0(fileData->fFileHeader.fControlData, lbid); + fCompressor.setLBIDByIndex(fileData->fFileHeader.fControlData, lbid, 0); if (writeHeader(fileData, __LINE__) != NO_ERROR) { @@ -1423,7 +1423,7 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int6 int rc = NO_ERROR; char* hdr = pFileData->fFileHeader.fControlData; fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount); - fCompressor.setLBID1(hdr, lbid); + fCompressor.setLBIDByIndex(hdr, lbid, 1); ChunkData* chunkData = (pFileData)->findChunk(0); if (chunkData != NULL) @@ -1514,7 +1514,11 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount); if (currentBlockCount) - fCompressor.setLBID1(hdr, lbid); + { + // Append to the end. + uint64_t lbidCount = fCompressor.getLBIDCount(hdr); + fCompressor.setLBIDByIndex(hdr, lbid, lbidCount); + } return rc; } diff --git a/writeengine/shared/we_config.cpp b/writeengine/shared/we_config.cpp index f4d163290..b75f1cb1a 100644 --- a/writeengine/shared/we_config.cpp +++ b/writeengine/shared/we_config.cpp @@ -153,10 +153,6 @@ void Config::checkReload( ) // Initialize extents per segment file //-------------------------------------------------------------------------- m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE; - string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile"); - - if ( epsf.length() != 0 ) - m_ExtentsPerSegmentFile = cf->uFromText(epsf); //-------------------------------------------------------------------------- // Initialize bulk load process priority diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 02dc6610e..52785a013 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -818,7 +818,7 @@ int FileOp::extendFile( { IDBCompressInterface compressor; compressor.initHdr(hdrs, width, colDataType, m_compressionType); - compressor.setLBID0(hdrs, startLbid); + compressor.setLBIDByIndex(hdrs, startLbid, 0); } } @@ -978,7 +978,7 @@ int FileOp::addExtentExactFile( { IDBCompressInterface compressor; compressor.initHdr(hdrs, width, colDataType, m_compressionType); - compressor.setLBID0(hdrs, startLbid); + compressor.setLBIDByIndex(hdrs, startLbid, 0); } } @@ -1067,7 +1067,7 @@ int FileOp::initColumnExtent( char hdrs[IDBCompressInterface::HDR_BUF_LEN * 2]; IDBCompressInterface compressor; compressor.initHdr(hdrs, width, colDataType, m_compressionType); - compressor.setLBID0(hdrs, lbid); + compressor.setLBIDByIndex(hdrs, lbid, 0); if (bAbbrevExtent) compressor.setBlockCount(hdrs, nBlocks); @@ -1349,7 +1349,7 @@ int FileOp::writeInitialCompColumnChunk( compressor.initHdr(hdrs, width, colDataType, m_compressionType); compressor.setBlockCount(hdrs, nBlocksAllocated); - compressor.setLBID0(hdrs, startLBID); + compressor.setLBIDByIndex(hdrs, startLBID, 0); // Store compression pointers in the header std::vector ptrs;