MCOL-4685: Eliminate some irrelevant settings (uncompressed data and extents per file).

This patch:
1. Removes the option to declare uncompressed columns (set columnstore_compression_type = 0).
2. Ignores [COMMENT '[compression=0] option at table or column level (no error messages, just disregard).
3. Removes the option to set more than 2 extents per file (ExtentsPreSegmentFile).
4. Updates rebuildEM tool to support up to 10 dictionary extent per dictionary segment file.
5. Adds check for `DBRootStorageType` for rebuildEM tool.
6. Renamed rebuildEM to mcsRebuildEM.

dbcon/joblist/columncommand-jl.cpp

@@ -74,17 +74,13 @@ ColumnCommandJL::ColumnCommandJL(const pColScanStep& scan, vector<BRM::LBID_t> l
 
     // @Bug 2889.  Drop partition enhancement.  Read FilesPerColumnPartition and ExtentsPerSegmentFile for use in RID calculation.
     fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION;
+    // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
     fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
     config::Config* cf = config::Config::makeConfig();
     string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition");
 
     if ( fpc.length() != 0 )
         fFilesPerColumnPartition = cf->uFromText(fpc);
-
-    string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
-
-    if ( epsf.length() != 0 )
-        fExtentsPerSegmentFile = cf->uFromText(epsf);
 }
 
 ColumnCommandJL::ColumnCommandJL(const pColStep& step)
@@ -124,17 +120,13 @@ ColumnCommandJL::ColumnCommandJL(const pColStep& step)
 
     // @Bug 2889.  Drop partition enhancement.  Read FilesPerColumnPartition and ExtentsPerSegmentFile for use in RID calculation.
     fFilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION;
+    // MCOL-4685 remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
     fExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
     config::Config* cf = config::Config::makeConfig();
     string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition");
 
     if ( fpc.length() != 0 )
         fFilesPerColumnPartition = cf->uFromText(fpc);
-
-    string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
-
-    if ( epsf.length() != 0 )
-        fExtentsPerSegmentFile = cf->uFromText(epsf);
 }
 
 ColumnCommandJL::~ColumnCommandJL()

dbcon/joblist/columncommand-jl.h

@@ -123,7 +123,10 @@ private:
     uint32_t dbroot;
 
     static const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 32;
-    static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE   =  4;
+
+  public:
+    // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile)
+    static const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
 };
 
 }

dbcon/joblist/tuple-bps.cpp

@@ -180,10 +180,8 @@ void TupleBPS::initializeConfigParms()
     fNumThreads = 0;
 
     config::Config* cf = config::Config::makeConfig();
-    string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
 
-    if ( epsf.length() != 0 )
+    fExtentsPerSegFile = DEFAULT_EXTENTS_PER_SEG_FILE;
-        fExtentsPerSegFile = cf->uFromText(epsf);
 
     if (fRequestSize >= fMaxOutstandingRequests)
         fRequestSize = 1;

dbcon/mysql/ha_mcs_ddl.cpp

@@ -189,6 +189,11 @@ int parseCompressionComment ( std::string comment )
     else
         compressiontype = MAX_INT;
 
+    // MCOL-4685: ignore [COMMENT '[compression=0] option at table or column level (no error
+    // messages, just disregard);
+    if (compressiontype == 0)
+      compressiontype = 2;
+
     return compressiontype;
 }
 
@@ -2321,7 +2326,10 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
 
     int compressiontype = get_compression_type(thd);
 
-    if (compressiontype == 1) compressiontype = 2;
+    // MCOL-4685:
+    // remove the option to declare uncompressed columns (set infinidb_compression_type = 0).
+    if (compressiontype == 1 || compressiontype == 0)
+        compressiontype = 2;
 
     //string tablecomment;
     if (table_arg->s->comment.length > 0 )
@@ -2331,7 +2339,13 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
     }
 
     if (compressiontype == MAX_INT)
+    {
         compressiontype = get_compression_type(thd);
+        // MCOL-4685:
+        // remove the option to declare uncompressed columns (set infinidb_compression_type = 0).
+        if (compressiontype == 0)
+            compressiontype = 2;
+    }
     else if ( compressiontype < 0 )
     {
         string emsg = IDBErrorInfo::instance()->errorMsg(ERR_INVALID_COMPRESSION_TYPE);
@@ -2644,16 +2658,19 @@ extern "C"
         if ( thd->db.length )
             db = thd->db.str;
 
+        // MCOL-4685:
+        // remove the option to declare uncompressed columns (set infinidb_compression_type = 0).
         int compressiontype = get_compression_type(thd);
 
-        if (compressiontype == 1) compressiontype = 2;
+        if (compressiontype == 1 || compressiontype == 0)
+            compressiontype = 2;
 
         if (compressiontype == MAX_INT)
-            compressiontype = get_compression_type(thd);
-
-        //hdfs
-        if ((compressiontype == 0) && (useHdfs))
         {
+            // MCOL-4685: remove the option to declare uncompressed columns (set
+            // infinidb_compression_type = 0).
+            compressiontype = get_compression_type(thd);
+            if (compressiontype == 0)
                 compressiontype = 2;
         }
 

dbcon/mysql/ha_mcs_sysvars.cpp

@@ -21,7 +21,7 @@
 #include "ha_mcs_sysvars.h"
 
 const char* mcs_compression_type_names[] = {
-    "NO_COMPRESSION",
+    "UNUSED",
     "SNAPPY",
     NullS
 };

debian/mariadb-plugin-columnstore.install

@@ -3,7 +3,7 @@ etc/columnstore/ErrorMessage.txt
 etc/columnstore/MessageFile.txt
 etc/columnstore/storagemanager.cnf
 etc/mysql/mariadb.conf.d/columnstore.cnf
-usr/bin/rebuildEM
+usr/bin/mcsRebuildEM
 usr/bin/DDLProc
 usr/bin/DMLProc
 usr/bin/ExeMgr

mysql-test/columnstore/basic/r/mcs229_data_compression_type.result

@@ -39,11 +39,11 @@ t2	CREATE TABLE `t2` (
 ) ENGINE=Columnstore DEFAULT CHARSET=latin1
 SELECT `schema`, tablename, columnname, compressiontype FROM calpontsys.syscolumn WHERE `schema`='mcs229_db' ORDER BY 2;
 schema	tablename	columnname	compressiontype
-mcs229_db	t1	t1_int	0
+mcs229_db	t1	t1_int	2
-mcs229_db	t1	t1_text	0
+mcs229_db	t1	t1_text	2
-mcs229_db	t1	t1_date	0
+mcs229_db	t1	t1_date	2
-mcs229_db	t1	t1_time	0
+mcs229_db	t1	t1_time	2
-mcs229_db	t1	t1_decimal	0
+mcs229_db	t1	t1_decimal	2
 mcs229_db	t2	t2_date	2
 mcs229_db	t2	t2_int	2
 mcs229_db	t2	t2_text	2

oam/etc/Columnstore.xml

@@ -478,7 +478,6 @@
 		it cannot be changed!. Extent size is 8M rows.
 		-->
 		<FilesPerColumnPartition>4</FilesPerColumnPartition> <!-- should be multiple of DBRootCount -->
-		<ExtentsPerSegmentFile>2</ExtentsPerSegmentFile>
 		<BRM_UID>0x0</BRM_UID>
 	</ExtentMap>
 	<HashJoin>

oam/etc/Columnstore.xml.singleserver

@@ -471,7 +471,6 @@
 		it cannot be changed!. Valid values for ExtentRows are: 64M, 8M and 1M. The default is 8M.
 		-->
 		<FilesPerColumnPartition>4</FilesPerColumnPartition> <!-- should be multiple of DBRootCount -->
-		<ExtentsPerSegmentFile>2</ExtentsPerSegmentFile>
 		<BRM_UID>0x0</BRM_UID>
 	</ExtentMap>
 	<HashJoin>

tools/rebuildEM/CMakeLists.txt

@@ -1,6 +1,6 @@
 include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(rebuildEM_SRCS main.cpp rebuildEM.cpp)
-add_executable(rebuildEM ${rebuildEM_SRCS})
+add_executable(mcsRebuildEM ${rebuildEM_SRCS})
-target_link_libraries(rebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem)
+target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem boost_system)
-install(TARGETS rebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
+install(TARGETS mcsRebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

tools/rebuildEM/main.cpp

@@ -19,6 +19,7 @@
 #include <string>
 #include <ftw.h>
 #include <boost/filesystem.hpp>
+#include <boost/algorithm/string/case_conv.hpp>
 
 #include "configcpp.h"
 #include "rebuildEM.h"
@@ -84,10 +85,36 @@ int main(int argc, char** argv)
         return 0;
     }
 
+    // MCOL-4685
+    std::cout << "The launch of mcsRebuildEM tool must be sanctioned by MariaDB support. "
+              << std::endl;
+    std::cout << "Requirement: all DBRoots must be on this node. " << std::endl;
+    std::cout << "Note: that the launch can break the cluster." << std::endl;
+    std::cout << "Do you want to continue Y/N? " << std::endl;
+    std::string confirmation;
+    cin >> confirmation;
+    if (confirmation.size() == 0)
+        return 0;
+
+    boost::algorithm::to_lower(confirmation);
+    if (!(confirmation == "y" || confirmation == "yes"))
+        return 0;
+
     auto* config = config::Config::makeConfig();
+
+    // Check for storage type.
+    const auto DBRootStorageType = config->getConfig("Installation", "DBRootStorageType");
+    if (DBRootStorageType != "internal")
+    {
+        std::cout << "Only internal DBRootStorageType is supported, provided: " << DBRootStorageType
+                  << std::endl;
+        return 0;
+    }
+
     const auto BRMSavesEM =
         config->getConfig("SystemConfig", "DBRMRoot") + "_em";
     // Check for `BRM_saves_em` file presents.
+    // TODO: Should we add force option to remove file?
     if (boost::filesystem::exists(BRMSavesEM))
     {
         std::cout << BRMSavesEM << " file exists. " << std::endl;

tools/rebuildEM/rebuildEM.cpp

@@ -132,7 +132,7 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
     auto colDataType = compressor.getColDataType(fileHeader);
     auto colWidth = compressor.getColumnWidth(fileHeader);
     auto blockCount = compressor.getBlockCount(fileHeader);
-    auto lbid = compressor.getLBID0(fileHeader);
+    auto lbidCount = compressor.getLBIDCount(fileHeader);
 
     if (colDataType == execplan::CalpontSystemCatalog::UNDEFINED)
     {
@@ -146,7 +146,6 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
     auto isDict = isDictFile(colDataType, colWidth);
     if (isDict)
         colWidth = 8;
-    uint64_t hwm = 0;
 
     if (doVerbose())
     {
@@ -154,48 +153,55 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
         std::cout << "Block count: " << blockCount << std::endl;
     }
 
-    rc =
+    uint64_t hwm = 0;
-        searchHWMInSegmentFile(oid, getDBRoot(), partition, segment,
+    rc = searchHWMInSegmentFile(oid, getDBRoot(), partition, segment, colDataType, colWidth,
-                               colDataType, colWidth, blockCount, isDict, hwm);
+                                blockCount, isDict, hwm);
     if (rc != 0)
     {
         return rc;
     }
 
-    const uint32_t extentMaxBlockCount =
-        getEM().getExtentRows() * colWidth / BLOCK_SIZE;
-
-    // We found multiple extents per one segment file.
-    if (hwm >= extentMaxBlockCount)
-    {
-        auto lbid = compressor.getLBID1(fileHeader);
-        FileId fileId(oid, partition, segment, colWidth, colDataType, lbid,
-                      hwm, isDict);
-        extentMap.push_back(fileId);
-
-        // Update HWM.
-        hwm = extentMaxBlockCount - 1;
-        if (doVerbose())
-        {
-            std::cout << "Found multiple extents per segment file "
-                      << std::endl;
-            std::cout << "FileId is collected " << fileId << std::endl;
-        }
-    }
-
     if (doVerbose())
     {
         std::cout << "HWM is: " << hwm << std::endl;
     }
 
-    FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm,
+    const uint32_t extentMaxBlockCount = getEM().getExtentRows() * colWidth / BLOCK_SIZE;
-                  isDict);
+    // We found multiple extents per one segment file.
+    if (hwm >= extentMaxBlockCount)
+    {
+        for (uint32_t lbidIndex = 0; lbidIndex < lbidCount - 1; ++lbidIndex)
+        {
+            auto lbid = compressor.getLBIDByIndex(fileHeader, lbidIndex);
+            FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, /*hwm*/ 0, isDict);
+            extentMap.push_back(fileId);
+        }
+
+        // Last one has an actual HWM.
+        auto lbid = compressor.getLBIDByIndex(fileHeader, lbidCount - 1);
+        FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict);
+        extentMap.push_back(fileId);
+
+        if (doVerbose())
+        {
+            std::cout << "Found multiple extents per segment file "
+                      << std::endl;
+            std::cout << "FileId is collected " << fileId << std::endl;
+        }
+    }
+    else
+    {
+        // One extent per segment file.
+        auto lbid = compressor.getLBIDByIndex(fileHeader, 0);
+        FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict);
         extentMap.push_back(fileId);
 
         if (doVerbose())
         {
             std::cout << "FileId is collected " << fileId << std::endl;
         }
+    }
+
     return 0;
 }
 

utils/compress/idbcompress.cpp

@@ -63,6 +63,9 @@ const uint8_t CHUNK_MAGIC2 = 0xfe;
  */
 const uint8_t CHUNK_MAGIC3 = 0xfd;
 
+// The max number of lbids to be stored in segment file.
+const uint32_t LBID_MAX_SIZE = 10;
+
 struct CompressedDBFileHeader
 {
     uint64_t fMagicNumber;
@@ -72,8 +75,8 @@ struct CompressedDBFileHeader
     uint64_t fBlockCount;
     uint64_t fColumnWidth;
     execplan::CalpontSystemCatalog::ColDataType fColDataType;
-    uint64_t fLBID0;
+    uint64_t fLBIDCount;
-    uint64_t fLBID1;
+    uint64_t fLBIDS[LBID_MAX_SIZE];
 };
 
 // Make the header to be 4K, regardless number of fields being defined/used in header.
@@ -96,8 +99,8 @@ void initCompressedDBFileHeader(
     hdr->fHeader.fHeaderSize      = hdrSize;
     hdr->fHeader.fColumnWidth     = columnWidth;
     hdr->fHeader.fColDataType     = colDataType;
-    hdr->fHeader.fLBID0           = 0;
+    hdr->fHeader.fLBIDCount       = 0;
-    hdr->fHeader.fLBID1           = 0;
+    std::memset(hdr->fHeader.fLBIDS, 0, sizeof(hdr->fHeader.fLBIDS));
 }
 
 } // namespace
@@ -432,37 +435,31 @@ uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const
 }
 
 //------------------------------------------------------------------------------
-// Get start LBID
+// Get LBID by index
 //------------------------------------------------------------------------------
-uint64_t IDBCompressInterface::getLBID0(const void* hdrBuf) const
+uint64_t IDBCompressInterface::getLBIDByIndex(const void* hdrBuf, uint64_t index) const
 {
-    return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID0);
+    if (index < LBID_MAX_SIZE)
+        return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBIDS[index]);
+    return 0;
 }
 
 //------------------------------------------------------------------------------
-// Set start LBID
+// Set LBID by index
 //------------------------------------------------------------------------------
-void IDBCompressInterface::setLBID0(void* hdrBuf, uint64_t lbid) const
+void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index) const
 {
-    if (lbid)
+    if (lbid && index < LBID_MAX_SIZE)
-        reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID0 = lbid;
+    {
+        reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDS[index] = lbid;
+        reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDCount =
+            std::max(index + 1, reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDCount);
+    }
 }
 
-//------------------------------------------------------------------------------
+uint64_t IDBCompressInterface::getLBIDCount(void* hdrBuf) const
-// Get start LBID
-//------------------------------------------------------------------------------
-uint64_t IDBCompressInterface::getLBID1(const void* hdrBuf) const
 {
-    return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID1);
+    return reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBIDCount;
-}
-
-//------------------------------------------------------------------------------
-// Set start LBID
-//------------------------------------------------------------------------------
-void IDBCompressInterface::setLBID1(void* hdrBuf, uint64_t lbid) const
-{
-    if (lbid)
-        reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID1 = lbid;
 }
 
 //------------------------------------------------------------------------------

utils/compress/idbcompress.h

@@ -234,24 +234,19 @@ public:
     EXPORT uint64_t getColumnWidth(const void* hdrBuf) const;
 
     /**
-     * getLBID
+     * getLBIDByIndex
      */
-    EXPORT uint64_t getLBID0(const void* hdrBuf) const;
+    EXPORT uint64_t getLBIDByIndex(const void* hdrBuf, uint64_t index) const;
 
     /**
-     * setBID
+     * setLBIDByIndex
      */
-    EXPORT void setLBID0(void* hdrBuf, uint64_t lbid) const;
+    EXPORT void setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index) const;
 
     /**
-     * getLBID
+     * getLBIDCount
      */
-    EXPORT uint64_t getLBID1(const void* hdrBuf) const;
+    EXPORT uint64_t getLBIDCount(void* hdrBuf) const;
-
-    /**
-     * setBID
-     */
-    EXPORT void setLBID1(void* hdrBuf, uint64_t lbid) const;
 
     /**
      * Mutator methods for the user padding bytes
@@ -367,15 +362,17 @@ IDBCompressInterface::getColDataType(const void* hdrBuf) const
 {
     return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
 }
-inline uint64_t getColumnWidth(const void* hdrBuf) const { return 0; }
+inline uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const { return 0; }
 inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize)
 {
     return uncompSize;
 }
-inline uint64_t getLBID0(const void* hdrBuf) const { return 0; }
+inline uint64_t IDBCompressInterface::getLBIDByIndex(const void* hdrBuf, uint32_t index) const
-void setLBID0(void* hdrBuf, uint64_t lbid) const {}
+{
-inline uint64_t getLBID1(const void* hdrBuf) const { return 0; }
+    return 0;
-void setLBID1(void* hdrBuf, uint64_t lbid) const {}
+}
+void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint32_t index) const {}
+void IDBCompressInterface::getLBIDCount(void* hdrBuf) const {}
 inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen)
 {
     return false;

versioning/BRM/extentmap.cpp

@@ -59,6 +59,7 @@ namespace bi = boost::interprocess;
 #include "oamcache.h"
 #include "IDBDataFile.h"
 #include "IDBPolicy.h"
+#include "columncommand-jl.h"
 #ifdef BRM_INFO
 #include "tracer.h"
 #include "configcpp.h"
@@ -96,7 +97,7 @@ namespace
 unsigned ExtentSize = 0; // dmc-need to deprecate
 unsigned ExtentRows              = 0;
 unsigned filesPerColumnPartition = 0;
-unsigned extentsPerSegmentFile   = 0;
+unsigned extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE;
 
 // Increment CP sequence (version) number, and wrap-around when applicable
 inline void incSeqNum(int32_t& seqNum)
@@ -6033,11 +6034,8 @@ void ExtentMap::checkReloadConfig()
     //--------------------------------------------------------------------------
     // Initialize extents per segment file
     //--------------------------------------------------------------------------
-    string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
+    // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
-    extentsPerSegmentFile = cf->uFromText(epsf);
+    extentsPerSegmentFile = joblist::ColumnCommandJL::DEFAULT_EXTENTS_PER_SEGMENT_FILE;
-
-    if (extentsPerSegmentFile == 0)
-        extentsPerSegmentFile = 2;
 }
 
 //------------------------------------------------------------------------------

writeengine/bulk/we_colbufcompressed.cpp

@@ -584,21 +584,18 @@ int ColumnBufferCompressed::saveCompressionHeaders( )
     char hdrBuf[IDBCompressInterface::HDR_BUF_LEN * 2];
     RETURN_ON_ERROR(fColInfo->colOp->readHeaders(fFile, hdrBuf));
 
-    auto lbid = fCompressor->getLBID0(hdrBuf);
+    auto lbid = fCompressor->getLBIDByIndex(hdrBuf, 0);
     fCompressor->initHdr(hdrBuf, fColInfo->column.width,
                          fColInfo->column.dataType,
                          fColInfo->column.compressionType);
     fCompressor->setBlockCount(hdrBuf,
                                (fColInfo->getFileSize() / BYTE_PER_BLOCK) );
+    // If lbid written in the header is not 0 - we are running for the next extent for column
+    // segment file.
     if (lbid)
-    {
+        fCompressor->setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 1);
-        fCompressor->setLBID0(hdrBuf, lbid);
-        fCompressor->setLBID1(hdrBuf, fColInfo->getLastUpdatedLBID());
-    }
     else
-    {
+        fCompressor->setLBIDByIndex(hdrBuf, fColInfo->getLastUpdatedLBID(), 0);
-        fCompressor->setLBID0(hdrBuf, fColInfo->getLastUpdatedLBID());
-    }
 
     std::vector<uint64_t> ptrs;
 

writeengine/server/we_ddlcommandproc.cpp

@@ -59,10 +59,8 @@ WE_DDLCommandProc::WE_DDLCommandProc()
     if (fpc.length() != 0)
         filesPerColumnPartition = cf->uFromText(fpc);
 
-    string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
+    // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
-
+    extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
-    if (epsf.length() != 0)
-        extentsPerSegmentFile = cf->uFromText(epsf);
 
     string dbct = cf->getConfig("SystemConfig", "DBRootCount");
 

writeengine/server/we_ddlcommandproc.h

@@ -112,7 +112,7 @@ private:
     WriteEngineWrapper fWEWrapper;
     BRM::DBRM fDbrm;
     unsigned  extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
-
+    static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
 };
 }
 #undef EXPORT

writeengine/server/we_dmlcommandproc.cpp

@@ -70,10 +70,8 @@ WE_DMLCommandProc::WE_DMLCommandProc()
     if (fpc.length() != 0)
         filesPerColumnPartition = cf->uFromText(fpc);
 
-    string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
+    // MCOL-4685: remove the option to set more than 2 extents per file (ExtentsPreSegmentFile).
-
+    extentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
-    if (epsf.length() != 0)
-        extentsPerSegmentFile = cf->uFromText(epsf);
 
     string dbct = cf->getConfig("SystemConfig", "DBRootCount");
 

writeengine/server/we_dmlcommandproc.h

@@ -132,6 +132,7 @@ private:
     BRM::DBRM fDbrm;
     unsigned  extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
     Log fLog;
+    static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
 };
 
 }

writeengine/shared/we_chunkmanager.cpp

@@ -466,7 +466,7 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
                         fileData->fFileHeader.fPtrSection,
                         /*colWidth=*/0, fileData->fColDataType,
                         fFileOp->compressionType(), hdrSize);
-    fCompressor.setLBID0(fileData->fFileHeader.fControlData, lbid);
+    fCompressor.setLBIDByIndex(fileData->fFileHeader.fControlData, lbid, 0);
 
     if (writeHeader(fileData, __LINE__) != NO_ERROR)
     {
@@ -1423,7 +1423,7 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int6
     int rc = NO_ERROR;
     char* hdr = pFileData->fFileHeader.fControlData;
     fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
-    fCompressor.setLBID1(hdr, lbid);
+    fCompressor.setLBIDByIndex(hdr, lbid, 1);
     ChunkData* chunkData = (pFileData)->findChunk(0);
 
     if (chunkData != NULL)
@@ -1514,7 +1514,11 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount,
         fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
 
     if (currentBlockCount)
-        fCompressor.setLBID1(hdr, lbid);
+    {
+        // Append to the end.
+        uint64_t lbidCount = fCompressor.getLBIDCount(hdr);
+        fCompressor.setLBIDByIndex(hdr, lbid, lbidCount);
+    }
     return rc;
 }
 

writeengine/shared/we_config.cpp

@@ -153,10 +153,6 @@ void Config::checkReload( )
     // Initialize extents per segment file
     //--------------------------------------------------------------------------
     m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
-    string epsf = cf->getConfig("ExtentMap", "ExtentsPerSegmentFile");
-
-    if ( epsf.length() != 0 )
-        m_ExtentsPerSegmentFile = cf->uFromText(epsf);
 
     //--------------------------------------------------------------------------
     // Initialize bulk load process priority

writeengine/shared/we_fileop.cpp

@@ -818,7 +818,7 @@ int FileOp::extendFile(
         {
             IDBCompressInterface compressor;
             compressor.initHdr(hdrs, width, colDataType, m_compressionType);
-            compressor.setLBID0(hdrs, startLbid);
+            compressor.setLBIDByIndex(hdrs, startLbid, 0);
         }
     }
 
@@ -978,7 +978,7 @@ int FileOp::addExtentExactFile(
         {
             IDBCompressInterface compressor;
             compressor.initHdr(hdrs, width, colDataType, m_compressionType);
-            compressor.setLBID0(hdrs, startLbid);
+            compressor.setLBIDByIndex(hdrs, startLbid, 0);
         }
     }
 
@@ -1067,7 +1067,7 @@ int FileOp::initColumnExtent(
         char hdrs[IDBCompressInterface::HDR_BUF_LEN * 2];
         IDBCompressInterface compressor;
         compressor.initHdr(hdrs, width, colDataType, m_compressionType);
-        compressor.setLBID0(hdrs, lbid);
+        compressor.setLBIDByIndex(hdrs, lbid, 0);
 
         if (bAbbrevExtent)
             compressor.setBlockCount(hdrs, nBlocks);
@@ -1349,7 +1349,7 @@ int FileOp::writeInitialCompColumnChunk(
 
     compressor.initHdr(hdrs, width, colDataType, m_compressionType);
     compressor.setBlockCount(hdrs, nBlocksAllocated);
-    compressor.setLBID0(hdrs, startLBID);
+    compressor.setLBIDByIndex(hdrs, startLBID, 0);
 
     // Store compression pointers in the header
     std::vector<uint64_t> ptrs;