MCOL-4912 This patch introduces Extent Map index to improve EM scaleability

EM scaleability project has two parts: phase1 and phase2.
        This is phase1 that brings EM index to speed up(from O(n) down
        to the speed of boost::unordered_map) EM lookups looking for
        <dbroot, oid, partition> tuple to turn it into LBID,
        e.g. most bulk insertion meta info operations.
        The basis is boost::shared_managed_object where EMIndex is
        stored. Whilst it is not debug-friendly it allows to put a
        nested structs into shmem. EMIndex has 3 tiers. Top down description:
        vector of dbroots, map of oids to partition vectors, partition
        vectors that have EM indices.
        Separate EM methods now queries index before they do EM run.
        EMIndex has a separate shmem file with the fixed id
        MCS-shm-00060001.

dbcon/mysql/ha_mcs_client_udfs.cpp

@@ -1,5 +1,5 @@
 /* Copyright (C) 2014 InfiniDB, Inc.
-   Copyright (C) 2016 MariaDB Corporation
+   Copyright (C) 2016-2022 MariaDB Corporation
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
@@ -52,6 +52,8 @@ extern "C"
   const char* SetParmsPrelude = "Updated ";
   const char* SetParmsError = "Invalid parameter: ";
   const char* InvalidParmSize = "Invalid parameter size: Input value cannot be larger than ";
+  const char* MsgEMIndexSizeInitErrMsg = "mcs_emindex_size() takes no arguments";
+  const char* MsgEMIndexFreeInitErrMsg = "mcs_emindex_free() takes no arguments";
 
   const size_t Plen = strlen(SetParmsPrelude);
   const size_t Elen = strlen(SetParmsError);
@@ -1156,4 +1158,64 @@ extern "C"
   {
   }
 
+#ifdef _MSC_VER
+  __declspec(dllexport)
+#endif
+      long long mcs_emindex_size(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error)
+  {
+    DBRM dbrm;
+    return dbrm.EMIndexShmemSize();
+  }
+
+#ifdef _MSC_VER
+  __declspec(dllexport)
+#endif
+      my_bool mcs_emindex_size_init(UDF_INIT* initid, UDF_ARGS* args, char* message)
+  {
+    if (args->arg_count != 0)
+    {
+      strcpy(message, MsgEMIndexSizeInitErrMsg);
+      return 1;
+    }
+
+    return 0;
+  }
+
+#ifdef _MSC_VER
+  __declspec(dllexport)
+#endif
+      void mcs_emindex_size_deinit(UDF_INIT* initid)
+  {
+  }
+
+#ifdef _MSC_VER
+  __declspec(dllexport)
+#endif
+      long long mcs_emindex_free(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error)
+  {
+    DBRM dbrm;
+    return dbrm.EMIndexShmemFree();
+  }
+
+#ifdef _MSC_VER
+  __declspec(dllexport)
+#endif
+      my_bool mcs_emindex_free_init(UDF_INIT* initid, UDF_ARGS* args, char* message)
+  {
+    if (args->arg_count != 0)
+    {
+      strcpy(message, MsgEMIndexFreeInitErrMsg);
+      return 1;
+    }
+
+    return 0;
+  }
+
+#ifdef _MSC_VER
+  __declspec(dllexport)
+#endif
+      void mcs_emindex_free_deinit(UDF_INIT* initid)
+  {
+  }
+
 }  // extern "C"

tests/CMakeLists.txt

@@ -62,6 +62,11 @@ if (WITH_UNITTESTS)
     target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
     add_test(NAME columnstore:comparators_tests, COMMAND comparators_tests)
 
+    # standalone EM routines test
+    # add_executable(brm_em_standalone brm-em-standalone.cpp)
+    # target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    # install(TARGETS brm_em_standalone DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
+
 endif()
 
 # Saving this as the example of the microbench

tools/clearShm/main.cpp

@@ -60,7 +60,7 @@ void shmDoit(key_t shm_key, const string& label)
       bi::offset_t memSize = 0;
       memObj.get_size(memSize);
       std::lock_guard<std::mutex> lk(coutMutex);
-      cout << label << ": shm_key: " << shm_key << "; key_name: " << key_name << "; size: " << memSize
+      cout << label << ": shm|sem_key: " << shm_key << "; key_name: " << key_name << "; size: " << memSize
            << endl;
     }
     catch (...)
@@ -74,6 +74,11 @@ void shmDoit(key_t shm_key, const string& label)
   }
 }
 
+void semDoit(key_t sem_key, const string& label)
+{
+  shmDoit(sem_key, label);
+}
+
 void shmDoitRange(key_t shm_key, const string& label)
 {
   if (shm_key == 0)
@@ -87,32 +92,6 @@ void shmDoitRange(key_t shm_key, const string& label)
   }
 }
 
-void semDoit(key_t sem_key, const string& label)
-{
-  string key_name = ShmKeys::keyToName(sem_key);
-
-  if (vFlg)
-  {
-    try
-    {
-      bi::shared_memory_object memObj(bi::open_only, key_name.c_str(), bi::read_only);
-      bi::offset_t memSize = 0;
-      memObj.get_size(memSize);
-      std::lock_guard<std::mutex> lk(coutMutex);
-      cout << label << ": sem_key: " << sem_key << "; key_name: " << key_name << "; size: " << memSize
-           << endl;
-    }
-    catch (...)
-    {
-    }
-  }
-
-  if (!nFlg)
-  {
-    bi::shared_memory_object::remove(key_name.c_str());
-  }
-}
-
 void usage()
 {
   cout << "usage: clearShm [-cvnh]" << endl;
@@ -206,6 +185,8 @@ int main(int argc, char** argv)
   tg.add_thread(tp);
   tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_VSS_BASE, "VSS        "));
   tg.add_thread(tp);
+  tp = new boost::thread(ThdFunc(BrmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE, "EXTMAP_INDX"));
+  tg.add_thread(tp);
   tg.join_all();
 
   shmDoit(BrmKeys.MST_SYSVKEY, "MST        ");
@@ -226,6 +207,7 @@ int main(int argc, char** argv)
   semDoit(BrmKeys.KEYRANGE_EMFREELIST_BASE, "EXTMAP_FREE");
   semDoit(BrmKeys.KEYRANGE_VBBM_BASE, "VBBM       ");
   semDoit(BrmKeys.KEYRANGE_VSS_BASE, "VSS        ");
+  semDoit(BrmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE, "EXTMAP_INDX");
   semDoit(BrmKeys.MST_SYSVKEY, "MST        ");
 
   if (!cFlg)

utils/rwlock/rwlock.h

@@ -1,4 +1,5 @@
 /* Copyright (C) 2014 InfiniDB, Inc.
+   Copyright (C) 2016-2022 MariaDB Corporation
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
@@ -26,6 +27,8 @@
 
 #pragma once
 
+#include <array>
+
 #include <unistd.h>
 #include <stdexcept>
 
@@ -42,6 +45,10 @@
 
 namespace rwlock
 {
+const std::array<const std::string, 7> RWLockNames = {
+    "all", "VSS", "ExtentMap", "FreeList", "VBBM", "CopyLocks", "ExtentMapIndex",
+};
+
 /// the layout of the shmseg
 struct State
 {

versioning/BRM/brmshmimpl.cpp

@@ -40,7 +40,15 @@ namespace bi = boost::interprocess;
 
 namespace BRM
 {
-BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : fKey(key), fSize(size), fReadOnly(readOnly)
+const constexpr uint32_t ShmCreateMaxRetries = 10;
+const constexpr unsigned int NapTimer = 500000;
+
+BRMShmImplParent::BRMShmImplParent(unsigned key, off_t size, bool readOnly)
+ : fKey(key), fSize(size), fReadOnly(readOnly){};
+
+BRMShmImplParent::~BRMShmImplParent(){};
+
+BRMShmImpl::BRMShmImpl(unsigned key, off_t size, bool readOnly) : BRMShmImplParent(key, size, readOnly)
 {
   string keyName = ShmKeys::keyToName(fKey);
 
@@ -238,5 +246,167 @@ void BRMShmImpl::destroy()
     bi::shared_memory_object::remove(oldName.c_str());
 }
 
+BRMManagedShmImpl::BRMManagedShmImpl(unsigned key, off_t size, bool readOnly)
+ : BRMShmImplParent(key, size, readOnly)
+{
+  string keyName = ShmKeys::keyToName(fKey);
+  off_t curSize = 0;
+
+  for (uint32_t tries = 0; fSize == 0 && tries <= ShmCreateMaxRetries; ++tries)
+  {
+    try
+    {
+      auto* shmSegment = new boost::interprocess::managed_shared_memory(bi::open_only, keyName.c_str());
+      curSize = shmSegment->get_size();
+
+      if (curSize == 0)
+      {
+        delete shmSegment;
+        throw bi::interprocess_exception("shared memory segment size is 0.");
+      }
+      else
+      {
+        fShmSegment = shmSegment;
+        fSize = curSize;
+        return;
+      }
+    }
+    catch (bi::interprocess_exception&)
+    {
+      if (tries == ShmCreateMaxRetries)
+      {
+        log("BRMManagedShmImpl::BRMManagedShmImpl(): re-creating shared memory segment\
+ b/c of its size == 0. Re-throw.");
+        throw;
+      }
+
+      cerr << "BRMManagedShmImpl::BRMManagedShmImpl(): re-creating shared memory segment\
+ b/c of its size == 0"
+           << endl;
+      usleep(NapTimer);
+    }
+  }
+
+  try
+  {
+    bi::permissions perms;
+    perms.set_unrestricted();
+    fShmSegment = new bi::managed_shared_memory(bi::create_only, keyName.c_str(), fSize,
+                                                0,  // use a default address to map the segment
+                                                perms);
+    // fSize == 0 on any process startup but managed_shared_memory ctor throws
+    // so control flow doesn't get here.
+    idbassert(fSize > 0);
+  }
+  catch (bi::interprocess_exception& b)
+  {
+    if (b.get_error_code() != bi::already_exists_error)
+    {
+      ostringstream o;
+      o << "BRM caught an exception creating a shared memory segment: " << b.what();
+      log(o.str());
+      throw;
+    }
+    bi::managed_shared_memory* shmSegment = nullptr;
+    try
+    {
+      if (fReadOnly)
+        shmSegment = new bi::managed_shared_memory(bi::open_read_only, keyName.c_str());
+      else
+        shmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str());
+    }
+    catch (exception& e)
+    {
+      ostringstream o;
+      o << "BRM caught an exception attaching to a shared memory segment (" << keyName << "): " << b.what();
+      log(o.str());
+      throw;
+    }
+    off_t curSize = shmSegment->get_size();
+
+    idbassert(curSize > 0);
+    idbassert(curSize >= fSize);
+    fShmSegment = shmSegment;
+    fSize = curSize;
+  }
+}
+
+int BRMManagedShmImpl::grow(off_t newSize)
+{
+  auto keyName = ShmKeys::keyToName(fKey);
+
+  if (newSize > fSize)
+  {
+    const auto incSize = newSize - fSize;
+    if (fShmSegment)
+    {
+      // Call destructor to unmap the segment.
+      delete fShmSegment;
+      // Grow the segment.
+      bi::managed_shared_memory::grow(keyName.c_str(), incSize);
+      // Open only with the assumption ::grow() can be called on read-write shmem.
+      fShmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str());
+      // Update size.
+      fSize = newSize;
+    }
+  }
+
+  return 0;
+}
+
+// Dummy method that has no references in the code.
+int BRMManagedShmImpl::clear(unsigned newKey, off_t newSize)
+{
+  return 0;
+}
+
+// This method calls for all related shmem pointers to be refreshed.
+void BRMManagedShmImpl::setReadOnly()
+{
+  if (fReadOnly)
+    return;
+  const bool readOnly = true;
+  remap(readOnly);
+  fReadOnly = true;
+}
+
+void BRMManagedShmImpl::swap(BRMManagedShmImpl& rhs)
+{
+  fShmSegment->swap(*rhs.fShmSegment);
+  std::swap(fKey, rhs.fKey);
+  std::swap(fSize, rhs.fSize);
+  std::swap(fReadOnly, rhs.fReadOnly);
+}
+
+// The method was copied from non-managed shmem impl class
+// and it has no refences in MCS 6.x code.
+void BRMManagedShmImpl::destroy()
+{
+  string keyName = ShmKeys::keyToName(fKey);
+  try
+  {
+    bi::shared_memory_object::remove(keyName.c_str());
+  }
+  catch (bi::interprocess_exception& b)
+  {
+    std::ostringstream o;
+    o << "BRMManagedShmImpl::destroy caught an exception removing a managed shared memory segment: "
+      << b.what();
+    log(o.str());
+    throw;
+  }
+}
+
+void BRMManagedShmImpl::remap(const bool readOnly)
+{
+  delete fShmSegment;
+  fShmSegment = nullptr;
+  string keyName = ShmKeys::keyToName(fKey);
+  if (readOnly)
+    fShmSegment = new bi::managed_shared_memory(bi::open_read_only, keyName.c_str());
+  else
+    fShmSegment = new bi::managed_shared_memory(bi::open_only, keyName.c_str());
+}
+
 }  // namespace BRM
 

versioning/BRM/brmshmimpl.h

@@ -30,17 +30,18 @@
 //#define NDEBUG
 #include <cassert>
 #include <boost/interprocess/shared_memory_object.hpp>
+#include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/mapped_region.hpp>
 
+namespace bi = boost::interprocess;
+
 namespace BRM
 {
-class BRMShmImpl
+class BRMShmImplParent
 {
  public:
-  BRMShmImpl(unsigned key, off_t size, bool readOnly = false);
+  BRMShmImplParent(unsigned key, off_t size, bool readOnly = false);
-  ~BRMShmImpl()
+  virtual ~BRMShmImplParent();
-  {
-  }
 
   inline unsigned key() const
   {
@@ -55,23 +56,63 @@ class BRMShmImpl
     return fReadOnly;
   }
 
-  void setReadOnly();
+  virtual void setReadOnly() = 0;
-  int grow(unsigned newKey, off_t newSize);
+  virtual int clear(unsigned newKey, off_t newSize) = 0;
-  int clear(unsigned newKey, off_t newSize);
+  virtual void destroy() = 0;
-
-  void swap(BRMShmImpl& rhs);
-  void destroy();
-
-  boost::interprocess::shared_memory_object fShmobj;
-  boost::interprocess::mapped_region fMapreg;
-
- private:
-  BRMShmImpl(const BRMShmImpl& rhs);
-  BRMShmImpl& operator=(const BRMShmImpl& rhs);
 
+ protected:
   unsigned fKey;
   off_t fSize;
   bool fReadOnly;
 };
 
+class BRMShmImpl : public BRMShmImplParent
+{
+ public:
+  BRMShmImpl(unsigned key, off_t size, bool readOnly = false);
+  BRMShmImpl(const BRMShmImpl& rhs) = delete;
+  BRMShmImpl& operator=(const BRMShmImpl& rhs) = delete;
+  ~BRMShmImpl()
+  {
+  }
+
+  int clear(unsigned newKey, off_t newSize) override;
+  void destroy() override;
+  void setReadOnly() override;
+
+  int grow(unsigned newKey, off_t newSize);
+  void swap(BRMShmImpl& rhs);
+
+  bi::shared_memory_object fShmobj;
+  bi::mapped_region fMapreg;
+};
+
+class BRMManagedShmImpl : public BRMShmImplParent
+{
+ public:
+  BRMManagedShmImpl(unsigned key, off_t size, bool readOnly = false);
+  BRMManagedShmImpl(const BRMManagedShmImpl& rhs) = delete;
+  BRMManagedShmImpl& operator=(const BRMManagedShmImpl& rhs) = delete;
+  ~BRMManagedShmImpl()
+  {
+    delete fShmSegment;
+  }
+
+  int clear(unsigned newKey, off_t newSize) override;
+  void destroy() override;
+  void setReadOnly() override;
+
+  int grow(off_t newSize);
+  void remap(const bool readOnly = false);
+  void swap(BRMManagedShmImpl& rhs);
+  bi::managed_shared_memory* getManagedSegment()
+  {
+    assert(fShmSegment);
+    return fShmSegment;
+  }
+
+ private:
+  bi::managed_shared_memory* fShmSegment;
+};
+
 }  // namespace BRM

versioning/BRM/dbrm.cpp

@@ -98,7 +98,7 @@ DBRM::DBRM(const DBRM& brm)
   throw logic_error("DBRM: Don't use the copy constructor.");
 }
 
-DBRM::~DBRM() throw()
+DBRM::~DBRM()
 {
   if (msgClient != NULL)
     MessageQueueClientPool::releaseInstance(msgClient);
@@ -461,7 +461,7 @@ int DBRM::markExtentsInvalid(const vector<LBID_t>& lbids,
 }
 
 template <typename T>
-int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw()
+int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum)
 {
 #ifdef BRM_INFO
 
@@ -489,7 +489,7 @@ int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) th
   }
 }
 
-int DBRM::getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw()
+int DBRM::getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin)
 {
   try
   {
@@ -4555,10 +4555,19 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN
   setExtentsMaxMin(cpInfos);
 }
 
-template int DBRM::getExtentMaxMin<int128_t>(const LBID_t lbid, int128_t& max, int128_t& min,
+size_t DBRM::EMIndexShmemSize()
-                                             int32_t& seqNum) throw();
+{
+  return em->EMIndexShmemSize();
+}
 
-template int DBRM::getExtentMaxMin<int64_t>(const LBID_t lbid, int64_t& max, int64_t& min,
+size_t DBRM::EMIndexShmemFree()
-                                            int32_t& seqNum) throw();
+{
+  return em->EMIndexShmemFree();
+}
+
+template int DBRM::getExtentMaxMin<int128_t>(const LBID_t lbid, int128_t& max, int128_t& min,
+                                             int32_t& seqNum);
+
+template int DBRM::getExtentMaxMin<int64_t>(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum);
 
 }  // namespace BRM

versioning/BRM/dbrm.h

@@ -103,7 +103,7 @@ class DBRM
   // The param noBRMFcns suppresses init of the ExtentMap, VSS, VBBM, and CopyLocks.
   // It can speed up init if the caller only needs the other structures.
   EXPORT DBRM(bool noBRMFcns = false);
-  EXPORT ~DBRM() throw();
+  EXPORT ~DBRM();
 
   EXPORT static void refreshShm()
   {
@@ -780,12 +780,12 @@ class DBRM
                                 const std::vector<execplan::CalpontSystemCatalog::ColDataType>& colDataTypes)
       DBRM_THROW;
   template <typename T>
-  EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw();
+  EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum);
 
   EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min,
                              const int32_t seqNum) DBRM_THROW;
 
-  EXPORT int getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin) throw();
+  EXPORT int getExtentCPMaxMin(const LBID_t lbid, CPMaxMin& cpMaxMin);
 
   /** @brief Updates the max and min casual partitioning info for the passed extents.
    *
@@ -984,6 +984,9 @@ class DBRM
   EXPORT void invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN txnid, bool allExtents,
                                                std::vector<LBID_t>* plbidList = NULL);
 
+  size_t EMIndexShmemSize();
+  size_t EMIndexShmemFree();
+
  private:
   DBRM(const DBRM& brm);
   DBRM& operator=(const DBRM& brm);

versioning/BRM/extentmap.h

@@ -1,4 +1,5 @@
 /* Copyright (C) 2014 InfiniDB, Inc.
+   Copyright (C) 2016-2022 MariaDB Corporation
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
@@ -29,15 +30,18 @@
 #include <sys/types.h>
 #include <vector>
 #include <set>
-#ifdef _MSC_VER
 #include <unordered_map>
-#else
 #include <tr1/unordered_map>
-#endif
+#include <mutex>
+
 //#define NDEBUG
 #include <cassert>
 #include <boost/interprocess/shared_memory_object.hpp>
 #include <boost/interprocess/mapped_region.hpp>
+#include <boost/interprocess/managed_shared_memory.hpp>
+#include <boost/interprocess/allocators/allocator.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/functional/hash.hpp>  //boost::hash
 
 #include "shmkeys.h"
 #include "brmtypes.h"
@@ -63,6 +67,8 @@
 #define EXPORT
 #endif
 
+namespace bi = boost::interprocess;
+
 namespace oam
 {
 typedef std::vector<uint16_t> DBRootConfigList;
@@ -75,6 +81,15 @@ class IDBDataFile;
 
 namespace BRM
 {
+using PartitionNumberT = uint32_t;
+using DBRootT = uint16_t;
+using SegmentT = uint16_t;
+using LastExtentIndexT = int;
+using EmptyEMEntry = int;
+using HighestOffset = uint32_t;
+using LastIndEmptyIndEmptyInd = std::pair<LastExtentIndexT, EmptyEMEntry>;
+using DBRootVec = std::vector<DBRootT>;
+
 // assumed column width when calculating dictionary store extent size
 #define DICT_COL_WIDTH 8
 
@@ -98,8 +113,6 @@ const char CP_INVALID = 0;
 const char CP_UPDATING = 1;
 const char CP_VALID = 2;
 
-// The _v4 structs are defined below for upgrading extent map
-// from v4 to v5; see ExtentMap::loadVersion4or5 for details.
 struct EMCasualPartition_struct_v4
 {
   RangePartitionData_t hi_val;  // This needs to be reinterpreted as unsigned for uint64_t column types.
@@ -113,16 +126,15 @@ struct EMPartition_struct_v4
 {
   EMCasualPartition_struct_v4 cprange;
 };
-
 struct EMEntry_v4
 {
   InlineLBIDRange range;
   int fileID;
   uint32_t blockOffset;
   HWM_t HWM;
-  uint32_t partitionNum;  // starts at 0
+  PartitionNumberT partitionNum;  // starts at 0
   uint16_t segmentNum;            // starts at 0
-  uint16_t dbRoot;        // starts at 1 to match Columnstore.xml
+  DBRootT dbRoot;                 // starts at 1 to match Columnstore.xml
   uint16_t colWid;
   int16_t status;  // extent avail for query or not, or out of service
   EMPartition_struct_v4 partition;
@@ -151,7 +163,7 @@ struct EMCasualPartition_struct
   EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em);
   EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em);
 };
-typedef EMCasualPartition_struct EMCasualPartition_t;
+using EMCasualPartition_t = EMCasualPartition_struct;
 
 struct EMPartition_struct
 {
@@ -165,9 +177,9 @@ struct EMEntry
   int fileID;
   uint32_t blockOffset;
   HWM_t HWM;
-  uint32_t partitionNum;  // starts at 0
+  PartitionNumberT partitionNum;  // starts at 0
   uint16_t segmentNum;            // starts at 0
-  uint16_t dbRoot;        // starts at 1 to match Columnstore.xml
+  DBRootT dbRoot;                 // starts at 1 to match Columnstore.xml
   uint16_t colWid;
   int16_t status;  // extent avail for query or not, or out of service
   EMPartition_t partition;
@@ -319,6 +331,146 @@ class FreeListImpl
   static FreeListImpl* fInstance;
 };
 
+using ShmSegmentManagerT = bi::managed_shared_memory::segment_manager;
+using ShmVoidAllocator = bi::allocator<void, ShmSegmentManagerT>;
+
+using ExtentMapIdxT = size_t;
+using ExtentMapIdxTAlloc = bi::allocator<ExtentMapIdxT, ShmSegmentManagerT>;
+using PartitionNumberTAlloc = bi::allocator<PartitionNumberT, ShmSegmentManagerT>;
+using ExtentMapIndicesT = std::vector<ExtentMapIdxT, ExtentMapIdxTAlloc>;
+
+using PartitionIndexContainerKeyT = PartitionNumberT;
+using PartitionIndexContainerValT = std::pair<const PartitionIndexContainerKeyT, ExtentMapIndicesT>;
+using PartitionIndexContainerValTAlloc = bi::allocator<PartitionIndexContainerValT, ShmSegmentManagerT>;
+// Can't use std::unordered_map presumably b/c the map's pointer type doesn't use offset_type as boost::u_map
+// does
+using PartitionIndexContainerT =
+    boost::unordered_map<PartitionIndexContainerKeyT, ExtentMapIndicesT,
+                         boost::hash<PartitionIndexContainerKeyT>, std::equal_to<PartitionIndexContainerKeyT>,
+                         PartitionIndexContainerValTAlloc>;
+
+using OIDIndexContainerKeyT = OID_t;
+using OIDIndexContainerValT = std::pair<const OIDIndexContainerKeyT, PartitionIndexContainerT>;
+using OIDIndexContainerValTAlloc = bi::allocator<OIDIndexContainerValT, ShmSegmentManagerT>;
+using OIDIndexContainerT =
+    boost::unordered_map<OIDIndexContainerKeyT, PartitionIndexContainerT, boost::hash<OIDIndexContainerKeyT>,
+                         std::equal_to<OIDIndexContainerKeyT>, OIDIndexContainerValTAlloc>;
+
+using DBRootIndexTAlloc = bi::allocator<OIDIndexContainerT, ShmSegmentManagerT>;
+using DBRootIndexContainerT = std::vector<OIDIndexContainerT, DBRootIndexTAlloc>;
+using ExtentMapIndex = DBRootIndexContainerT;
+using ExtentMapIndexFindResult = std::vector<ExtentMapIdxT>;
+using InsertUpdateShmemKeyPair = std::pair<bool, bool>;
+
+class ExtentMapIndexImpl
+{
+ public:
+  ~ExtentMapIndexImpl(){};
+
+  static ExtentMapIndexImpl* makeExtentMapIndexImpl(unsigned key, off_t size, bool readOnly = false);
+  static void refreshShm()
+  {
+    if (fInstance_)
+    {
+      delete fInstance_;
+      fInstance_ = nullptr;
+    }
+  }
+
+  // The multipliers and constants here are pure theoretical
+  // tested using customer's data.
+  static size_t estimateEMIndexSize(uint32_t numberOfExtents)
+  {
+    // These are just educated guess values to calculate initial
+    // managed shmem size.
+    constexpr const size_t tablesNumber_ = 100ULL;
+    constexpr const size_t columnsNumber_ = 200ULL;
+    constexpr const size_t dbRootsNumber_ = 3ULL;
+    constexpr const size_t filesInPartition_ = 4ULL;
+    constexpr const size_t extentsInPartition_ = filesInPartition_ * 2;
+    return numberOfExtents * emIdentUnitSize_ +
+           numberOfExtents / extentsInPartition_ * partitionContainerUnitSize_ +
+           dbRootsNumber_ * tablesNumber_ * columnsNumber_;
+  }
+
+  bool growIfNeeded(const size_t memoryNeeded);
+
+  inline void grow(off_t size)
+  {
+    int rc = fBRMManagedShmMemImpl_.grow(size);
+    idbassert(rc == 0);
+  }
+  // After this call one needs to refresh any refs or ptrs sourced
+  // from this shmem.
+  inline void makeReadOnly()
+  {
+    fBRMManagedShmMemImpl_.setReadOnly();
+  }
+
+  inline void swapout(BRMManagedShmImpl& rhs)
+  {
+    fBRMManagedShmMemImpl_.swap(rhs);
+  }
+
+  inline unsigned key() const
+  {
+    return fBRMManagedShmMemImpl_.key();
+  }
+
+  unsigned getShmemSize()
+  {
+    return fBRMManagedShmMemImpl_.getManagedSegment()->get_size();
+  }
+
+  size_t getShmemFree()
+  {
+    return fBRMManagedShmMemImpl_.getManagedSegment()->get_free_memory();
+  }
+
+  unsigned getShmemImplSize()
+  {
+    return fBRMManagedShmMemImpl_.size();
+  }
+
+  void createExtentMapIndexIfNeeded();
+  ExtentMapIndex* get();
+  InsertUpdateShmemKeyPair insert(const EMEntry& emEntry, const size_t emIdx);
+  InsertUpdateShmemKeyPair insert2ndLayerWrapper(OIDIndexContainerT& oids, const EMEntry& emEntry,
+                                                 const size_t emIdx, const bool aShmemHasGrown);
+  InsertUpdateShmemKeyPair insert2ndLayer(OIDIndexContainerT& oids, const EMEntry& emEntry,
+                                          const size_t emIdx, const bool aShmemHasGrown);
+  InsertUpdateShmemKeyPair insert3dLayerWrapper(PartitionIndexContainerT& partitions, const EMEntry& emEntry,
+                                                const size_t emIdx, const bool aShmemHasGrown);
+  InsertUpdateShmemKeyPair insert3dLayer(PartitionIndexContainerT& partitions, const EMEntry& emEntry,
+                                         const size_t emIdx, const bool aShmemHasGrown);
+  ExtentMapIndexFindResult find(const DBRootT dbroot, const OID_t oid,
+                                const PartitionNumberT partitionNumber);
+  ExtentMapIndexFindResult find(const DBRootT dbroot, const OID_t oid);
+  ExtentMapIndexFindResult search2ndLayer(OIDIndexContainerT& oids, const OID_t oid,
+                                          const PartitionNumberT partitionNumber);
+  ExtentMapIndexFindResult search2ndLayer(OIDIndexContainerT& oids, const OID_t oid);
+  ExtentMapIndexFindResult search3dLayer(PartitionIndexContainerT& partitions,
+                                         const PartitionNumberT partitionNumber);
+  void deleteDbRoot(const DBRootT dbroot);
+  void deleteOID(const DBRootT dbroot, const OID_t oid);
+  void deleteEMEntry(const EMEntry& emEntry, const ExtentMapIdxT emIdent);
+
+ private:
+  BRMManagedShmImpl fBRMManagedShmMemImpl_;
+  ExtentMapIndexImpl(unsigned key, off_t size, bool readOnly = false);
+  ExtentMapIndexImpl(const ExtentMapIndexImpl& rhs);
+  ExtentMapIndexImpl& operator=(const ExtentMapIndexImpl& rhs);
+
+  static std::mutex fInstanceMutex_;
+  static ExtentMapIndexImpl* fInstance_;
+  static const constexpr uint32_t dbRootContainerUnitSize_ = 64ULL;
+  static const constexpr uint32_t oidContainerUnitSize_ = 352ULL;        // 2 * map overhead
+  static const constexpr uint32_t partitionContainerUnitSize_ = 368ULL;  // single map overhead
+  static const constexpr uint32_t emIdentUnitSize_ = sizeof(uint64_t);
+  static const constexpr uint32_t extraUnits_ = 2;
+  static const constexpr size_t freeSpaceThreshold_ = 256 * 1024;
+};
+
 /** @brief This class encapsulates the extent map functionality of the system
  *
  * This class encapsulates the extent map functionality of the system.  It
@@ -345,7 +497,7 @@ class ExtentMap : public Undoable
    */
   EXPORT void load(const std::string& filename, bool fixFL = false);
 
-  /** @brief Loads the ExtentMap entries from a binayr blob.
+  /** @brief Loads the ExtentMap entries from a binary blob.
    *
    * Loads the ExtentMap entries from a file.  This will
    * clear out any existing entries.  The intention is that before
@@ -886,6 +1038,9 @@ class ExtentMap : public Undoable
   EXPORT void dumpTo(std::ostream& os);
   EXPORT const bool* getEMLockStatus();
   EXPORT const bool* getEMFLLockStatus();
+  EXPORT const bool* getEMIndexLockStatus();
+  size_t EMIndexShmemSize();
+  size_t EMIndexShmemFree();
 
 #ifdef BRM_DEBUG
   EXPORT void printEM() const;
@@ -895,11 +1050,11 @@ class ExtentMap : public Undoable
 #endif
 
  private:
-  static const size_t EM_INCREMENT_ROWS = 100;
+  static const constexpr size_t EM_INCREMENT_ROWS = 100;
-  static const size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry);
+  static const constexpr size_t EM_INITIAL_SIZE = EM_INCREMENT_ROWS * 10 * sizeof(EMEntry);
-  static const size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry);
+  static const constexpr size_t EM_INCREMENT = EM_INCREMENT_ROWS * sizeof(EMEntry);
-  static const size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange);
+  static const constexpr size_t EM_FREELIST_INITIAL_SIZE = 50 * sizeof(InlineLBIDRange);
-  static const size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange);
+  static const constexpr size_t EM_FREELIST_INCREMENT = 50 * sizeof(InlineLBIDRange);
 
   ExtentMap(const ExtentMap& em);
   ExtentMap& operator=(const ExtentMap& em);
@@ -910,6 +1065,7 @@ class ExtentMap : public Undoable
   key_t fCurrentFLShmkey;
   MSTEntry* fEMShminfo;
   MSTEntry* fFLShminfo;
+  MSTEntry* fEMIndexShminfo;
   const MasterSegmentTable fMST;
   bool r_only;
   typedef std::tr1::unordered_map<int, oam::DBRootConfigList*> PmDbRootMap_t;
@@ -917,8 +1073,9 @@ class ExtentMap : public Undoable
   time_t fCacheTime;  // timestamp associated with config cache
 
   int numUndoRecords;
-  bool flLocked, emLocked;
+  bool flLocked, emLocked, emIndexLocked;
   static boost::mutex mutex;  // @bug5355 - made mutex static
+  static boost::mutex emIndexMutex;
   boost::mutex fConfigCacheMutex;  // protect access to Config Cache
 
   enum OPS
@@ -930,6 +1087,12 @@ class ExtentMap : public Undoable
 
   OPS EMLock, FLLock;
 
+  LastIndEmptyIndEmptyInd _createExtentCommonSearch(const OID_t OID, const DBRootT dbRoot,
+                                                    const PartitionNumberT partitionNum,
+                                                    const SegmentT segmentNum);
+
+  void logAndSetEMIndexReadOnly(const std::string& funcName);
+
   LBID_t _createColumnExtent_DBroot(uint32_t size, int OID, uint32_t colWidth, uint16_t dbRoot,
                                     execplan::CalpontSystemCatalog::ColDataType colDataType,
                                     uint32_t& partitionNum, uint16_t& segmentNum, uint32_t& startBlockOffset);
@@ -941,24 +1104,32 @@ class ExtentMap : public Undoable
                                 uint16_t segmentNum);
   template <typename T>
   bool isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const;
-  void deleteExtent(int emIndex);
+  void deleteExtent(const int emIndex, const bool clearEMIndex = true);
   LBID_t getLBIDsFromFreeList(uint32_t size);
   void reserveLBIDRange(LBID_t start, uint8_t size);  // used by load() to allocate pre-existing LBIDs
 
-  key_t chooseEMShmkey();  // see the code for how keys are segmented
+  key_t chooseEMShmkey();
-  key_t chooseFLShmkey();  // see the code for how keys are segmented
+  key_t chooseFLShmkey();
+  key_t chooseEMIndexShmkey();
+  key_t getInitialEMIndexShmkey() const;
+  // see the code for how keys are segmented
+  key_t chooseShmkey(const MSTEntry* masterTableEntry, const uint32_t keyRangeBase) const;
   void grabEMEntryTable(OPS op);
   void grabFreeList(OPS op);
+  void grabEMIndex(OPS op);
   void releaseEMEntryTable(OPS op);
   void releaseFreeList(OPS op);
+  void releaseEMIndex(OPS op);
   void growEMShmseg(size_t nrows = 0);
   void growFLShmseg();
+  void growEMIndexShmseg(const size_t suggestedSize = 0);
   void finishChanges();
 
   EXPORT unsigned getFilesPerColumnPartition();
   unsigned getExtentsPerSegmentFile();
   unsigned getDbRootCount();
   void getPmDbRoots(int pm, std::vector<int>& dbRootList);
+  DBRootVec getAllDbRoots();
   void checkReloadConfig();
   ShmKeys fShmKeys;
 
@@ -979,6 +1150,7 @@ class ExtentMap : public Undoable
 
   ExtentMapImpl* fPExtMapImpl;
   FreeListImpl* fPFreeListImpl;
+  ExtentMapIndexImpl* fPExtMapIndexImpl_;
 };
 
 inline std::ostream& operator<<(std::ostream& os, ExtentMap& rhs)

versioning/BRM/lock_grabber.cpp

@@ -1,4 +1,5 @@
 /* Copyright (C) 2014 InfiniDB, Inc.
+   Copyright (C) 2016-2022 MariaDB Corporation
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
@@ -21,6 +22,7 @@
  *    third, lock or unlock it
  */
 
+#include <string>
 #include <iostream>
 #include <stdlib.h>
 #include <rwlock.h>
@@ -32,10 +34,15 @@ char* name;
 
 void usage()
 {
-  cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n"
+  std::cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock" << std::endl;
-       << "   which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n"
+  size_t lockId = 0;
-       << "   which_side_to_use: r|w   (read or write)\n"
+  for (auto& lockName : RWLockNames)
-       << "   lock_or_unlock:    l|u   (lock or unlock)\n";
+  {
+    std::cout << "         " << lockId++ << "=" << lockName << " ";
+  }
+  std::cout << std::endl
+            << "   which_side_to_use: r|w   (read or write)" << std::endl
+            << "   lock_or_unlock:    l|u   (lock or unlock)" << std::endl;
   exit(1);
 }
 
@@ -54,10 +61,21 @@ int main(int argc, char** argv)
   if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1)
     usage();
 
-  which_lock = atoi(argv[1]);
+  try
-
+  {
-  if (which_lock < 1 || which_lock > 5)
+    which_lock = std::stoi(argv[1]);
+  }
+  catch (std::exception const& e)
+  {
+    std::cerr << "Cannot convert the lock id: " << e.what() << std::endl;
     usage();
+  }
+
+  if (which_lock >= RWLockNames.size())
+    usage();
+
+  size_t minLockId = (which_lock > 0) ? which_lock : 1;
+  size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1;
 
   if (argv[2][0] == 'r')
     which_side = 0;
@@ -73,17 +91,28 @@ int main(int argc, char** argv)
   else
     usage();
 
+  for (size_t i = minLockId; i <= maxLockId; ++i)
+  {
     rwlock = new RWLock(0x10000 * which_lock);
 
     if (which_side == 0)
+    {
       if (lock_unlock == 0)
         rwlock->read_lock();
       else
         rwlock->read_unlock();
+    }
     else if (lock_unlock == 0)
+    {
       rwlock->write_lock();
+    }
     else
+    {
       rwlock->write_unlock();
+    }
+
+    delete rwlock;
+  }
 
   return 0;
 }

versioning/BRM/lock_state.cpp

@@ -1,4 +1,5 @@
 /* Copyright (C) 2014 InfiniDB, Inc.
+   Copyright (C) 2016-2022 MariaDB Corporation
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
@@ -22,6 +23,7 @@
  */
 
 #include <iostream>
+#include <string>
 #include <stdlib.h>
 #include <rwlock.h>
 
@@ -32,14 +34,18 @@ char* name;
 
 void usage()
 {
-  cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock\n"
+  std::cout << "Usage " << name << "   which_lock_to_use:" << std::endl;
-       << "   which_lock_to_use: 1=VSS 2=ExtentMap 3=FreeList 4=VBBM 5=CopyLocks\n";
+  size_t lockId = 0;
+  for (auto& lockName : RWLockNames)
+  {
+    std::cout << "         " << lockId++ << "=" << lockName << std::endl;
+  }
   exit(1);
 }
 
 int main(int argc, char** argv)
 {
-  uint32_t which_lock;  // 1-5
+  uint32_t which_lock;  // 0-6
   RWLock* rwlock;
   LockState state;
 
@@ -51,18 +57,35 @@ int main(int argc, char** argv)
   if (strlen(argv[1]) != 1)
     usage();
 
-  which_lock = atoi(argv[1]);
+  try
+  {
+    which_lock = std::stoi(argv[1]);
+  }
+  catch (std::exception const& e)
+  {
+    std::cerr << "Cannot convert the lock id: " << e.what() << std::endl;
+    usage();
+  }
 
-  if (which_lock < 1 || which_lock > 5)
+  if (which_lock >= RWLockNames.size())
     usage();
 
-  rwlock = new RWLock(0x10000 * which_lock);
+  size_t minLockId = (which_lock > 0) ? which_lock : 1;
+  size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1;
+
+  for (size_t i = minLockId; i <= maxLockId; ++i)
+  {
+    rwlock = new RWLock(0x10000 * i);
     state = rwlock->getLockState();
-  cout << "readers = " << state.reading << endl
+
-       << "writers = " << state.writing << endl
+    cout << RWLockNames[i] << " RWLock" << std::endl
-       << "readers waiting = " << state.readerswaiting << endl
+         << "   readers = " << state.reading << std::endl
-       << "writers waiting = " << state.writerswaiting << endl
+         << "   writers = " << state.writing << std::endl
-       << "mutex locked = " << (int)state.mutexLocked << endl;
+         << "   readers waiting = " << state.readerswaiting << std::endl
+         << "   writers waiting = " << state.writerswaiting << std::endl
+         << "   mutex locked = " << (int)state.mutexLocked << std::endl;
+    delete rwlock;
+  }
 
   return 0;
 }

versioning/BRM/mastersegmenttable.cpp

@@ -138,6 +138,7 @@ MasterSegmentTable::MasterSegmentTable()
   RWLockKeys[2] = fShmKeys.KEYRANGE_VBBM_BASE;
   RWLockKeys[3] = fShmKeys.KEYRANGE_VSS_BASE;
   RWLockKeys[4] = fShmKeys.KEYRANGE_CL_BASE;
+  RWLockKeys[5] = fShmKeys.KEYRANGE_EXTENTMAP_INDEX_BASE;
 
   try
   {

versioning/BRM/mastersegmenttable.h

@@ -109,8 +109,10 @@ class MasterSegmentTable
   static const int VSSSegment = 3;
   /// specifies the copy lock segment
   static const int CLSegment = 4;
+  /// specifies the EM Index segment
+  static const int EMIndex = 5;
   /// the number of tables currently defined
-  static const int nTables = 5;
+  static const int nTables = 6;
 
   /** @brief This function gets the specified table.
    *

versioning/BRM/shmkeys.cpp

@@ -50,6 +50,7 @@ ShmKeys::ShmKeys()
   KEYRANGE_EMFREELIST_BASE = 0x30000 | (BRM_UID << 20);
   KEYRANGE_VBBM_BASE = 0x40000 | (BRM_UID << 20);
   KEYRANGE_CL_BASE = 0x50000 | (BRM_UID << 20);
+  KEYRANGE_EXTENTMAP_INDEX_BASE = 0x60000 | (BRM_UID << 20);
   MST_SYSVKEY = 0xff000000 | BRM_UID;
   PROCESSSTATUS_SYSVKEY = 0xfd000000 | BRM_UID;
   SYSTEMSTATUS_SYSVKEY = 0xfc000000 | BRM_UID;
@@ -62,7 +63,7 @@ ShmKeys::ShmKeys()
 string ShmKeys::keyToName(unsigned key)
 {
   ostringstream oss;
-  oss << "InfiniDB-shm-";
+  oss << "MCS-shm-";
   oss << setw(8) << setfill('0') << hex << key;
   return oss.str();
 }

versioning/BRM/shmkeys.h

@@ -55,6 +55,7 @@ struct ShmKeys
   uint32_t KEYRANGE_EMFREELIST_BASE;
   uint32_t KEYRANGE_VBBM_BASE;
   uint32_t KEYRANGE_VSS_BASE;
+  uint32_t KEYRANGE_EXTENTMAP_INDEX_BASE;
 
   /****** Fixed location assignments *******/
   uint32_t MST_SYSVKEY;

versioning/BRM/slavedbrmnode.cpp

@@ -1492,6 +1492,11 @@ const bool* SlaveDBRMNode::getEMLockStatus()
   return em.getEMLockStatus();
 }
 
+const bool* SlaveDBRMNode::getEMIndexLockStatus()
+{
+  return em.getEMIndexLockStatus();
+}
+
 const bool* SlaveDBRMNode::getVBBMLockStatus()
 {
   return &locked[0];

versioning/BRM/slavedbrmnode.h

@@ -461,6 +461,7 @@ class SlaveDBRMNode
 
   EXPORT const bool* getEMFLLockStatus();
   EXPORT const bool* getEMLockStatus();
+  EXPORT const bool* getEMIndexLockStatus();
   EXPORT const bool* getVBBMLockStatus();
   EXPORT const bool* getVSSLockStatus();
 

versioning/BRM/slavenode.cpp

@@ -147,6 +147,8 @@ int ServiceWorkerNode::Child()
   monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE));
   monitorThreads.create_thread(RWLockMonitor(&die, slave.getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE));
   monitorThreads.create_thread(RWLockMonitor(&die, slave.getVSSLockStatus(), keys.KEYRANGE_VSS_BASE));
+  monitorThreads.create_thread(
+      RWLockMonitor(&die, slave.getEMIndexLockStatus(), keys.KEYRANGE_EXTENTMAP_INDEX_BASE));
 
   try
   {

writeengine/bulk/cpimport.cpp

@@ -916,16 +916,24 @@ void getTableOID(const std::string& xmlGenSchema, const std::string& xmlGenTable
 void constructTempXmlFile(const std::string& tempJobDir, const std::string& sJobIdStr,
                           const std::string& xmlGenSchema, const std::string& xmlGenTable,
                           const std::string& alternateImportDir, const std::string& S3Bucket,
-                          boost::filesystem::path& sFileName)
+                          const std::string& tableOIDStr, boost::filesystem::path& sFileName)
 {
   // Construct the job description file name
   std::string xmlErrMsg;
   int rc = 0;
-  std::string tableOIDStr;
+  std::string localTableOIDStr;
-  getTableOID(xmlGenSchema, xmlGenTable, tableOIDStr);
+  if (tableOIDStr.empty())
+  {
+    getTableOID(xmlGenSchema, xmlGenTable, localTableOIDStr);
+  }
+  else
+  {
+    localTableOIDStr = tableOIDStr;
+  }
+
   rc = XMLJob::genJobXMLFileName(std::string(), tempJobDir, sJobIdStr,
                                  true,  // using temp job xml file
-                                 xmlGenSchema, xmlGenTable, sFileName, xmlErrMsg, tableOIDStr);
+                                 xmlGenSchema, xmlGenTable, sFileName, xmlErrMsg, localTableOIDStr);
 
   if (rc != NO_ERROR)
   {
@@ -945,7 +953,7 @@ void constructTempXmlFile(const std::string& tempJobDir, const std::string& sJob
   {
     genProc.startXMLFile();
     execplan::CalpontSystemCatalog::TableName tbl(xmlGenSchema, xmlGenTable);
-    genProc.makeTableData(tbl);
+    genProc.makeTableData(tbl, localTableOIDStr);
 
     if (!genProc.makeColumnData(tbl))
     {
@@ -1222,9 +1230,9 @@ int main(int argc, char** argv)
     if (!xmlGenSchema.empty())  // create temporary job file name
     {
       // If JobID is not provided, then default to the table OID
+      std::string tableOIDStr{""};
       if (sJobIdStr.empty())
       {
-        std::string tableOIDStr;
         getTableOID(xmlGenSchema, xmlGenTable, tableOIDStr);
 
         if (!(BulkLoad::disableConsoleOutput()))
@@ -1239,7 +1247,7 @@ int main(int argc, char** argv)
 
       bUseTempJobFile = true;
       constructTempXmlFile(curJob.getTempJobDir(), sJobIdStr, xmlGenSchema, xmlGenTable,
-                           curJob.getAlternateImportDir(), curJob.getS3Bucket(), sFileName);
+                           curJob.getAlternateImportDir(), curJob.getS3Bucket(), tableOIDStr, sFileName);
     }
     else  // create user's persistent job file name
     {

writeengine/xml/we_xmlgenproc.cpp

@@ -168,7 +168,21 @@ void XMLGenProc::startXMLFile()
 // makeTableData
 // Create XML tag for a table.
 //------------------------------------------------------------------------------
+// This method is used by colxml only and it can be relatively slower doing tableRID()
+// first call. All subsequent calls will re-use data from CalpontSystemCatalog cache.
 void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table)
+{
+  boost::shared_ptr<CalpontSystemCatalog> cat =
+      CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID);
+  cat->identity(CalpontSystemCatalog::EC);
+  std::ostringstream oss;
+  // tableRID method might take a lot with a significant EM.
+  oss << cat->tableRID(table).objnum;
+
+  makeTableData(table, oss.str());
+}
+
+void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table, const std::string& tableOIDStr)
 {
   static unsigned kount;
 
@@ -180,11 +194,8 @@ void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table)
   {
     try
     {
-      boost::shared_ptr<CalpontSystemCatalog> cat =
+      xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_TBL_OID], "%s",
-          CalpontSystemCatalog::makeCalpontSystemCatalog(BULK_SYSCAT_SESSION_ID);
+                                        tableOIDStr.c_str());
-      cat->identity(CalpontSystemCatalog::EC);
-      xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_TBL_OID], "%d",
-                                        cat->tableRID(table).objnum);
     }
     catch (std::exception& ex)
     {

writeengine/xml/we_xmlgenproc.h

@@ -75,6 +75,9 @@ class XMLGenProc
    *
    * @param table Name of table for which the table tag is to be generated.
    */
+  EXPORT void makeTableData(const execplan::CalpontSystemCatalog::TableName& table,
+                            const std::string& tableOIDStr);
+
   EXPORT void makeTableData(const execplan::CalpontSystemCatalog::TableName& table);
 
   /** @brief Creates column tags for the specified table.

writeengine/xml/we_xmljob.cpp

@@ -1194,7 +1194,8 @@ void XMLJob::validateAllColumnsHaveTags(const execplan::CalpontSystemCatalog::RI
 /* static */
 int XMLJob::genJobXMLFileName(const string& sXMLJobDir, const string& jobDir, const string& jobId,
                               bool bTempFile, const string& schemaName, const string& tableName,
-                              boost::filesystem::path& xmlFilePath, string& errMsg, std::string& tableOIDStr)
+                              boost::filesystem::path& xmlFilePath, string& errMsg,
+                              const std::string& tableOIDStr)
 {
   // get full file directory path for XML job description file
   if (sXMLJobDir.empty())

writeengine/xml/we_xmljob.h

@@ -74,7 +74,7 @@ class XMLJob : public XMLOp
   EXPORT static int genJobXMLFileName(const std::string& sXMLJobDir, const std::string& jobDir,
                                       const std::string& jobId, bool bTempFile, const std::string& schemaName,
                                       const std::string& tableName, boost::filesystem::path& xmlDirPath,
-                                      std::string& errMsg, std::string& tableOIDStr);
+                                      std::string& errMsg, const std::string& tableOIDStr);
 
   /**
    * @brief Get job structure