fix(syscat): MCOL-5816 23.02 -> 23.10 upgrade issues (#3346) (#3374)

* feat(BRM,tools): couple utilities to watch/operate shared memory locks and extent map

* feat(BRM,tools): merged two utilities and added some extra dbbuilder output in case of upgrade

* fix(dbbuilder): extra output to log upgrade detection.

CMakeLists.txt

@@ -136,6 +136,7 @@ SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 INCLUDE(columnstore_version)
 INCLUDE(misc)
 INCLUDE(boost)
+INCLUDE(CLI11)
 INCLUDE(thrift)
 INCLUDE(arrow)
 

cmake/CLI11.cmake

@@ -0,0 +1,15 @@
+set(EXTERNAL_INCLUDE_DIR "${CMAKE_BINARY_DIR}/external/include")
+file(MAKE_DIRECTORY "${EXTERNAL_INCLUDE_DIR}")
+
+set(C11CLI_URL "https://github.com/CLIUtils/CLI11/releases/download/v2.4.2/CLI11.hpp")
+set(C11CLI_HEADER "${EXTERNAL_INCLUDE_DIR}/CLI11.hpp")
+set(CLI11_INCLUDE_DIR "${EXTERNAL_INCLUDE_DIR}")
+
+file(DOWNLOAD 
+    ${C11CLI_URL}
+    ${C11CLI_HEADER}
+    SHOW_PROGRESS STATUS download_status
+)
+
+add_library(CLI11 INTERFACE)
+target_include_directories(CLI11 INTERFACE ${CLI11_INCLUDE_DIR})

debian/mariadb-plugin-columnstore.install

@@ -37,6 +37,9 @@ usr/bin/mcs-loadbrm.py
 usr/bin/mcs_parquet_ddl
 usr/bin/mcs_parquet_gen
 usr/bin/mcs-stop-controllernode.sh
+usr/bin/mcs-load-brm-from-file
+usr/bin/mcs-load-em
+usr/bin/mcs-shmem-locks
 usr/bin/mcsGetConfig
 usr/bin/mcsSetConfig
 usr/bin/mycnfUpgrade

tools/dbbuilder/dbbuilder.cpp

@@ -84,12 +84,12 @@ void usage()
 
 const unsigned sysCatalogErr = logging::M0060;
 
-void errorHandler(const unsigned mid, const string& src, const string& msg, bool isCritErr = true)
+void messageHandler(const string& src, const string& msg, bool isCritErr = true)
 {
   logging::LoggingID lid(19);
   logging::MessageLog ml(lid);
   logging::Message::Args args;
-  logging::Message message(mid);
+  logging::Message message(sysCatalogErr);
 
   if (isCritErr)
   {
@@ -143,28 +143,6 @@ int main(int argc, char* argv[])
     return 1;
   }
 
-  oamModuleInfo_t t;
-  bool parentOAMModuleFlag = false;
-
-  // get local module info; validate running on Active Parent OAM Module
-  try
-  {
-    t = oam.getModuleInfo();
-    parentOAMModuleFlag = boost::get<4>(t);
-  }
-  catch (exception&)
-  {
-    parentOAMModuleFlag = true;
-  }
-
-  if (!parentOAMModuleFlag)
-  {
-    cerr << "Exiting, dbbuilder can only be run on the Active "
-            "Parent OAM Module"
-         << endl;
-    return 1;
-  }
-
   logFile = string(MCSLOGDIR) + "/install/dbbuilder.status";
 
   buildOption = atoi(argv[optind++]);
@@ -203,14 +181,11 @@ int main(int argc, char* argv[])
                                                        // note that the candidate and reference OID
                                                        // datatypes and colwidths are assumed to be the
                                                        // same in SystemCatalog::upgrade().
-    upgradeOidMap[OID_SYSCOLUMN_CHARSETNUM] =
+    upgradeOidMap[OID_SYSCOLUMN_CHARSETNUM] = std::make_pair(OID_SYSCOLUMN_OBJECTID, false);
-      std::make_pair(OID_SYSCOLUMN_OBJECTID, false);
 
     std::unordered_map<int, OidTypeT> upgradeOidTypeMap;
-    upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] =
+    upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] = std::make_pair(CalpontSystemCatalog::INT, 4);
-      std::make_pair(CalpontSystemCatalog::INT, 4);
+    upgradeOidTypeMap[OID_SYSCOLUMN_CHARSETNUM] = std::make_pair(CalpontSystemCatalog::INT, 4);
-    upgradeOidTypeMap[OID_SYSCOLUMN_CHARSETNUM] =
-      std::make_pair(CalpontSystemCatalog::INT, 4);
 
     std::unordered_map<int, std::string> upgradeOidDefaultValStrMap;
     upgradeOidDefaultValStrMap[OID_SYSTABLE_AUXCOLUMNOID] = "0";
@@ -227,11 +202,12 @@ int main(int argc, char* argv[])
             (iter->second).second = true;
             isUpgrade = true;
           }
+          messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false);
         }
 
         if (!isUpgrade)
         {
-          string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile;
+          string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > " + logFile;
 
           if (canWrite)
           {
@@ -242,7 +218,7 @@ int main(int argc, char* argv[])
             cerr << cmd << endl;
           }
 
-          errorHandler(sysCatalogErr, "Build system catalog",
+          messageHandler("Build system catalog",
                          "System catalog appears to exist.  It will remain intact "
                          "for reuse.  The database is not recreated.",
                          false);
@@ -276,7 +252,7 @@ int main(int argc, char* argv[])
         else
           cerr << cmd << endl;
 
-        errorHandler(sysCatalogErr, "Build system catalog", ex.what(), false);
+        messageHandler("Build system catalog", ex.what(), false);
         return 1;
       }
       catch (...)
@@ -288,7 +264,7 @@ int main(int argc, char* argv[])
         else
           cerr << cmd << endl;
 
-        errorHandler(sysCatalogErr, "Build system catalog", "HDFS check failed.", false);
+        messageHandler("Build system catalog", "HDFS check failed.", false);
         return 1;
       }
 
@@ -305,6 +281,15 @@ int main(int argc, char* argv[])
       }
       else
       {
+        string cmd = "echo 'Upgrade system catalog' > " + logFile;
+        if (canWrite)
+        {
+          rc = system(cmd.c_str());
+        }
+        else
+        {
+          cerr << cmd << endl;
+        }
         sysCatalog.upgrade(upgradeOidMap, upgradeOidTypeMap, upgradeOidDefaultValStrMap);
       }
 
@@ -326,7 +311,7 @@ int main(int argc, char* argv[])
           ostringstream os;
           os << "Warning: running " << cmd << " failed.  This is usually non-fatal.";
           cerr << os.str() << endl;
-          errorHandler(sysCatalogErr, "Save BRM", os.str());
+          messageHandler("Save BRM", os.str());
         }
       }
       else
@@ -343,7 +328,7 @@ int main(int argc, char* argv[])
       else
         cerr << cmd << endl;
 
-      errorHandler(sysCatalogErr, "Build system catalog", ex.what());
+      messageHandler("Build system catalog", ex.what());
     }
     catch (...)
     {
@@ -355,7 +340,7 @@ int main(int argc, char* argv[])
         cerr << cmd << endl;
 
       string err("Caught unknown exception!");
-      errorHandler(sysCatalogErr, "Build system catalog", err);
+      messageHandler("Build system catalog", err);
     }
   }
   else

versioning/BRM/CMakeLists.txt

@@ -116,3 +116,15 @@ add_executable(load_brm ${load_brm_SRCS})
 target_link_libraries(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES})
 
 install(TARGETS load_brm DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
+
+add_executable(mcs-load-em load_em.cpp)
+target_link_libraries(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES})
+install(TARGETS mcs-load-em DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
+
+add_executable(mcs-load-brm-from-file load_brm_from_file.cpp)
+target_link_libraries(mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11)
+install(TARGETS mcs-load-brm-from-file DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
+
+add_executable(mcs-shmem-locks shmem_locks.cpp)
+target_link_libraries(mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11)
+install(TARGETS mcs-shmem-locks DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

versioning/BRM/extentmap.cpp

@@ -61,17 +61,10 @@
 #include "configcpp.h"
 #endif
 
-#define EXTENTMAP_DLLEXPORT
 #include "extentmap.h"
-#undef EXTENTMAP_DLLEXPORT
 
 #define EM_MAX_SEQNUM 2000000000
 #define MAX_IO_RETRIES 10
-#define EM_MAGIC_V1 0x76f78b1c
-#define EM_MAGIC_V2 0x76f78b1d
-#define EM_MAGIC_V3 0x76f78b1e
-#define EM_MAGIC_V4 0x76f78b1f
-#define EM_MAGIC_V5 0x76f78b20
 
 #ifndef NDEBUG
 #define ASSERT(x)                                                                          \
@@ -122,6 +115,12 @@ EMCasualPartition_struct::EMCasualPartition_struct()
   isValid = CP_INVALID;
 }
 
+EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum,
+                                                   const char status)
+ : sequenceNum(seqNum), isValid(status), loVal(lo), hiVal(hi)
+{
+}
+
 EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum)
 {
   loVal = lo;
@@ -172,6 +171,22 @@ EMEntry::EMEntry()
   status = 0;
 }
 
+EMEntry::EMEntry(const InlineLBIDRange& range, int fileID, uint32_t blockOffset, HWM_t hwm,
+                 PartitionNumberT partitionNum, uint16_t segmentNum, DBRootT dbRoot, uint16_t colWid,
+                 int16_t status, EMPartition_t partition)
+ : range(range)
+ , fileID(fileID)
+ , blockOffset(blockOffset)
+ , HWM(hwm)
+ , partitionNum(partitionNum)
+ , segmentNum(segmentNum)
+ , dbRoot(dbRoot)
+ , colWid(colWid)
+ , status(status)
+ , partition(partition)
+{
+}
+
 EMEntry::EMEntry(const EMEntry& e)
 {
   range.start = e.range.start;
@@ -1682,17 +1697,17 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5)
 
   fEMRBTreeShminfo->currentSize = (emNumElements * EM_RB_TREE_NODE_SIZE) + EM_RB_TREE_EMPTY_SIZE;
 
-#ifdef DUMP_EXTENT_MAP
   cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl;
 
-  for (const auto& lbidEMEntryPair : *fExtentMapRBTRee)
+  // for (const auto& lbidEMEntryPair : *fExtentMapRBTRee)
+  for (auto& lbidEMEntryPair : *fExtentMapRBTree)
   {
     const EMEntry& emEntry = lbidEMEntryPair.second;
-    cout << emEntry.start << '\t' << emEntry.size << '\t' << emEntry.fileID << '\t' << emEntry.blockOffset
+    cout << emEntry.range.start << '\t' << emEntry.range.size << '\t' << emEntry.fileID << '\t'
-         << '\t' << emEntry.HWM << '\t' << emEntry.partitionNum << '\t' << emEntry.segmentNum << '\t'
+         << emEntry.blockOffset << '\t' << emEntry.HWM << '\t' << emEntry.partitionNum << '\t'
-         << emEntry.dbRoot << '\t' << emEntry.status << '\t' << emEntry.partition.cprange.hiVal << '\t'
+         << emEntry.segmentNum << '\t' << emEntry.dbRoot << '\t' << emEntry.status << '\t'
-         << emEntry.partition.cprange.loVal << '\t' << emEntry.partition.cprange.sequenceNum << '\t'
+         << emEntry.partition.cprange.hiVal << '\t' << emEntry.partition.cprange.loVal << '\t'
-         << (int)(emEntry.partition.cprange.isValid) << endl;
+         << emEntry.partition.cprange.sequenceNum << '\t' << (int)(emEntry.partition.cprange.isValid) << endl;
   }
 
   cout << "Free list entries:" << endl;
@@ -1700,8 +1715,6 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5)
 
   for (uint32_t i = 0; i < flNumElements; i++)
     cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl;
-
-#endif
 }
 
 void ExtentMap::load(const string& filename, bool fixFL)
@@ -3245,8 +3258,7 @@ void ExtentMap::logAndSetEMIndexReadOnly(const std::string& funcName)
 {
   fPExtMapIndexImpl_->makeReadOnly();
   ostringstream os;
-  os << "ExtentMap::" << funcName << ": "
+  os << "ExtentMap::" << funcName << ": " << "Can not update EM Index. EM Index shmem segment is set to"
-     << "Can not update EM Index. EM Index shmem segment is set to"
      << " readonly. Please restart Columnstore.";
   log(os.str(), logging::LOG_TYPE_CRITICAL);
 
@@ -4613,11 +4625,11 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v&
       if (emDbRoot.status == EXTENTUNAVAILABLE)
       {
         ostringstream oss;
-        oss << "ExtentMap::getDbRootHWMInfo(): "
+        oss << "ExtentMap::getDbRootHWMInfo(): " << "OID " << OID
-            << "OID " << OID << " has HWM extent that is UNAVAILABLE for "
+            << " has HWM extent that is UNAVAILABLE for " << "DBRoot" << emDbRoot.dbRoot
-            << "DBRoot" << emDbRoot.dbRoot << "; part#: " << emDbRoot.partitionNum
+            << "; part#: " << emDbRoot.partitionNum << ", seg#: " << emDbRoot.segmentNum
-            << ", seg#: " << emDbRoot.segmentNum << ", fbo: " << emDbRoot.fbo
+            << ", fbo: " << emDbRoot.fbo << ", localHWM: " << emDbRoot.localHWM
-            << ", localHWM: " << emDbRoot.localHWM << ", lbid: " << emDbRoot.startLbid << endl;
+            << ", lbid: " << emDbRoot.startLbid << endl;
         log(oss.str(), logging::LOG_TYPE_CRITICAL);
         throw runtime_error(oss.str());
       }

versioning/BRM/extentmap.h

@@ -79,6 +79,12 @@ class IDBDataFile;
 
 namespace BRM
 {
+#define EM_MAGIC_V1 0x76f78b1c
+#define EM_MAGIC_V2 0x76f78b1d
+#define EM_MAGIC_V3 0x76f78b1e
+#define EM_MAGIC_V4 0x76f78b1f
+#define EM_MAGIC_V5 0x76f78b20
+
 using PartitionNumberT = uint32_t;
 using DBRootT = uint16_t;
 using SegmentT = uint16_t;
@@ -158,6 +164,8 @@ struct EMCasualPartition_struct
   EXPORT EMCasualPartition_struct();
   EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum);
   EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum);
+  EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum,
+                                  const char status);
   EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em);
   EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em);
 };
@@ -182,6 +190,8 @@ struct EMEntry
   int16_t status;  // extent avail for query or not, or out of service
   EMPartition_t partition;
   EXPORT EMEntry();
+  EMEntry(const InlineLBIDRange& range, int fileID, uint32_t bOffset, HWM_t hwm, PartitionNumberT pNum,
+          uint16_t sNum, DBRootT dRoot, uint16_t cWid, int16_t s, EMPartition_t partition);
   EXPORT EMEntry(const EMEntry&);
   EXPORT EMEntry& operator=(const EMEntry&);
   EXPORT bool operator<(const EMEntry&) const;

versioning/BRM/load_brm_from_file.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
+/* Copyright (C) 2024 MariaDB Corporation, Inc.
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
@@ -20,67 +20,81 @@
  * If you re-compile extentmap.cpp to dump the extent map as it loads, you'll get a csv file on stdout.
  * Save this to a file and edit it as needed (remove the cruft at the top & bottom for sure). Then use
  * this tool to create a binary BRM_saves_em file.
- *
- * compile with
- * g++ -g -Wall -o load_brm_from_file -I$HOME/genii/export/include -I/usr/include/libxml2
- * load_brm_from_file.cpp
- *
  */
+
 #include <iostream>
 #include <stdint.h>
 #include <fstream>
 #include <cerrno>
 #include <string>
 #include <cstdlib>
-//#define NDEBUG
 #include <cassert>
 #include <limits>
 using namespace std;
 
-#include <boost/tokenizer.hpp>
+#include "CLI11.hpp"
-using namespace boost;
-
 #include "extentmap.h"
-using namespace BRM;
 
-#define EM_MAGIC_V4 0x76f78b1f
+static const char* BIN_NAME = "mcs-load-brm-from-file";
 
-namespace BRM
+template <typename T>
+T parseField(std::stringstream& ss, const char delimiter)
 {
-EMEntry::EMEntry()
+  std::string field;
-{
+  std::getline(ss, field, delimiter);
-  fileID = 0;
+  return std::stoll(field);
-  blockOffset = 0;
-  HWM = 0;
-  partitionNum = 0;
-  segmentNum = 0;
-  dbRoot = 0;
-  colWid = 0;
-  status = 0;
 }
-EMCasualPartition_struct::EMCasualPartition_struct()
+
+BRM::EMEntry parseLine(const std::string& line, char delimiter = '|')
 {
-  lo_val = numeric_limits<int64_t>::min();
+  std::stringstream ss(line);
-  hi_val = numeric_limits<int64_t>::max();
+  std::string field;
-  sequenceNum = 0;
+
-  isValid = CP_INVALID;
+  auto rangeStart = parseField<int64_t>(ss, delimiter);
+  auto rangeSize = parseField<uint32_t>(ss, delimiter);
+  BRM::InlineLBIDRange range{rangeStart, rangeSize};
+
+  auto fileID = parseField<int>(ss, delimiter);
+  auto blockOffset = parseField<uint32_t>(ss, delimiter);
+  auto hwm = parseField<BRM::HWM_t>(ss, delimiter);
+  auto partitionNum = parseField<BRM::PartitionNumberT>(ss, delimiter);
+  auto segmentNum = parseField<uint16_t>(ss, delimiter);
+  auto dbRoot = parseField<BRM::DBRootT>(ss, delimiter);
+  auto colWid = parseField<uint16_t>(ss, delimiter);
+  auto status = parseField<int16_t>(ss, delimiter);
+
+  auto hiVal = parseField<int64_t>(ss, delimiter);
+  auto loVal = parseField<int64_t>(ss, delimiter);
+  auto sequenceNum = parseField<int32_t>(ss, delimiter);
+  auto isValid = parseField<char>(ss, delimiter);
+  auto partition = BRM::EMCasualPartition_t{loVal, hiVal, sequenceNum, isValid};
+
+  return BRM::EMEntry{range,      fileID, blockOffset, hwm,    partitionNum,
+                      segmentNum, dbRoot, colWid,      status, {partition}};
 }
-}  // namespace BRM
 
 int main(int argc, char** argv)
 {
-  int e;
+  CLI::App app{BIN_NAME};
+  app.description(
+      "A tool to build Extent Map image file from its text representation. A text representation can be obtained using 'editem -i'"
+      "display the lock state.");
+  std::string srcFilename;
+  std::string dstFilename;
+  bool debug = false;
 
-  int loadSize[3];
+  app.add_option("-i,--input-filename", srcFilename,
+                 "Extent Map as its text representation.")
+      ->required();
+  app.add_option("-o,--output-filename", dstFilename,
+                 "Extent Map output image file, default as input-filename.out")
+      ->default_val("");
+  app.add_option("-d,--debug", debug, "Print extra output")->default_val(false);
 
-  if (argc < 2)
+  CLI11_PARSE(app, argc, argv);
-  {
-    cerr << "filename arg needed" << endl;
-    return 1;
-  }
 
-  ifstream in(argv[1]);
+  ifstream in(srcFilename);
-  e = errno;
+  int e = errno;
 
   if (!in)
   {
@@ -90,81 +104,43 @@ int main(int argc, char** argv)
 
   // Brute force count the number of lines
   int numEMEntries = 0;
-
+  {
     string line;
-
     getline(in, line);
-
     while (!in.eof())
     {
       numEMEntries++;
       getline(in, line);
     }
+  }
 
-  // start at the beginning again...
+  std::cout << "Number of entries: " << numEMEntries << std::endl;
-  in.clear();
-  in.seekg(0, ios_base::beg);
 
-  idbassert(in.good());
+  if (dstFilename.empty())
-  idbassert(in.tellg() == static_cast<streampos>(0));
-
-  string outname(argv[1]);
-  outname += ".out";
-
-  ofstream out(outname.c_str());
-  e = errno;
-
-  if (!out)
   {
-    cerr << "file write error: " << strerror(e) << endl;
+    dstFilename = srcFilename + ".out";
+  }
+  ofstream outFile(dstFilename);
+
+  BRM::InlineLBIDRange maxLBIDinUse{0, 0};
+
+  std::ifstream infile(srcFilename);
+  if (!infile.is_open())
+  {
+    std::cerr << "Can not open file " << srcFilename << std::endl;
     return 1;
   }
 
-  loadSize[0] = EM_MAGIC_V4;
+  int loadSize[3];
+  loadSize[0] = EM_MAGIC_V5;
   loadSize[1] = numEMEntries;
   loadSize[2] = 1;  // one free list entry
-  out.write((char*)&loadSize, (3 * sizeof(int)));
+  outFile.write((char*)&loadSize, (3 * sizeof(int)));
 
-  InlineLBIDRange fl;
+  string line;
-  fl.start = 0;
+  while (std::getline(infile, line))
-  // the max lbid is 2^54-1, the size is in units of 1k
-  fl.size = numeric_limits<uint32_t>::max();
-
-  InlineLBIDRange maxLBIDinUse;
-  maxLBIDinUse.start = 0;
-  maxLBIDinUse.size = 0;
-
-  getline(in, line);
-
-  while (!in.eof())
   {
-    EMEntry em;
+    BRM::EMEntry em = parseLine(line);
-    int64_t v;
-    tokenizer<> tok(line);
-    tokenizer<>::iterator beg = tok.begin();
-#if 0
-        emSrc[i].range.start
-                << '\t' << emSrc[i].range.size
-                << '\t' << emSrc[i].fileID
-                << '\t' << emSrc[i].blockOffset
-                << '\t' << emSrc[i].HWM
-                << '\t' << emSrc[i].partitionNum
-                << '\t' << emSrc[i].segmentNum
-                << '\t' << emSrc[i].dbRoot
-                << '\t' << emSrc[i].colWid
-                << '\t' << emSrc[i].status
-                << '\t' << emSrc[i].partition.cprange.hi_val
-                << '\t' << emSrc[i].partition.cprange.lo_val
-                << '\t' << emSrc[i].partition.cprange.sequenceNum
-                << '\t' << (int)(emSrc[i].partition.cprange.isValid)
-#endif
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.range.start = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.range.size = v;
 
     if (em.range.start > maxLBIDinUse.start)
     {
@@ -172,66 +148,26 @@ int main(int argc, char** argv)
       maxLBIDinUse.size = em.range.size;
     }
 
-    v = strtoll(beg->c_str(), 0, 0);
+    if (debug)
-    ++beg;
+    {
-    em.fileID = v;
+      std::cout << em.range.start << '\t' << em.range.size << '\t' << em.fileID << '\t' << em.blockOffset
-
+                << '\t' << em.HWM << '\t' << em.partitionNum << '\t' << em.segmentNum << '\t' << em.dbRoot
-    v = strtoll(beg->c_str(), 0, 0);
+                << '\t' << em.colWid << '\t' << em.status << '\t' << em.partition.cprange.hiVal << '\t'
-    ++beg;
+                << em.partition.cprange.loVal << '\t' << em.partition.cprange.sequenceNum << '\t'
-    em.blockOffset = v;
+                << (short int)(em.partition.cprange.isValid) << std::endl;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.HWM = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.partitionNum = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.segmentNum = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.dbRoot = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.colWid = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.status = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.partition.cprange.hi_val = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.partition.cprange.lo_val = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.partition.cprange.sequenceNum = v;
-
-    v = strtoll(beg->c_str(), 0, 0);
-    ++beg;
-    em.partition.cprange.isValid = v;
-
-    out.write((char*)&em, sizeof(em));
-
-    getline(in, line);
     }
 
-  fl.start = maxLBIDinUse.start + maxLBIDinUse.size * 1024;
+    outFile.write((char*)&em, sizeof(em));
-  fl.size -= fl.start / 1024;
+  }
+  infile.close();
 
-  out.write((char*)&fl, sizeof(fl));
+  auto flStart = maxLBIDinUse.start + maxLBIDinUse.size * 1024;
+  assert(flStart / 1024 <= numeric_limits<uint32_t>::max());
+  uint32_t flEnd = numeric_limits<uint32_t>::max() - flStart / 1024;
+  BRM::InlineLBIDRange fl{flStart, flEnd};
 
-  out.close();
+  outFile.write((char*)&fl, sizeof(fl));
-  in.close();
+  outFile.close();
 
   return 0;
 }

versioning/BRM/lock_grabber.cpp

@@ -1,118 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-   Copyright (C) 2016-2022 MariaDB Corporation
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-/* Takes two params,
- *    first, which lock use
- *    second, which side to use (read or write)
- *    third, lock or unlock it
- */
-
-#include <string>
-#include <iostream>
-#include <stdlib.h>
-#include <rwlock.h>
-
-using namespace std;
-using namespace rwlock;
-
-char* name;
-
-void usage()
-{
-  std::cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock" << std::endl;
-  size_t lockId = 0;
-  for (auto& lockName : RWLockNames)
-  {
-    std::cout << "         " << lockId++ << "=" << lockName << " ";
-  }
-  std::cout << std::endl
-            << "   which_side_to_use: r|w   (read or write)" << std::endl
-            << "   lock_or_unlock:    l|u   (lock or unlock)" << std::endl;
-  exit(1);
-}
-
-int main(int argc, char** argv)
-{
-  uint32_t which_lock;   // 1-5
-  uint32_t which_side;   // 0 or 1
-  uint32_t lock_unlock;  // 0 or 1
-  RWLock* rwlock;
-
-  name = argv[0];
-
-  if (argc != 4)
-    usage();
-
-  if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1)
-    usage();
-
-  try
-  {
-    which_lock = std::stoi(argv[1]);
-  }
-  catch (std::exception const& e)
-  {
-    std::cerr << "Cannot convert the lock id: " << e.what() << std::endl;
-    usage();
-  }
-
-  if (which_lock >= RWLockNames.size())
-    usage();
-
-  size_t minLockId = (which_lock > 0) ? which_lock : 1;
-  size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1;
-
-  if (argv[2][0] == 'r')
-    which_side = 0;
-  else if (argv[2][0] == 'w')
-    which_side = 1;
-  else
-    usage();
-
-  if (argv[3][0] == 'l')
-    lock_unlock = 0;
-  else if (argv[3][0] == 'u')
-    lock_unlock = 1;
-  else
-    usage();
-
-  for (size_t i = minLockId; i <= maxLockId; ++i)
-  {
-    rwlock = new RWLock(0x10000 * which_lock);
-
-    if (which_side == 0)
-    {
-      if (lock_unlock == 0)
-        rwlock->read_lock();
-      else
-        rwlock->read_unlock();
-    }
-    else if (lock_unlock == 0)
-    {
-      rwlock->write_lock();
-    }
-    else
-    {
-      rwlock->write_unlock();
-    }
-
-    delete rwlock;
-  }
-
-  return 0;
-}

versioning/BRM/lock_state.cpp

@@ -1,91 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-   Copyright (C) 2016-2022 MariaDB Corporation
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-/* Takes two params,
- *    first, which lock use
- *    second, which side to use (read or write)
- *    third, lock or unlock it
- */
-
-#include <iostream>
-#include <string>
-#include <stdlib.h>
-#include <rwlock.h>
-
-using namespace std;
-using namespace rwlock;
-
-char* name;
-
-void usage()
-{
-  std::cout << "Usage " << name << "   which_lock_to_use:" << std::endl;
-  size_t lockId = 0;
-  for (auto& lockName : RWLockNames)
-  {
-    std::cout << "         " << lockId++ << "=" << lockName << std::endl;
-  }
-  exit(1);
-}
-
-int main(int argc, char** argv)
-{
-  uint32_t which_lock;  // 0-6
-  RWLock* rwlock;
-  LockState state;
-
-  name = argv[0];
-
-  if (argc != 2)
-    usage();
-
-  if (strlen(argv[1]) != 1)
-    usage();
-
-  try
-  {
-    which_lock = std::stoi(argv[1]);
-  }
-  catch (std::exception const& e)
-  {
-    std::cerr << "Cannot convert the lock id: " << e.what() << std::endl;
-    usage();
-  }
-
-  if (which_lock >= RWLockNames.size())
-    usage();
-
-  size_t minLockId = (which_lock > 0) ? which_lock : 1;
-  size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1;
-
-  for (size_t i = minLockId; i <= maxLockId; ++i)
-  {
-    rwlock = new RWLock(0x10000 * i);
-    state = rwlock->getLockState();
-
-    cout << RWLockNames[i] << " RWLock" << std::endl
-         << "   readers = " << state.reading << std::endl
-         << "   writers = " << state.writing << std::endl
-         << "   readers waiting = " << state.readerswaiting << std::endl
-         << "   writers waiting = " << state.writerswaiting << std::endl
-         << "   mutex locked = " << (int)state.mutexLocked << std::endl;
-    delete rwlock;
-  }
-
-  return 0;
-}

versioning/BRM/shmem_locks.cpp

@@ -0,0 +1,128 @@
+/* Copyright (C) 2016-2024 MariaDB Corporation
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <stdlib.h>
+#include <rwlock.h>
+
+#include "CLI11.hpp"
+
+using namespace std;
+using namespace rwlock;
+
+static const char* BIN_NAME = "mcs-load-brm-from-file";
+
+std::string getShmemLocksList()
+{
+  std::ostringstream oss;
+  size_t lockId = 0;
+  oss << std::endl;
+  for (auto& lockName : RWLockNames)
+  {
+    oss << "         " << lockId++ << "=" << lockName << std::endl;
+  }
+  return oss.str();
+}
+
+int viewLock(uint8_t lockId)
+{
+  size_t minLockId = (lockId > 0) ? lockId : 1;
+  size_t maxLockId = (lockId > 0) ? lockId : RWLockNames.size() - 1;
+
+  for (size_t i = minLockId; i <= maxLockId; ++i)
+  {
+    auto rwlock = RWLock(0x10000 * i);
+    auto state = rwlock.getLockState();
+
+    cout << RWLockNames[i] << " RWLock" << std::endl
+         << "   readers = " << state.reading << std::endl
+         << "   writers = " << state.writing << std::endl
+         << "   readers waiting = " << state.readerswaiting << std::endl
+         << "   writers waiting = " << state.writerswaiting << std::endl
+         << "   mutex locked = " << (int)state.mutexLocked << std::endl;
+  }
+  return 0;
+}
+
+int lockOp(size_t minLockId, size_t maxLockId, bool lock, bool read)
+{
+  for (size_t i = minLockId; i <= maxLockId; ++i)
+  {
+    auto rwlock = RWLock(0x10000 * i);
+
+    if (read)
+    {
+      if (lock)
+        rwlock.read_lock();
+      else
+        rwlock.read_unlock();
+    }
+    else if (lock)
+    {
+      rwlock.write_lock();
+    }
+    else
+    {
+      rwlock.write_unlock();
+    }
+  }
+  return 0;
+}
+
+int main(int argc, char** argv)
+{
+  CLI::App app{BIN_NAME};
+  app.description(
+      "A tool to operate or view shmem locks. If neither read nor write operation is specified, the tool will "
+      "display the lock state.");
+  uint8_t lockId;
+  bool debug = false;
+  bool read = false;
+  bool write = false;
+  bool lock = false;
+  bool unlock = false;
+
+  app.add_option<uint8_t>("-i, --lock-id", lockId, "Shmem lock numerical id: " + getShmemLocksList())
+      ->expected(0, RWLockNames.size())
+      ->required();
+  app.add_flag("-r, --read-lock", read, "Use read lock.")->default_val(false);
+  app.add_flag("-w, --write-lock", write, "Use write lock..")->default_val(false)->excludes("-r");
+  app.add_flag("-l, --lock", lock, "Lock the corresponding shmem lock.")->default_val(false);
+  app.add_flag("-u, --unlock", unlock, "Unlock the corresponding shmem write lock.")
+      ->default_val(false)
+      ->excludes("-l");
+  app.add_flag("-d,--debug", debug, "Print extra output.")->default_val(false);
+
+  CLI11_PARSE(app, argc, argv);
+
+  if (!read && !write)
+  {
+    return viewLock(lockId);
+  }
+
+  if (lock || unlock)
+  {
+    size_t minLockId = (lockId > 0) ? lockId : 1;
+    size_t maxLockId = (lockId > 0) ? lockId : RWLockNames.size() - 1;
+    return lockOp(minLockId, maxLockId, lock, read);
+  }
+
+  return 0;
+}
+

versioning/BRM/tablelockserver.cpp

@@ -110,7 +110,7 @@ void TableLockServer::load()
   if (!in)
   {
     ostringstream os;
-    os << "TableLockServer::load(): could not open the save file" << filename;
+    os << "TableLockServer::load(): either this is the first cluster start or could not open the save file" << filename;
     log(os.str(), logging::LOG_TYPE_DEBUG);
     return;
   }