From dd8fac35ae8bab1835b349a1dc0db367117cfe3c Mon Sep 17 00:00:00 2001 From: drrtuy Date: Thu, 26 Dec 2024 16:26:34 +0000 Subject: [PATCH] fix(syscat): MCOL-5816 23.02 -> 23.10 upgrade issues (#3346) (#3374) * feat(BRM,tools): couple utilities to watch/operate shared memory locks and extent map * feat(BRM,tools): merged two utilities and added some extra dbbuilder output in case of upgrade * fix(dbbuilder): extra output to log upgrade detection. --- CMakeLists.txt | 1 + cmake/CLI11.cmake | 15 ++ debian/mariadb-plugin-columnstore.install | 3 + tools/dbbuilder/dbbuilder.cpp | 85 +++----- versioning/BRM/CMakeLists.txt | 12 ++ versioning/BRM/extentmap.cpp | 72 ++++--- versioning/BRM/extentmap.h | 10 + versioning/BRM/load_brm_from_file.cpp | 248 ++++++++-------------- versioning/BRM/lock_grabber.cpp | 118 ---------- versioning/BRM/lock_state.cpp | 91 -------- versioning/BRM/shmem_locks.cpp | 128 +++++++++++ versioning/BRM/tablelockserver.cpp | 2 +- 12 files changed, 339 insertions(+), 446 deletions(-) create mode 100644 cmake/CLI11.cmake delete mode 100644 versioning/BRM/lock_grabber.cpp delete mode 100644 versioning/BRM/lock_state.cpp create mode 100644 versioning/BRM/shmem_locks.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f8cd0737..e26483967 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,7 @@ SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) INCLUDE(columnstore_version) INCLUDE(misc) INCLUDE(boost) +INCLUDE(CLI11) INCLUDE(thrift) INCLUDE(arrow) diff --git a/cmake/CLI11.cmake b/cmake/CLI11.cmake new file mode 100644 index 000000000..d86e9f99c --- /dev/null +++ b/cmake/CLI11.cmake @@ -0,0 +1,15 @@ +set(EXTERNAL_INCLUDE_DIR "${CMAKE_BINARY_DIR}/external/include") +file(MAKE_DIRECTORY "${EXTERNAL_INCLUDE_DIR}") + +set(C11CLI_URL "https://github.com/CLIUtils/CLI11/releases/download/v2.4.2/CLI11.hpp") +set(C11CLI_HEADER "${EXTERNAL_INCLUDE_DIR}/CLI11.hpp") +set(CLI11_INCLUDE_DIR "${EXTERNAL_INCLUDE_DIR}") + +file(DOWNLOAD + ${C11CLI_URL} + ${C11CLI_HEADER} + SHOW_PROGRESS STATUS download_status +) + +add_library(CLI11 INTERFACE) +target_include_directories(CLI11 INTERFACE ${CLI11_INCLUDE_DIR}) \ No newline at end of file diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index 17539eaa9..6f0ca724b 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -37,6 +37,9 @@ usr/bin/mcs-loadbrm.py usr/bin/mcs_parquet_ddl usr/bin/mcs_parquet_gen usr/bin/mcs-stop-controllernode.sh +usr/bin/mcs-load-brm-from-file +usr/bin/mcs-load-em +usr/bin/mcs-shmem-locks usr/bin/mcsGetConfig usr/bin/mcsSetConfig usr/bin/mycnfUpgrade diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index 33c3323e3..9958d2eef 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -84,12 +84,12 @@ void usage() const unsigned sysCatalogErr = logging::M0060; -void errorHandler(const unsigned mid, const string& src, const string& msg, bool isCritErr = true) +void messageHandler(const string& src, const string& msg, bool isCritErr = true) { logging::LoggingID lid(19); logging::MessageLog ml(lid); logging::Message::Args args; - logging::Message message(mid); + logging::Message message(sysCatalogErr); if (isCritErr) { @@ -143,28 +143,6 @@ int main(int argc, char* argv[]) return 1; } - oamModuleInfo_t t; - bool parentOAMModuleFlag = false; - - // get local module info; validate running on Active Parent OAM Module - try - { - t = oam.getModuleInfo(); - parentOAMModuleFlag = boost::get<4>(t); - } - catch (exception&) - { - parentOAMModuleFlag = true; - } - - if (!parentOAMModuleFlag) - { - cerr << "Exiting, dbbuilder can only be run on the Active " - "Parent OAM Module" - << endl; - return 1; - } - logFile = string(MCSLOGDIR) + "/install/dbbuilder.status"; buildOption = atoi(argv[optind++]); @@ -193,24 +171,21 @@ int main(int argc, char* argv[]) bool isUpgrade = false; std::unordered_map> upgradeOidMap; - upgradeOidMap[OID_SYSTABLE_AUXCOLUMNOID] = // This is the candidate OID for the upgrade. - std::make_pair(OID_SYSTABLE_OBJECTID, false); // std::pair::first is the reference OID used - // to fill the candidate OID with default vals - // std::pair::second is set to false by default - // which means the candidate OID will not be - // upgraded. This is set to true in the code - // below if a specific condition is met. Please - // note that the candidate and reference OID - // datatypes and colwidths are assumed to be the - // same in SystemCatalog::upgrade(). - upgradeOidMap[OID_SYSCOLUMN_CHARSETNUM] = - std::make_pair(OID_SYSCOLUMN_OBJECTID, false); + upgradeOidMap[OID_SYSTABLE_AUXCOLUMNOID] = // This is the candidate OID for the upgrade. + std::make_pair(OID_SYSTABLE_OBJECTID, false); // std::pair::first is the reference OID used + // to fill the candidate OID with default vals + // std::pair::second is set to false by default + // which means the candidate OID will not be + // upgraded. This is set to true in the code + // below if a specific condition is met. Please + // note that the candidate and reference OID + // datatypes and colwidths are assumed to be the + // same in SystemCatalog::upgrade(). + upgradeOidMap[OID_SYSCOLUMN_CHARSETNUM] = std::make_pair(OID_SYSCOLUMN_OBJECTID, false); std::unordered_map upgradeOidTypeMap; - upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] = - std::make_pair(CalpontSystemCatalog::INT, 4); - upgradeOidTypeMap[OID_SYSCOLUMN_CHARSETNUM] = - std::make_pair(CalpontSystemCatalog::INT, 4); + upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] = std::make_pair(CalpontSystemCatalog::INT, 4); + upgradeOidTypeMap[OID_SYSCOLUMN_CHARSETNUM] = std::make_pair(CalpontSystemCatalog::INT, 4); std::unordered_map upgradeOidDefaultValStrMap; upgradeOidDefaultValStrMap[OID_SYSTABLE_AUXCOLUMNOID] = "0"; @@ -227,11 +202,12 @@ int main(int argc, char* argv[]) (iter->second).second = true; isUpgrade = true; } + messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false); } if (!isUpgrade) { - string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; + string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; if (canWrite) { @@ -242,10 +218,10 @@ int main(int argc, char* argv[]) cerr << cmd << endl; } - errorHandler(sysCatalogErr, "Build system catalog", - "System catalog appears to exist. It will remain intact " - "for reuse. The database is not recreated.", - false); + messageHandler("Build system catalog", + "System catalog appears to exist. It will remain intact " + "for reuse. The database is not recreated.", + false); return 1; } } @@ -276,7 +252,7 @@ int main(int argc, char* argv[]) else cerr << cmd << endl; - errorHandler(sysCatalogErr, "Build system catalog", ex.what(), false); + messageHandler("Build system catalog", ex.what(), false); return 1; } catch (...) @@ -288,7 +264,7 @@ int main(int argc, char* argv[]) else cerr << cmd << endl; - errorHandler(sysCatalogErr, "Build system catalog", "HDFS check failed.", false); + messageHandler("Build system catalog", "HDFS check failed.", false); return 1; } @@ -305,6 +281,15 @@ int main(int argc, char* argv[]) } else { + string cmd = "echo 'Upgrade system catalog' > " + logFile; + if (canWrite) + { + rc = system(cmd.c_str()); + } + else + { + cerr << cmd << endl; + } sysCatalog.upgrade(upgradeOidMap, upgradeOidTypeMap, upgradeOidDefaultValStrMap); } @@ -326,7 +311,7 @@ int main(int argc, char* argv[]) ostringstream os; os << "Warning: running " << cmd << " failed. This is usually non-fatal."; cerr << os.str() << endl; - errorHandler(sysCatalogErr, "Save BRM", os.str()); + messageHandler("Save BRM", os.str()); } } else @@ -343,7 +328,7 @@ int main(int argc, char* argv[]) else cerr << cmd << endl; - errorHandler(sysCatalogErr, "Build system catalog", ex.what()); + messageHandler("Build system catalog", ex.what()); } catch (...) { @@ -355,7 +340,7 @@ int main(int argc, char* argv[]) cerr << cmd << endl; string err("Caught unknown exception!"); - errorHandler(sysCatalogErr, "Build system catalog", err); + messageHandler("Build system catalog", err); } } else diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 2b4b74db5..203360bad 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -116,3 +116,15 @@ add_executable(load_brm ${load_brm_SRCS}) target_link_libraries(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) install(TARGETS load_brm DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) + +add_executable(mcs-load-em load_em.cpp) +target_link_libraries(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +install(TARGETS mcs-load-em DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) + +add_executable(mcs-load-brm-from-file load_brm_from_file.cpp) +target_link_libraries(mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11) +install(TARGETS mcs-load-brm-from-file DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) + +add_executable(mcs-shmem-locks shmem_locks.cpp) +target_link_libraries(mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11) +install(TARGETS mcs-shmem-locks DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) \ No newline at end of file diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 11e72f0ef..b1e3e9890 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -61,17 +61,10 @@ #include "configcpp.h" #endif -#define EXTENTMAP_DLLEXPORT #include "extentmap.h" -#undef EXTENTMAP_DLLEXPORT #define EM_MAX_SEQNUM 2000000000 #define MAX_IO_RETRIES 10 -#define EM_MAGIC_V1 0x76f78b1c -#define EM_MAGIC_V2 0x76f78b1d -#define EM_MAGIC_V3 0x76f78b1e -#define EM_MAGIC_V4 0x76f78b1f -#define EM_MAGIC_V5 0x76f78b20 #ifndef NDEBUG #define ASSERT(x) \ @@ -122,6 +115,12 @@ EMCasualPartition_struct::EMCasualPartition_struct() isValid = CP_INVALID; } +EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum, + const char status) + : sequenceNum(seqNum), isValid(status), loVal(lo), hiVal(hi) +{ +} + EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum) { loVal = lo; @@ -172,6 +171,22 @@ EMEntry::EMEntry() status = 0; } +EMEntry::EMEntry(const InlineLBIDRange& range, int fileID, uint32_t blockOffset, HWM_t hwm, + PartitionNumberT partitionNum, uint16_t segmentNum, DBRootT dbRoot, uint16_t colWid, + int16_t status, EMPartition_t partition) + : range(range) + , fileID(fileID) + , blockOffset(blockOffset) + , HWM(hwm) + , partitionNum(partitionNum) + , segmentNum(segmentNum) + , dbRoot(dbRoot) + , colWid(colWid) + , status(status) + , partition(partition) +{ +} + EMEntry::EMEntry(const EMEntry& e) { range.start = e.range.start; @@ -1682,17 +1697,17 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) fEMRBTreeShminfo->currentSize = (emNumElements * EM_RB_TREE_NODE_SIZE) + EM_RB_TREE_EMPTY_SIZE; -#ifdef DUMP_EXTENT_MAP cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl; - for (const auto& lbidEMEntryPair : *fExtentMapRBTRee) + // for (const auto& lbidEMEntryPair : *fExtentMapRBTRee) + for (auto& lbidEMEntryPair : *fExtentMapRBTree) { const EMEntry& emEntry = lbidEMEntryPair.second; - cout << emEntry.start << '\t' << emEntry.size << '\t' << emEntry.fileID << '\t' << emEntry.blockOffset - << '\t' << emEntry.HWM << '\t' << emEntry.partitionNum << '\t' << emEntry.segmentNum << '\t' - << emEntry.dbRoot << '\t' << emEntry.status << '\t' << emEntry.partition.cprange.hiVal << '\t' - << emEntry.partition.cprange.loVal << '\t' << emEntry.partition.cprange.sequenceNum << '\t' - << (int)(emEntry.partition.cprange.isValid) << endl; + cout << emEntry.range.start << '\t' << emEntry.range.size << '\t' << emEntry.fileID << '\t' + << emEntry.blockOffset << '\t' << emEntry.HWM << '\t' << emEntry.partitionNum << '\t' + << emEntry.segmentNum << '\t' << emEntry.dbRoot << '\t' << emEntry.status << '\t' + << emEntry.partition.cprange.hiVal << '\t' << emEntry.partition.cprange.loVal << '\t' + << emEntry.partition.cprange.sequenceNum << '\t' << (int)(emEntry.partition.cprange.isValid) << endl; } cout << "Free list entries:" << endl; @@ -1700,8 +1715,6 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) for (uint32_t i = 0; i < flNumElements; i++) cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl; - -#endif } void ExtentMap::load(const string& filename, bool fixFL) @@ -2745,7 +2758,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co highestSegNum = emEntry.segmentNum; highestOffset = emEntry.blockOffset; } // found extentmap entry for specified OID - } // Loop through extent map entries + } // Loop through extent map entries DBRootVec dbRootVec(getAllDbRoots()); // 2. for empty DBRoot track hi seg# in user specified part# @@ -2832,7 +2845,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co } } } // loop over em idents - } // current dbroot == target dbroot + } // current dbroot == target dbroot else { // 4. Track hi seg for hwm+1 partition @@ -2853,8 +2866,8 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co partHighSeg = emEntry.segmentNum; } } // current dbroot != target dbroot - } // loop over dbroots - } // (lastExtentIndex >= 0) + } // loop over dbroots + } // (lastExtentIndex >= 0) //-------------------------------------------------------------------------- // Third Step: Select partition and segment number for new extent @@ -3038,7 +3051,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co newBlockOffset = lastExtent->blockOffset; } } - } // lastExtentIndex >= 0 + } // lastExtentIndex >= 0 else // Empty DBRoot; use part# that the user specifies { if (bHighEmptySegNumSet) @@ -3245,8 +3258,7 @@ void ExtentMap::logAndSetEMIndexReadOnly(const std::string& funcName) { fPExtMapIndexImpl_->makeReadOnly(); ostringstream os; - os << "ExtentMap::" << funcName << ": " - << "Can not update EM Index. EM Index shmem segment is set to" + os << "ExtentMap::" << funcName << ": " << "Can not update EM Index. EM Index shmem segment is set to" << " readonly. Please restart Columnstore."; log(os.str(), logging::LOG_TYPE_CRITICAL); @@ -4123,7 +4135,7 @@ void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) emIt = deleteExtent(emIt); } } // em iterarors loop - } // em info map loop + } // em info map loop } else { @@ -4193,7 +4205,7 @@ void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo) } } } // em iterarors loop - } // em info map loop + } // em info map loop } } //------------------------------------------------------------------------------ @@ -4613,11 +4625,11 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v& if (emDbRoot.status == EXTENTUNAVAILABLE) { ostringstream oss; - oss << "ExtentMap::getDbRootHWMInfo(): " - << "OID " << OID << " has HWM extent that is UNAVAILABLE for " - << "DBRoot" << emDbRoot.dbRoot << "; part#: " << emDbRoot.partitionNum - << ", seg#: " << emDbRoot.segmentNum << ", fbo: " << emDbRoot.fbo - << ", localHWM: " << emDbRoot.localHWM << ", lbid: " << emDbRoot.startLbid << endl; + oss << "ExtentMap::getDbRootHWMInfo(): " << "OID " << OID + << " has HWM extent that is UNAVAILABLE for " << "DBRoot" << emDbRoot.dbRoot + << "; part#: " << emDbRoot.partitionNum << ", seg#: " << emDbRoot.segmentNum + << ", fbo: " << emDbRoot.fbo << ", localHWM: " << emDbRoot.localHWM + << ", lbid: " << emDbRoot.startLbid << endl; log(oss.str(), logging::LOG_TYPE_CRITICAL); throw runtime_error(oss.str()); } diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index ab7c3f4ee..cdfa738a4 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -79,6 +79,12 @@ class IDBDataFile; namespace BRM { +#define EM_MAGIC_V1 0x76f78b1c +#define EM_MAGIC_V2 0x76f78b1d +#define EM_MAGIC_V3 0x76f78b1e +#define EM_MAGIC_V4 0x76f78b1f +#define EM_MAGIC_V5 0x76f78b20 + using PartitionNumberT = uint32_t; using DBRootT = uint16_t; using SegmentT = uint16_t; @@ -158,6 +164,8 @@ struct EMCasualPartition_struct EXPORT EMCasualPartition_struct(); EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum, + const char status); EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em); }; @@ -182,6 +190,8 @@ struct EMEntry int16_t status; // extent avail for query or not, or out of service EMPartition_t partition; EXPORT EMEntry(); + EMEntry(const InlineLBIDRange& range, int fileID, uint32_t bOffset, HWM_t hwm, PartitionNumberT pNum, + uint16_t sNum, DBRootT dRoot, uint16_t cWid, int16_t s, EMPartition_t partition); EXPORT EMEntry(const EMEntry&); EXPORT EMEntry& operator=(const EMEntry&); EXPORT bool operator<(const EMEntry&) const; diff --git a/versioning/BRM/load_brm_from_file.cpp b/versioning/BRM/load_brm_from_file.cpp index bc7c05d9a..f70baafd7 100644 --- a/versioning/BRM/load_brm_from_file.cpp +++ b/versioning/BRM/load_brm_from_file.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 InfiniDB, Inc. +/* Copyright (C) 2024 MariaDB Corporation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -20,67 +20,81 @@ * If you re-compile extentmap.cpp to dump the extent map as it loads, you'll get a csv file on stdout. * Save this to a file and edit it as needed (remove the cruft at the top & bottom for sure). Then use * this tool to create a binary BRM_saves_em file. - * - * compile with - * g++ -g -Wall -o load_brm_from_file -I$HOME/genii/export/include -I/usr/include/libxml2 - * load_brm_from_file.cpp - * */ + #include #include #include #include #include #include -//#define NDEBUG #include #include using namespace std; -#include -using namespace boost; - +#include "CLI11.hpp" #include "extentmap.h" -using namespace BRM; -#define EM_MAGIC_V4 0x76f78b1f +static const char* BIN_NAME = "mcs-load-brm-from-file"; -namespace BRM +template +T parseField(std::stringstream& ss, const char delimiter) { -EMEntry::EMEntry() -{ - fileID = 0; - blockOffset = 0; - HWM = 0; - partitionNum = 0; - segmentNum = 0; - dbRoot = 0; - colWid = 0; - status = 0; + std::string field; + std::getline(ss, field, delimiter); + return std::stoll(field); } -EMCasualPartition_struct::EMCasualPartition_struct() + +BRM::EMEntry parseLine(const std::string& line, char delimiter = '|') { - lo_val = numeric_limits::min(); - hi_val = numeric_limits::max(); - sequenceNum = 0; - isValid = CP_INVALID; + std::stringstream ss(line); + std::string field; + + auto rangeStart = parseField(ss, delimiter); + auto rangeSize = parseField(ss, delimiter); + BRM::InlineLBIDRange range{rangeStart, rangeSize}; + + auto fileID = parseField(ss, delimiter); + auto blockOffset = parseField(ss, delimiter); + auto hwm = parseField(ss, delimiter); + auto partitionNum = parseField(ss, delimiter); + auto segmentNum = parseField(ss, delimiter); + auto dbRoot = parseField(ss, delimiter); + auto colWid = parseField(ss, delimiter); + auto status = parseField(ss, delimiter); + + auto hiVal = parseField(ss, delimiter); + auto loVal = parseField(ss, delimiter); + auto sequenceNum = parseField(ss, delimiter); + auto isValid = parseField(ss, delimiter); + auto partition = BRM::EMCasualPartition_t{loVal, hiVal, sequenceNum, isValid}; + + return BRM::EMEntry{range, fileID, blockOffset, hwm, partitionNum, + segmentNum, dbRoot, colWid, status, {partition}}; } -} // namespace BRM int main(int argc, char** argv) { - int e; + CLI::App app{BIN_NAME}; + app.description( + "A tool to build Extent Map image file from its text representation. A text representation can be obtained using 'editem -i'" + "display the lock state."); + std::string srcFilename; + std::string dstFilename; + bool debug = false; - int loadSize[3]; + app.add_option("-i,--input-filename", srcFilename, + "Extent Map as its text representation.") + ->required(); + app.add_option("-o,--output-filename", dstFilename, + "Extent Map output image file, default as input-filename.out") + ->default_val(""); + app.add_option("-d,--debug", debug, "Print extra output")->default_val(false); - if (argc < 2) - { - cerr << "filename arg needed" << endl; - return 1; - } + CLI11_PARSE(app, argc, argv); - ifstream in(argv[1]); - e = errno; + ifstream in(srcFilename); + int e = errno; if (!in) { @@ -90,81 +104,43 @@ int main(int argc, char** argv) // Brute force count the number of lines int numEMEntries = 0; - - string line; - - getline(in, line); - - while (!in.eof()) { - numEMEntries++; + string line; getline(in, line); + while (!in.eof()) + { + numEMEntries++; + getline(in, line); + } } - // start at the beginning again... - in.clear(); - in.seekg(0, ios_base::beg); + std::cout << "Number of entries: " << numEMEntries << std::endl; - idbassert(in.good()); - idbassert(in.tellg() == static_cast(0)); - - string outname(argv[1]); - outname += ".out"; - - ofstream out(outname.c_str()); - e = errno; - - if (!out) + if (dstFilename.empty()) { - cerr << "file write error: " << strerror(e) << endl; + dstFilename = srcFilename + ".out"; + } + ofstream outFile(dstFilename); + + BRM::InlineLBIDRange maxLBIDinUse{0, 0}; + + std::ifstream infile(srcFilename); + if (!infile.is_open()) + { + std::cerr << "Can not open file " << srcFilename << std::endl; return 1; } - loadSize[0] = EM_MAGIC_V4; + int loadSize[3]; + loadSize[0] = EM_MAGIC_V5; loadSize[1] = numEMEntries; loadSize[2] = 1; // one free list entry - out.write((char*)&loadSize, (3 * sizeof(int))); + outFile.write((char*)&loadSize, (3 * sizeof(int))); - InlineLBIDRange fl; - fl.start = 0; - // the max lbid is 2^54-1, the size is in units of 1k - fl.size = numeric_limits::max(); - - InlineLBIDRange maxLBIDinUse; - maxLBIDinUse.start = 0; - maxLBIDinUse.size = 0; - - getline(in, line); - - while (!in.eof()) + string line; + while (std::getline(infile, line)) { - EMEntry em; - int64_t v; - tokenizer<> tok(line); - tokenizer<>::iterator beg = tok.begin(); -#if 0 - emSrc[i].range.start - << '\t' << emSrc[i].range.size - << '\t' << emSrc[i].fileID - << '\t' << emSrc[i].blockOffset - << '\t' << emSrc[i].HWM - << '\t' << emSrc[i].partitionNum - << '\t' << emSrc[i].segmentNum - << '\t' << emSrc[i].dbRoot - << '\t' << emSrc[i].colWid - << '\t' << emSrc[i].status - << '\t' << emSrc[i].partition.cprange.hi_val - << '\t' << emSrc[i].partition.cprange.lo_val - << '\t' << emSrc[i].partition.cprange.sequenceNum - << '\t' << (int)(emSrc[i].partition.cprange.isValid) -#endif - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.range.start = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.range.size = v; + BRM::EMEntry em = parseLine(line); if (em.range.start > maxLBIDinUse.start) { @@ -172,66 +148,26 @@ int main(int argc, char** argv) maxLBIDinUse.size = em.range.size; } - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.fileID = v; + if (debug) + { + std::cout << em.range.start << '\t' << em.range.size << '\t' << em.fileID << '\t' << em.blockOffset + << '\t' << em.HWM << '\t' << em.partitionNum << '\t' << em.segmentNum << '\t' << em.dbRoot + << '\t' << em.colWid << '\t' << em.status << '\t' << em.partition.cprange.hiVal << '\t' + << em.partition.cprange.loVal << '\t' << em.partition.cprange.sequenceNum << '\t' + << (short int)(em.partition.cprange.isValid) << std::endl; + } - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.blockOffset = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.HWM = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partitionNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.segmentNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.dbRoot = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.colWid = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.status = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.hi_val = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.lo_val = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.sequenceNum = v; - - v = strtoll(beg->c_str(), 0, 0); - ++beg; - em.partition.cprange.isValid = v; - - out.write((char*)&em, sizeof(em)); - - getline(in, line); + outFile.write((char*)&em, sizeof(em)); } + infile.close(); - fl.start = maxLBIDinUse.start + maxLBIDinUse.size * 1024; - fl.size -= fl.start / 1024; + auto flStart = maxLBIDinUse.start + maxLBIDinUse.size * 1024; + assert(flStart / 1024 <= numeric_limits::max()); + uint32_t flEnd = numeric_limits::max() - flStart / 1024; + BRM::InlineLBIDRange fl{flStart, flEnd}; - out.write((char*)&fl, sizeof(fl)); - - out.close(); - in.close(); + outFile.write((char*)&fl, sizeof(fl)); + outFile.close(); return 0; } diff --git a/versioning/BRM/lock_grabber.cpp b/versioning/BRM/lock_grabber.cpp deleted file mode 100644 index 7c6884bb8..000000000 --- a/versioning/BRM/lock_grabber.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016-2022 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* Takes two params, - * first, which lock use - * second, which side to use (read or write) - * third, lock or unlock it - */ - -#include -#include -#include -#include - -using namespace std; -using namespace rwlock; - -char* name; - -void usage() -{ - std::cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock" << std::endl; - size_t lockId = 0; - for (auto& lockName : RWLockNames) - { - std::cout << " " << lockId++ << "=" << lockName << " "; - } - std::cout << std::endl - << " which_side_to_use: r|w (read or write)" << std::endl - << " lock_or_unlock: l|u (lock or unlock)" << std::endl; - exit(1); -} - -int main(int argc, char** argv) -{ - uint32_t which_lock; // 1-5 - uint32_t which_side; // 0 or 1 - uint32_t lock_unlock; // 0 or 1 - RWLock* rwlock; - - name = argv[0]; - - if (argc != 4) - usage(); - - if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1) - usage(); - - try - { - which_lock = std::stoi(argv[1]); - } - catch (std::exception const& e) - { - std::cerr << "Cannot convert the lock id: " << e.what() << std::endl; - usage(); - } - - if (which_lock >= RWLockNames.size()) - usage(); - - size_t minLockId = (which_lock > 0) ? which_lock : 1; - size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1; - - if (argv[2][0] == 'r') - which_side = 0; - else if (argv[2][0] == 'w') - which_side = 1; - else - usage(); - - if (argv[3][0] == 'l') - lock_unlock = 0; - else if (argv[3][0] == 'u') - lock_unlock = 1; - else - usage(); - - for (size_t i = minLockId; i <= maxLockId; ++i) - { - rwlock = new RWLock(0x10000 * which_lock); - - if (which_side == 0) - { - if (lock_unlock == 0) - rwlock->read_lock(); - else - rwlock->read_unlock(); - } - else if (lock_unlock == 0) - { - rwlock->write_lock(); - } - else - { - rwlock->write_unlock(); - } - - delete rwlock; - } - - return 0; -} diff --git a/versioning/BRM/lock_state.cpp b/versioning/BRM/lock_state.cpp deleted file mode 100644 index 4b5cef63e..000000000 --- a/versioning/BRM/lock_state.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016-2022 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* Takes two params, - * first, which lock use - * second, which side to use (read or write) - * third, lock or unlock it - */ - -#include -#include -#include -#include - -using namespace std; -using namespace rwlock; - -char* name; - -void usage() -{ - std::cout << "Usage " << name << " which_lock_to_use:" << std::endl; - size_t lockId = 0; - for (auto& lockName : RWLockNames) - { - std::cout << " " << lockId++ << "=" << lockName << std::endl; - } - exit(1); -} - -int main(int argc, char** argv) -{ - uint32_t which_lock; // 0-6 - RWLock* rwlock; - LockState state; - - name = argv[0]; - - if (argc != 2) - usage(); - - if (strlen(argv[1]) != 1) - usage(); - - try - { - which_lock = std::stoi(argv[1]); - } - catch (std::exception const& e) - { - std::cerr << "Cannot convert the lock id: " << e.what() << std::endl; - usage(); - } - - if (which_lock >= RWLockNames.size()) - usage(); - - size_t minLockId = (which_lock > 0) ? which_lock : 1; - size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1; - - for (size_t i = minLockId; i <= maxLockId; ++i) - { - rwlock = new RWLock(0x10000 * i); - state = rwlock->getLockState(); - - cout << RWLockNames[i] << " RWLock" << std::endl - << " readers = " << state.reading << std::endl - << " writers = " << state.writing << std::endl - << " readers waiting = " << state.readerswaiting << std::endl - << " writers waiting = " << state.writerswaiting << std::endl - << " mutex locked = " << (int)state.mutexLocked << std::endl; - delete rwlock; - } - - return 0; -} diff --git a/versioning/BRM/shmem_locks.cpp b/versioning/BRM/shmem_locks.cpp new file mode 100644 index 000000000..917e64b81 --- /dev/null +++ b/versioning/BRM/shmem_locks.cpp @@ -0,0 +1,128 @@ +/* Copyright (C) 2016-2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#include +#include + +#include "CLI11.hpp" + +using namespace std; +using namespace rwlock; + +static const char* BIN_NAME = "mcs-load-brm-from-file"; + +std::string getShmemLocksList() +{ + std::ostringstream oss; + size_t lockId = 0; + oss << std::endl; + for (auto& lockName : RWLockNames) + { + oss << " " << lockId++ << "=" << lockName << std::endl; + } + return oss.str(); +} + +int viewLock(uint8_t lockId) +{ + size_t minLockId = (lockId > 0) ? lockId : 1; + size_t maxLockId = (lockId > 0) ? lockId : RWLockNames.size() - 1; + + for (size_t i = minLockId; i <= maxLockId; ++i) + { + auto rwlock = RWLock(0x10000 * i); + auto state = rwlock.getLockState(); + + cout << RWLockNames[i] << " RWLock" << std::endl + << " readers = " << state.reading << std::endl + << " writers = " << state.writing << std::endl + << " readers waiting = " << state.readerswaiting << std::endl + << " writers waiting = " << state.writerswaiting << std::endl + << " mutex locked = " << (int)state.mutexLocked << std::endl; + } + return 0; +} + +int lockOp(size_t minLockId, size_t maxLockId, bool lock, bool read) +{ + for (size_t i = minLockId; i <= maxLockId; ++i) + { + auto rwlock = RWLock(0x10000 * i); + + if (read) + { + if (lock) + rwlock.read_lock(); + else + rwlock.read_unlock(); + } + else if (lock) + { + rwlock.write_lock(); + } + else + { + rwlock.write_unlock(); + } + } + return 0; +} + +int main(int argc, char** argv) +{ + CLI::App app{BIN_NAME}; + app.description( + "A tool to operate or view shmem locks. If neither read nor write operation is specified, the tool will " + "display the lock state."); + uint8_t lockId; + bool debug = false; + bool read = false; + bool write = false; + bool lock = false; + bool unlock = false; + + app.add_option("-i, --lock-id", lockId, "Shmem lock numerical id: " + getShmemLocksList()) + ->expected(0, RWLockNames.size()) + ->required(); + app.add_flag("-r, --read-lock", read, "Use read lock.")->default_val(false); + app.add_flag("-w, --write-lock", write, "Use write lock..")->default_val(false)->excludes("-r"); + app.add_flag("-l, --lock", lock, "Lock the corresponding shmem lock.")->default_val(false); + app.add_flag("-u, --unlock", unlock, "Unlock the corresponding shmem write lock.") + ->default_val(false) + ->excludes("-l"); + app.add_flag("-d,--debug", debug, "Print extra output.")->default_val(false); + + CLI11_PARSE(app, argc, argv); + + if (!read && !write) + { + return viewLock(lockId); + } + + if (lock || unlock) + { + size_t minLockId = (lockId > 0) ? lockId : 1; + size_t maxLockId = (lockId > 0) ? lockId : RWLockNames.size() - 1; + return lockOp(minLockId, maxLockId, lock, read); + } + + return 0; +} + diff --git a/versioning/BRM/tablelockserver.cpp b/versioning/BRM/tablelockserver.cpp index 4d58c59c8..3ec2ebc50 100644 --- a/versioning/BRM/tablelockserver.cpp +++ b/versioning/BRM/tablelockserver.cpp @@ -110,7 +110,7 @@ void TableLockServer::load() if (!in) { ostringstream os; - os << "TableLockServer::load(): could not open the save file" << filename; + os << "TableLockServer::load(): either this is the first cluster start or could not open the save file" << filename; log(os.str(), logging::LOG_TYPE_DEBUG); return; }