database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-04-17 10:37:05 +03:00
Code

fix(syscat): MCOL-5816 23.02 -> 23.10 upgrade issues (#3346) (#3374)

Browse Source 
* feat(BRM,tools): couple utilities to watch/operate shared memory locks and extent map

* feat(BRM,tools): merged two utilities and added some extra dbbuilder output in case of upgrade

* fix(dbbuilder): extra output to log upgrade detection.
This commit is contained in:
drrtuy 2024-12-26 16:26:34 +00:00 committed by GitHub
parent 50a9f82f76
commit dd8fac35ae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 339 additions and 446 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -136,6 +136,7 @@ SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
INCLUDE(columnstore_version)
INCLUDE(misc)
INCLUDE(boost)
INCLUDE(CLI11)
INCLUDE(thrift)
INCLUDE(arrow)

15
cmake/CLI11.cmake Normal file
View File

@ -0,0 +1,15 @@
set(EXTERNAL_INCLUDE_DIR "${CMAKE_BINARY_DIR}/external/include")
file(MAKE_DIRECTORY "${EXTERNAL_INCLUDE_DIR}")
set(C11CLI_URL "https://github.com/CLIUtils/CLI11/releases/download/v2.4.2/CLI11.hpp")
set(C11CLI_HEADER "${EXTERNAL_INCLUDE_DIR}/CLI11.hpp")
set(CLI11_INCLUDE_DIR "${EXTERNAL_INCLUDE_DIR}")
file(DOWNLOAD
${C11CLI_URL}
${C11CLI_HEADER}
SHOW_PROGRESS STATUS download_status
)
add_library(CLI11 INTERFACE)
target_include_directories(CLI11 INTERFACE ${CLI11_INCLUDE_DIR})

3
debian/mariadb-plugin-columnstore.install vendored
View File

@ -37,6 +37,9 @@ usr/bin/mcs-loadbrm.py
usr/bin/mcs_parquet_ddl
usr/bin/mcs_parquet_gen
usr/bin/mcs-stop-controllernode.sh
usr/bin/mcs-load-brm-from-file
usr/bin/mcs-load-em
usr/bin/mcs-shmem-locks
usr/bin/mcsGetConfig
usr/bin/mcsSetConfig
usr/bin/mycnfUpgrade

85
tools/dbbuilder/dbbuilder.cpp
View File

@ -84,12 +84,12 @@ void usage()
const unsigned sysCatalogErr = logging::M0060;
void errorHandler(const unsigned mid, const string& src, const string& msg, bool isCritErr = true)
void messageHandler(const string& src, const string& msg, bool isCritErr = true)
{
logging::LoggingID lid(19);
logging::MessageLog ml(lid);
logging::Message::Args args;
logging::Message message(mid);
logging::Message message(sysCatalogErr);
if (isCritErr)
{
@ -143,28 +143,6 @@ int main(int argc, char* argv[])
return 1;
}
oamModuleInfo_t t;
bool parentOAMModuleFlag = false;
// get local module info; validate running on Active Parent OAM Module
try
{
t = oam.getModuleInfo();
parentOAMModuleFlag = boost::get<4>(t);
}
catch (exception&)
{
parentOAMModuleFlag = true;
}
if (!parentOAMModuleFlag)
{
cerr << "Exiting, dbbuilder can only be run on the Active "
"Parent OAM Module"
<< endl;
return 1;
}
logFile = string(MCSLOGDIR) + "/install/dbbuilder.status";
buildOption = atoi(argv[optind++]);
@ -193,24 +171,21 @@ int main(int argc, char* argv[])
bool isUpgrade = false;
std::unordered_map<int, std::pair<int, bool>> upgradeOidMap;
upgradeOidMap[OID_SYSTABLE_AUXCOLUMNOID] = // This is the candidate OID for the upgrade.
std::make_pair(OID_SYSTABLE_OBJECTID, false); // std::pair::first is the reference OID used
// to fill the candidate OID with default vals
// std::pair::second is set to false by default
// which means the candidate OID will not be
// upgraded. This is set to true in the code
// below if a specific condition is met. Please
// note that the candidate and reference OID
// datatypes and colwidths are assumed to be the
// same in SystemCatalog::upgrade().
upgradeOidMap[OID_SYSCOLUMN_CHARSETNUM] =
std::make_pair(OID_SYSCOLUMN_OBJECTID, false);
upgradeOidMap[OID_SYSTABLE_AUXCOLUMNOID] = // This is the candidate OID for the upgrade.
std::make_pair(OID_SYSTABLE_OBJECTID, false); // std::pair::first is the reference OID used
// to fill the candidate OID with default vals
// std::pair::second is set to false by default
// which means the candidate OID will not be
// upgraded. This is set to true in the code
// below if a specific condition is met. Please
// note that the candidate and reference OID
// datatypes and colwidths are assumed to be the
// same in SystemCatalog::upgrade().
upgradeOidMap[OID_SYSCOLUMN_CHARSETNUM] = std::make_pair(OID_SYSCOLUMN_OBJECTID, false);
std::unordered_map<int, OidTypeT> upgradeOidTypeMap;
upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] =
std::make_pair(CalpontSystemCatalog::INT, 4);
upgradeOidTypeMap[OID_SYSCOLUMN_CHARSETNUM] =
std::make_pair(CalpontSystemCatalog::INT, 4);
upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] = std::make_pair(CalpontSystemCatalog::INT, 4);
upgradeOidTypeMap[OID_SYSCOLUMN_CHARSETNUM] = std::make_pair(CalpontSystemCatalog::INT, 4);
std::unordered_map<int, std::string> upgradeOidDefaultValStrMap;
upgradeOidDefaultValStrMap[OID_SYSTABLE_AUXCOLUMNOID] = "0";
@ -227,11 +202,12 @@ int main(int argc, char* argv[])
(iter->second).second = true;
isUpgrade = true;
}
messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false);
}
if (!isUpgrade)
{
string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile;
string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > " + logFile;
if (canWrite)
{
@ -242,10 +218,10 @@ int main(int argc, char* argv[])
cerr << cmd << endl;
}
errorHandler(sysCatalogErr, "Build system catalog",
"System catalog appears to exist. It will remain intact "
"for reuse. The database is not recreated.",
false);
messageHandler("Build system catalog",
"System catalog appears to exist. It will remain intact "
"for reuse. The database is not recreated.",
false);
return 1;
}
}
@ -276,7 +252,7 @@ int main(int argc, char* argv[])
else
cerr << cmd << endl;
errorHandler(sysCatalogErr, "Build system catalog", ex.what(), false);
messageHandler("Build system catalog", ex.what(), false);
return 1;
}
catch (...)
@ -288,7 +264,7 @@ int main(int argc, char* argv[])
else
cerr << cmd << endl;
errorHandler(sysCatalogErr, "Build system catalog", "HDFS check failed.", false);
messageHandler("Build system catalog", "HDFS check failed.", false);
return 1;
}
@ -305,6 +281,15 @@ int main(int argc, char* argv[])
}
else
{
string cmd = "echo 'Upgrade system catalog' > " + logFile;
if (canWrite)
{
rc = system(cmd.c_str());
}
else
{
cerr << cmd << endl;
}
sysCatalog.upgrade(upgradeOidMap, upgradeOidTypeMap, upgradeOidDefaultValStrMap);
}
@ -326,7 +311,7 @@ int main(int argc, char* argv[])
ostringstream os;
os << "Warning: running " << cmd << " failed. This is usually non-fatal.";
cerr << os.str() << endl;
errorHandler(sysCatalogErr, "Save BRM", os.str());
messageHandler("Save BRM", os.str());
}
}
else
@ -343,7 +328,7 @@ int main(int argc, char* argv[])
else
cerr << cmd << endl;
errorHandler(sysCatalogErr, "Build system catalog", ex.what());
messageHandler("Build system catalog", ex.what());
}
catch (...)
{
@ -355,7 +340,7 @@ int main(int argc, char* argv[])
cerr << cmd << endl;
string err("Caught unknown exception!");
errorHandler(sysCatalogErr, "Build system catalog", err);
messageHandler("Build system catalog", err);
}
}
else

12
versioning/BRM/CMakeLists.txt
View File

@ -116,3 +116,15 @@ add_executable(load_brm ${load_brm_SRCS})
target_link_libraries(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES})
install(TARGETS load_brm DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
add_executable(mcs-load-em load_em.cpp)
target_link_libraries(mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES})
install(TARGETS mcs-load-em DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
add_executable(mcs-load-brm-from-file load_brm_from_file.cpp)
target_link_libraries(mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11)
install(TARGETS mcs-load-brm-from-file DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
add_executable(mcs-shmem-locks shmem_locks.cpp)
target_link_libraries(mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} CLI11)
install(TARGETS mcs-shmem-locks DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

72
versioning/BRM/extentmap.cpp
View File

@ -61,17 +61,10 @@
#include "configcpp.h"
#endif
#define EXTENTMAP_DLLEXPORT
#include "extentmap.h"
#undef EXTENTMAP_DLLEXPORT
#define EM_MAX_SEQNUM 2000000000
#define MAX_IO_RETRIES 10
#define EM_MAGIC_V1 0x76f78b1c
#define EM_MAGIC_V2 0x76f78b1d
#define EM_MAGIC_V3 0x76f78b1e
#define EM_MAGIC_V4 0x76f78b1f
#define EM_MAGIC_V5 0x76f78b20
#ifndef NDEBUG
#define ASSERT(x) \
@ -122,6 +115,12 @@ EMCasualPartition_struct::EMCasualPartition_struct()
isValid = CP_INVALID;
}
EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum,
const char status)
: sequenceNum(seqNum), isValid(status), loVal(lo), hiVal(hi)
{
}
EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum)
{
loVal = lo;
@ -172,6 +171,22 @@ EMEntry::EMEntry()
status = 0;
}
EMEntry::EMEntry(const InlineLBIDRange& range, int fileID, uint32_t blockOffset, HWM_t hwm,
PartitionNumberT partitionNum, uint16_t segmentNum, DBRootT dbRoot, uint16_t colWid,
int16_t status, EMPartition_t partition)
: range(range)
, fileID(fileID)
, blockOffset(blockOffset)
, HWM(hwm)
, partitionNum(partitionNum)
, segmentNum(segmentNum)
, dbRoot(dbRoot)
, colWid(colWid)
, status(status)
, partition(partition)
{
}
EMEntry::EMEntry(const EMEntry& e)
{
range.start = e.range.start;
@ -1682,17 +1697,17 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5)
fEMRBTreeShminfo->currentSize = (emNumElements * EM_RB_TREE_NODE_SIZE) + EM_RB_TREE_EMPTY_SIZE;
#ifdef DUMP_EXTENT_MAP
cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl;
for (const auto& lbidEMEntryPair : *fExtentMapRBTRee)
// for (const auto& lbidEMEntryPair : *fExtentMapRBTRee)
for (auto& lbidEMEntryPair : *fExtentMapRBTree)
{
const EMEntry& emEntry = lbidEMEntryPair.second;
cout << emEntry.start << '\t' << emEntry.size << '\t' << emEntry.fileID << '\t' << emEntry.blockOffset
<< '\t' << emEntry.HWM << '\t' << emEntry.partitionNum << '\t' << emEntry.segmentNum << '\t'
<< emEntry.dbRoot << '\t' << emEntry.status << '\t' << emEntry.partition.cprange.hiVal << '\t'
<< emEntry.partition.cprange.loVal << '\t' << emEntry.partition.cprange.sequenceNum << '\t'
<< (int)(emEntry.partition.cprange.isValid) << endl;
cout << emEntry.range.start << '\t' << emEntry.range.size << '\t' << emEntry.fileID << '\t'
<< emEntry.blockOffset << '\t' << emEntry.HWM << '\t' << emEntry.partitionNum << '\t'
<< emEntry.segmentNum << '\t' << emEntry.dbRoot << '\t' << emEntry.status << '\t'
<< emEntry.partition.cprange.hiVal << '\t' << emEntry.partition.cprange.loVal << '\t'
<< emEntry.partition.cprange.sequenceNum << '\t' << (int)(emEntry.partition.cprange.isValid) << endl;
}
cout << "Free list entries:" << endl;
@ -1700,8 +1715,6 @@ void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5)
for (uint32_t i = 0; i < flNumElements; i++)
cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl;
#endif
}
void ExtentMap::load(const string& filename, bool fixFL)
@ -2745,7 +2758,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co
highestSegNum = emEntry.segmentNum;
highestOffset = emEntry.blockOffset;
} // found extentmap entry for specified OID
} // Loop through extent map entries
} // Loop through extent map entries
DBRootVec dbRootVec(getAllDbRoots());
// 2. for empty DBRoot track hi seg# in user specified part#
@ -2832,7 +2845,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co
}
}
} // loop over em idents
} // current dbroot == target dbroot
} // current dbroot == target dbroot
else
{
// 4. Track hi seg for hwm+1 partition
@ -2853,8 +2866,8 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co
partHighSeg = emEntry.segmentNum;
}
} // current dbroot != target dbroot
} // loop over dbroots
} // (lastExtentIndex >= 0)
} // loop over dbroots
} // (lastExtentIndex >= 0)
//--------------------------------------------------------------------------
// Third Step: Select partition and segment number for new extent
@ -3038,7 +3051,7 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, uint32_t co
newBlockOffset = lastExtent->blockOffset;
}
}
} // lastExtentIndex >= 0
} // lastExtentIndex >= 0
else // Empty DBRoot; use part# that the user specifies
{
if (bHighEmptySegNumSet)
@ -3245,8 +3258,7 @@ void ExtentMap::logAndSetEMIndexReadOnly(const std::string& funcName)
{
fPExtMapIndexImpl_->makeReadOnly();
ostringstream os;
os << "ExtentMap::" << funcName << ": "
<< "Can not update EM Index. EM Index shmem segment is set to"
os << "ExtentMap::" << funcName << ": " << "Can not update EM Index. EM Index shmem segment is set to"
<< " readonly. Please restart Columnstore.";
log(os.str(), logging::LOG_TYPE_CRITICAL);
@ -4123,7 +4135,7 @@ void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo)
emIt = deleteExtent(emIt);
}
} // em iterarors loop
} // em info map loop
} // em info map loop
}
else
{
@ -4193,7 +4205,7 @@ void ExtentMap::deleteEmptyDictStoreExtents(const ExtentsInfoMap_t& extentsInfo)
}
}
} // em iterarors loop
} // em info map loop
} // em info map loop
}
}
//------------------------------------------------------------------------------
@ -4613,11 +4625,11 @@ void ExtentMap::getDbRootHWMInfo(int OID, uint16_t pmNumber, EmDbRootHWMInfo_v&
if (emDbRoot.status == EXTENTUNAVAILABLE)
{
ostringstream oss;
oss << "ExtentMap::getDbRootHWMInfo(): "
<< "OID " << OID << " has HWM extent that is UNAVAILABLE for "
<< "DBRoot" << emDbRoot.dbRoot << "; part#: " << emDbRoot.partitionNum
<< ", seg#: " << emDbRoot.segmentNum << ", fbo: " << emDbRoot.fbo
<< ", localHWM: " << emDbRoot.localHWM << ", lbid: " << emDbRoot.startLbid << endl;
oss << "ExtentMap::getDbRootHWMInfo(): " << "OID " << OID
<< " has HWM extent that is UNAVAILABLE for " << "DBRoot" << emDbRoot.dbRoot
<< "; part#: " << emDbRoot.partitionNum << ", seg#: " << emDbRoot.segmentNum
<< ", fbo: " << emDbRoot.fbo << ", localHWM: " << emDbRoot.localHWM
<< ", lbid: " << emDbRoot.startLbid << endl;
log(oss.str(), logging::LOG_TYPE_CRITICAL);
throw runtime_error(oss.str());
}

10
versioning/BRM/extentmap.h
View File

@ -79,6 +79,12 @@ class IDBDataFile;
namespace BRM
{
#define EM_MAGIC_V1 0x76f78b1c
#define EM_MAGIC_V2 0x76f78b1d
#define EM_MAGIC_V3 0x76f78b1e
#define EM_MAGIC_V4 0x76f78b1f
#define EM_MAGIC_V5 0x76f78b20
using PartitionNumberT = uint32_t;
using DBRootT = uint16_t;
using SegmentT = uint16_t;
@ -158,6 +164,8 @@ struct EMCasualPartition_struct
EXPORT EMCasualPartition_struct();
EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum);
EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum);
EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum,
const char status);
EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em);
EXPORT EMCasualPartition_struct& operator=(const EMCasualPartition_struct& em);
};
@ -182,6 +190,8 @@ struct EMEntry
int16_t status; // extent avail for query or not, or out of service
EMPartition_t partition;
EXPORT EMEntry();
EMEntry(const InlineLBIDRange& range, int fileID, uint32_t bOffset, HWM_t hwm, PartitionNumberT pNum,
uint16_t sNum, DBRootT dRoot, uint16_t cWid, int16_t s, EMPartition_t partition);
EXPORT EMEntry(const EMEntry&);
EXPORT EMEntry& operator=(const EMEntry&);
EXPORT bool operator<(const EMEntry&) const;

248
versioning/BRM/load_brm_from_file.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2014 InfiniDB, Inc.
/* Copyright (C) 2024 MariaDB Corporation, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -20,67 +20,81 @@
* If you re-compile extentmap.cpp to dump the extent map as it loads, you'll get a csv file on stdout.
* Save this to a file and edit it as needed (remove the cruft at the top & bottom for sure). Then use
* this tool to create a binary BRM_saves_em file.
*
* compile with
* g++ -g -Wall -o load_brm_from_file -I$HOME/genii/export/include -I/usr/include/libxml2
* load_brm_from_file.cpp
*
*/
#include <iostream>
#include <stdint.h>
#include <fstream>
#include <cerrno>
#include <string>
#include <cstdlib>
//#define NDEBUG
#include <cassert>
#include <limits>
using namespace std;
#include <boost/tokenizer.hpp>
using namespace boost;
#include "CLI11.hpp"
#include "extentmap.h"
using namespace BRM;
#define EM_MAGIC_V4 0x76f78b1f
static const char* BIN_NAME = "mcs-load-brm-from-file";
namespace BRM
template <typename T>
T parseField(std::stringstream& ss, const char delimiter)
{
EMEntry::EMEntry()
{
fileID = 0;
blockOffset = 0;
HWM = 0;
partitionNum = 0;
segmentNum = 0;
dbRoot = 0;
colWid = 0;
status = 0;
std::string field;
std::getline(ss, field, delimiter);
return std::stoll(field);
}
EMCasualPartition_struct::EMCasualPartition_struct()
BRM::EMEntry parseLine(const std::string& line, char delimiter = '|')
{
lo_val = numeric_limits<int64_t>::min();
hi_val = numeric_limits<int64_t>::max();
sequenceNum = 0;
isValid = CP_INVALID;
std::stringstream ss(line);
std::string field;
auto rangeStart = parseField<int64_t>(ss, delimiter);
auto rangeSize = parseField<uint32_t>(ss, delimiter);
BRM::InlineLBIDRange range{rangeStart, rangeSize};
auto fileID = parseField<int>(ss, delimiter);
auto blockOffset = parseField<uint32_t>(ss, delimiter);
auto hwm = parseField<BRM::HWM_t>(ss, delimiter);
auto partitionNum = parseField<BRM::PartitionNumberT>(ss, delimiter);
auto segmentNum = parseField<uint16_t>(ss, delimiter);
auto dbRoot = parseField<BRM::DBRootT>(ss, delimiter);
auto colWid = parseField<uint16_t>(ss, delimiter);
auto status = parseField<int16_t>(ss, delimiter);
auto hiVal = parseField<int64_t>(ss, delimiter);
auto loVal = parseField<int64_t>(ss, delimiter);
auto sequenceNum = parseField<int32_t>(ss, delimiter);
auto isValid = parseField<char>(ss, delimiter);
auto partition = BRM::EMCasualPartition_t{loVal, hiVal, sequenceNum, isValid};
return BRM::EMEntry{range, fileID, blockOffset, hwm, partitionNum,
segmentNum, dbRoot, colWid, status, {partition}};
}
} // namespace BRM
int main(int argc, char** argv)
{
int e;
CLI::App app{BIN_NAME};
app.description(
"A tool to build Extent Map image file from its text representation. A text representation can be obtained using 'editem -i'"
"display the lock state.");
std::string srcFilename;
std::string dstFilename;
bool debug = false;
int loadSize[3];
app.add_option("-i,--input-filename", srcFilename,
"Extent Map as its text representation.")
->required();
app.add_option("-o,--output-filename", dstFilename,
"Extent Map output image file, default as input-filename.out")
->default_val("");
app.add_option("-d,--debug", debug, "Print extra output")->default_val(false);
if (argc < 2)
{
cerr << "filename arg needed" << endl;
return 1;
}
CLI11_PARSE(app, argc, argv);
ifstream in(argv[1]);
e = errno;
ifstream in(srcFilename);
int e = errno;
if (!in)
{
@ -90,81 +104,43 @@ int main(int argc, char** argv)
// Brute force count the number of lines
int numEMEntries = 0;
string line;
getline(in, line);
while (!in.eof())
{
numEMEntries++;
string line;
getline(in, line);
while (!in.eof())
{
numEMEntries++;
getline(in, line);
}
}
// start at the beginning again...
in.clear();
in.seekg(0, ios_base::beg);
std::cout << "Number of entries: " << numEMEntries << std::endl;
idbassert(in.good());
idbassert(in.tellg() == static_cast<streampos>(0));
string outname(argv[1]);
outname += ".out";
ofstream out(outname.c_str());
e = errno;
if (!out)
if (dstFilename.empty())
{
cerr << "file write error: " << strerror(e) << endl;
dstFilename = srcFilename + ".out";
}
ofstream outFile(dstFilename);
BRM::InlineLBIDRange maxLBIDinUse{0, 0};
std::ifstream infile(srcFilename);
if (!infile.is_open())
{
std::cerr << "Can not open file " << srcFilename << std::endl;
return 1;
}
loadSize[0] = EM_MAGIC_V4;
int loadSize[3];
loadSize[0] = EM_MAGIC_V5;
loadSize[1] = numEMEntries;
loadSize[2] = 1; // one free list entry
out.write((char*)&loadSize, (3 * sizeof(int)));
outFile.write((char*)&loadSize, (3 * sizeof(int)));
InlineLBIDRange fl;
fl.start = 0;
// the max lbid is 2^54-1, the size is in units of 1k
fl.size = numeric_limits<uint32_t>::max();
InlineLBIDRange maxLBIDinUse;
maxLBIDinUse.start = 0;
maxLBIDinUse.size = 0;
getline(in, line);
while (!in.eof())
string line;
while (std::getline(infile, line))
{
EMEntry em;
int64_t v;
tokenizer<> tok(line);
tokenizer<>::iterator beg = tok.begin();
#if 0
emSrc[i].range.start
<< '\t' << emSrc[i].range.size
<< '\t' << emSrc[i].fileID
<< '\t' << emSrc[i].blockOffset
<< '\t' << emSrc[i].HWM
<< '\t' << emSrc[i].partitionNum
<< '\t' << emSrc[i].segmentNum
<< '\t' << emSrc[i].dbRoot
<< '\t' << emSrc[i].colWid
<< '\t' << emSrc[i].status
<< '\t' << emSrc[i].partition.cprange.hi_val
<< '\t' << emSrc[i].partition.cprange.lo_val
<< '\t' << emSrc[i].partition.cprange.sequenceNum
<< '\t' << (int)(emSrc[i].partition.cprange.isValid)
#endif
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.range.start = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.range.size = v;
BRM::EMEntry em = parseLine(line);
if (em.range.start > maxLBIDinUse.start)
{
@ -172,66 +148,26 @@ int main(int argc, char** argv)
maxLBIDinUse.size = em.range.size;
}
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.fileID = v;
if (debug)
{
std::cout << em.range.start << '\t' << em.range.size << '\t' << em.fileID << '\t' << em.blockOffset
<< '\t' << em.HWM << '\t' << em.partitionNum << '\t' << em.segmentNum << '\t' << em.dbRoot
<< '\t' << em.colWid << '\t' << em.status << '\t' << em.partition.cprange.hiVal << '\t'
<< em.partition.cprange.loVal << '\t' << em.partition.cprange.sequenceNum << '\t'
<< (short int)(em.partition.cprange.isValid) << std::endl;
}
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.blockOffset = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.HWM = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.partitionNum = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.segmentNum = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.dbRoot = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.colWid = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.status = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.partition.cprange.hi_val = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.partition.cprange.lo_val = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.partition.cprange.sequenceNum = v;
v = strtoll(beg->c_str(), 0, 0);
++beg;
em.partition.cprange.isValid = v;
out.write((char*)&em, sizeof(em));
getline(in, line);
outFile.write((char*)&em, sizeof(em));
}
infile.close();
fl.start = maxLBIDinUse.start + maxLBIDinUse.size * 1024;
fl.size -= fl.start / 1024;
auto flStart = maxLBIDinUse.start + maxLBIDinUse.size * 1024;
assert(flStart / 1024 <= numeric_limits<uint32_t>::max());
uint32_t flEnd = numeric_limits<uint32_t>::max() - flStart / 1024;
BRM::InlineLBIDRange fl{flStart, flEnd};
out.write((char*)&fl, sizeof(fl));
out.close();
in.close();
outFile.write((char*)&fl, sizeof(fl));
outFile.close();
return 0;
}

118
versioning/BRM/lock_grabber.cpp
View File

@ -1,118 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2016-2022 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/* Takes two params,
* first, which lock use
* second, which side to use (read or write)
* third, lock or unlock it
*/
#include <string>
#include <iostream>
#include <stdlib.h>
#include <rwlock.h>
using namespace std;
using namespace rwlock;
char* name;
void usage()
{
std::cout << "Usage " << name << " which_lock_to_use which_side_to_use lock_or_unlock" << std::endl;
size_t lockId = 0;
for (auto& lockName : RWLockNames)
{
std::cout << " " << lockId++ << "=" << lockName << " ";
}
std::cout << std::endl
<< " which_side_to_use: r|w (read or write)" << std::endl
<< " lock_or_unlock: l|u (lock or unlock)" << std::endl;
exit(1);
}
int main(int argc, char** argv)
{
uint32_t which_lock; // 1-5
uint32_t which_side; // 0 or 1
uint32_t lock_unlock; // 0 or 1
RWLock* rwlock;
name = argv[0];
if (argc != 4)
usage();
if (strlen(argv[1]) != 1 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1)
usage();
try
{
which_lock = std::stoi(argv[1]);
}
catch (std::exception const& e)
{
std::cerr << "Cannot convert the lock id: " << e.what() << std::endl;
usage();
}
if (which_lock >= RWLockNames.size())
usage();
size_t minLockId = (which_lock > 0) ? which_lock : 1;
size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1;
if (argv[2][0] == 'r')
which_side = 0;
else if (argv[2][0] == 'w')
which_side = 1;
else
usage();
if (argv[3][0] == 'l')
lock_unlock = 0;
else if (argv[3][0] == 'u')
lock_unlock = 1;
else
usage();
for (size_t i = minLockId; i <= maxLockId; ++i)
{
rwlock = new RWLock(0x10000 * which_lock);
if (which_side == 0)
{
if (lock_unlock == 0)
rwlock->read_lock();
else
rwlock->read_unlock();
}
else if (lock_unlock == 0)
{
rwlock->write_lock();
}
else
{
rwlock->write_unlock();
}
delete rwlock;
}
return 0;
}

91
versioning/BRM/lock_state.cpp
View File

@ -1,91 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2016-2022 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/* Takes two params,
* first, which lock use
* second, which side to use (read or write)
* third, lock or unlock it
*/
#include <iostream>
#include <string>
#include <stdlib.h>
#include <rwlock.h>
using namespace std;
using namespace rwlock;
char* name;
void usage()
{
std::cout << "Usage " << name << " which_lock_to_use:" << std::endl;
size_t lockId = 0;
for (auto& lockName : RWLockNames)
{
std::cout << " " << lockId++ << "=" << lockName << std::endl;
}
exit(1);
}
int main(int argc, char** argv)
{
uint32_t which_lock; // 0-6
RWLock* rwlock;
LockState state;
name = argv[0];
if (argc != 2)
usage();
if (strlen(argv[1]) != 1)
usage();
try
{
which_lock = std::stoi(argv[1]);
}
catch (std::exception const& e)
{
std::cerr << "Cannot convert the lock id: " << e.what() << std::endl;
usage();
}
if (which_lock >= RWLockNames.size())
usage();
size_t minLockId = (which_lock > 0) ? which_lock : 1;
size_t maxLockId = (which_lock > 0) ? which_lock : RWLockNames.size() - 1;
for (size_t i = minLockId; i <= maxLockId; ++i)
{
rwlock = new RWLock(0x10000 * i);
state = rwlock->getLockState();
cout << RWLockNames[i] << " RWLock" << std::endl
<< " readers = " << state.reading << std::endl
<< " writers = " << state.writing << std::endl
<< " readers waiting = " << state.readerswaiting << std::endl
<< " writers waiting = " << state.writerswaiting << std::endl
<< " mutex locked = " << (int)state.mutexLocked << std::endl;
delete rwlock;
}
return 0;
}

128
versioning/BRM/shmem_locks.cpp Normal file
View File

@ -0,0 +1,128 @@
/* Copyright (C) 2016-2024 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <iostream>
#include <sstream>
#include <string>
#include <stdlib.h>
#include <rwlock.h>
#include "CLI11.hpp"
using namespace std;
using namespace rwlock;
static const char* BIN_NAME = "mcs-load-brm-from-file";
std::string getShmemLocksList()
{
std::ostringstream oss;
size_t lockId = 0;
oss << std::endl;
for (auto& lockName : RWLockNames)
{
oss << " " << lockId++ << "=" << lockName << std::endl;
}
return oss.str();
}
int viewLock(uint8_t lockId)
{
size_t minLockId = (lockId > 0) ? lockId : 1;
size_t maxLockId = (lockId > 0) ? lockId : RWLockNames.size() - 1;
for (size_t i = minLockId; i <= maxLockId; ++i)
{
auto rwlock = RWLock(0x10000 * i);
auto state = rwlock.getLockState();
cout << RWLockNames[i] << " RWLock" << std::endl
<< " readers = " << state.reading << std::endl
<< " writers = " << state.writing << std::endl
<< " readers waiting = " << state.readerswaiting << std::endl
<< " writers waiting = " << state.writerswaiting << std::endl
<< " mutex locked = " << (int)state.mutexLocked << std::endl;
}
return 0;
}
int lockOp(size_t minLockId, size_t maxLockId, bool lock, bool read)
{
for (size_t i = minLockId; i <= maxLockId; ++i)
{
auto rwlock = RWLock(0x10000 * i);
if (read)
{
if (lock)
rwlock.read_lock();
else
rwlock.read_unlock();
}
else if (lock)
{
rwlock.write_lock();
}
else
{
rwlock.write_unlock();
}
}
return 0;
}
int main(int argc, char** argv)
{
CLI::App app{BIN_NAME};
app.description(
"A tool to operate or view shmem locks. If neither read nor write operation is specified, the tool will "
"display the lock state.");
uint8_t lockId;
bool debug = false;
bool read = false;
bool write = false;
bool lock = false;
bool unlock = false;
app.add_option<uint8_t>("-i, --lock-id", lockId, "Shmem lock numerical id: " + getShmemLocksList())
->expected(0, RWLockNames.size())
->required();
app.add_flag("-r, --read-lock", read, "Use read lock.")->default_val(false);
app.add_flag("-w, --write-lock", write, "Use write lock..")->default_val(false)->excludes("-r");
app.add_flag("-l, --lock", lock, "Lock the corresponding shmem lock.")->default_val(false);
app.add_flag("-u, --unlock", unlock, "Unlock the corresponding shmem write lock.")
->default_val(false)
->excludes("-l");
app.add_flag("-d,--debug", debug, "Print extra output.")->default_val(false);
CLI11_PARSE(app, argc, argv);
if (!read && !write)
{
return viewLock(lockId);
}
if (lock || unlock)
{
size_t minLockId = (lockId > 0) ? lockId : 1;
size_t maxLockId = (lockId > 0) ? lockId : RWLockNames.size() - 1;
return lockOp(minLockId, maxLockId, lock, read);
}
return 0;
}

2
versioning/BRM/tablelockserver.cpp
View File

@ -110,7 +110,7 @@ void TableLockServer::load()
if (!in)
{
ostringstream os;
os << "TableLockServer::load(): could not open the save file" << filename;
os << "TableLockServer::load(): either this is the first cluster start or could not open the save file" << filename;
log(os.str(), logging::LOG_TYPE_DEBUG);
return;
}