database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-08 14:22:09 +03:00
Code Activity

Revert blind config read patch

Browse Source
This commit is contained in:
Leonid Fedorov
2022-06-09 09:59:47 +00:00
parent 94ba91c687
commit 184a4b370b
6 changed files with 89 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
storage-manager/src/Config.cpp
View File

@@ -23,6 +23,7 @@
#include <boost/thread/mutex.hpp> #include <boost/thread/mutex.hpp>
#include <boost/property_tree/ini_parser.hpp> #include <boost/property_tree/ini_parser.hpp>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <boost/regex.hpp>
#include <stdlib.h> #include <stdlib.h>
#include <vector> #include <vector>
@@ -30,7 +31,6 @@
#include <sys/stat.h> #include <sys/stat.h>
#include <unistd.h> #include <unistd.h>
#include <iostream> #include <iostream>
#include <regex>
#include "SMLogging.h" #include "SMLogging.h"
@@ -165,13 +165,13 @@ bool Config::reload()
return rtn; return rtn;
} }
string use_envvar(const std::smatch& envvar) string use_envvar(const boost::smatch& envvar)
{ {
char* env = getenv(envvar[1].str().c_str()); char* env = getenv(envvar[1].str().c_str());
return (env ? env : ""); return (env ? env : "");
} }
string expand_numbers(const std::smatch& match) string expand_numbers(const boost::smatch& match)
{ {
long long num = stol(match[1].str()); long long num = stol(match[1].str());
char suffix = (char)::tolower(match[2].str()[0]); char suffix = (char)::tolower(match[2].str()[0]);
@@ -187,20 +187,6 @@ string expand_numbers(const std::smatch& match)
return ::to_string(num); return ::to_string(num);
} }
std::string regex_replace_with_format(const std::string& input,
const std::regex& regex,
std::function<std::string(std::smatch const& match)> format)
{
std::ostringstream output;
std::sregex_iterator begin(input.begin(), input.end(), regex), end;
for(; begin != end; begin++){
output << begin->prefix() << format(*begin);
}
output << input.substr(input.size() - begin->position());
return output.str();
}
string Config::getValue(const string& section, const string& key) const string Config::getValue(const string& section, const string& key) const
{ {
// if we care, move this envvar substition stuff to where the file is loaded // if we care, move this envvar substition stuff to where the file is loaded
@@ -216,15 +202,15 @@ string Config::getValue(const string& section, const string& key) const
} }
s.unlock(); s.unlock();
std::regex re("\\$\\{(.+)\\}"); boost::regex re("\\$\\{(.+)\\}");
ret = regex_replace_with_format(ret, re, use_envvar); ret = boost::regex_replace(ret, re, use_envvar);
// do the numeric substitutions. ex, the suffixes m, k, g // do the numeric substitutions. ex, the suffixes m, k, g
// ehhhhh. going to end up turning a string to a number, to a string, and then to a number again // ehhhhh. going to end up turning a string to a number, to a string, and then to a number again
// don't like that. OTOH who cares. // don't like that. OTOH who cares.
std::regex num_re("^([[:digit:]]+)([mMkKgG])$", std::regex::extended); boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended);
ret = regex_replace_with_format(ret, num_re, expand_numbers); ret = boost::regex_replace(ret, num_re, expand_numbers);
return ret; return ret;
} }

21
storage-manager/src/IOCoordinator.cpp
View File

@@ -25,6 +25,8 @@
#include <stdlib.h> #include <stdlib.h>
#include <errno.h> #include <errno.h>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#define BOOST_SPIRIT_THREADSAFE
#include <boost/property_tree/json_parser.hpp>
#include <iostream> #include <iostream>
#include "checks.h" #include "checks.h"
#include "vlarray.h" #include "vlarray.h"
@@ -1264,10 +1266,9 @@ boost::shared_array<uint8_t> IOCoordinator::mergeJournal(const char* object, con
boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead);
stringstream ss; stringstream ss;
ss << headertxt.get(); ss << headertxt.get();
boost::property_tree::ptree header;
nlohmann::json header = nlohmann::json::parse(ss); boost::property_tree::json_parser::read_json(ss, header);
assert(header.get<int>("version") == 1);
assert(header["version"] == 1);
// start processing the entries // start processing the entries
while (1) while (1)
@@ -1352,9 +1353,9 @@ int IOCoordinator::mergeJournalInMem(boost::shared_array<uint8_t>& objData, size
boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead);
stringstream ss; stringstream ss;
ss << headertxt.get(); ss << headertxt.get();
boost::property_tree::ptree header;
nlohmann::json header = nlohmann::json::parse(ss); boost::property_tree::json_parser::read_json(ss, header);
assert(header["version"] == 1); assert(header.get<int>("version") == 1);
// read the journal file into memory // read the journal file into memory
size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead; size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead;
@@ -1432,9 +1433,9 @@ int IOCoordinator::mergeJournalInMem_bigJ(boost::shared_array<uint8_t>& objData,
boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead);
stringstream ss; stringstream ss;
ss << headertxt.get(); ss << headertxt.get();
boost::property_tree::ptree header;
nlohmann::json header = nlohmann::json::parse(ss); boost::property_tree::json_parser::read_json(ss, header);
assert(header["version"] == 1); assert(header.get<int>("version") == 1);
// start processing the entries // start processing the entries
while (1) while (1)

107
storage-manager/src/MetadataFile.cpp
View File

@@ -20,17 +20,20 @@
*/ */
#include "MetadataFile.h" #include "MetadataFile.h"
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#define BOOST_SPIRIT_THREADSAFE
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/foreach.hpp>
#include <boost/uuid/uuid.hpp> #include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp> #include <boost/uuid/uuid_io.hpp>
#include <boost/uuid/random_generator.hpp> #include <boost/uuid/random_generator.hpp>
#include <cstdio>
#include <unistd.h> #include <unistd.h>
#include <fstream>
#define max(x, y) (x > y ? x : y) #define max(x, y) (x > y ? x : y)
#define min(x, y) (x < y ? x : y) #define min(x, y) (x < y ? x : y)
using namespace std; using namespace std;
namespace bpt = boost::property_tree;
namespace bf = boost::filesystem; namespace bf = boost::filesystem;
namespace namespace
@@ -117,13 +120,12 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename)
{ {
if (boost::filesystem::exists(mFilename)) if (boost::filesystem::exists(mFilename))
{ {
std::ifstream i(mFilename.string()); jsontree.reset(new bpt::ptree());
jsontree.reset(new nlohmann::json); boost::property_tree::read_json(mFilename.string(), *jsontree);
i >> *jsontree;
jsonCache.put(mFilename, jsontree); jsonCache.put(mFilename, jsontree);
s.unlock(); s.unlock();
mVersion = 1; mVersion = 1;
mRevision = (*jsontree)["revision"]; mRevision = jsontree->get<int>("revision");
} }
else else
{ {
@@ -138,7 +140,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename)
{ {
s.unlock(); s.unlock();
mVersion = 1; mVersion = 1;
mRevision = (*jsontree)["revision"];; mRevision = jsontree->get<int>("revision");
} }
++metadataFilesAccessed; ++metadataFilesAccessed;
} }
@@ -160,13 +162,12 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t,
if (boost::filesystem::exists(mFilename)) if (boost::filesystem::exists(mFilename))
{ {
_exists = true; _exists = true;
jsontree.reset(new nlohmann::json); jsontree.reset(new bpt::ptree());
std::ifstream i(mFilename.string()); boost::property_tree::read_json(mFilename.string(), *jsontree);
i >> *jsontree;
jsonCache.put(mFilename, jsontree); jsonCache.put(mFilename, jsontree);
s.unlock(); s.unlock();
mVersion = 1; mVersion = 1;
mRevision = (*jsontree)["revision"]; mRevision = jsontree->get<int>("revision");
} }
else else
{ {
@@ -181,7 +182,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t,
s.unlock(); s.unlock();
_exists = true; _exists = true;
mVersion = 1; mVersion = 1;
mRevision = (*jsontree)["revision"]; mRevision = jsontree->get<int>("revision");
} }
++metadataFilesAccessed; ++metadataFilesAccessed;
} }
@@ -192,10 +193,11 @@ MetadataFile::~MetadataFile()
void MetadataFile::makeEmptyJsonTree() void MetadataFile::makeEmptyJsonTree()
{ {
jsontree.reset(new nlohmann::json); jsontree.reset(new bpt::ptree());
(*jsontree)["version"] = mVersion; boost::property_tree::ptree objs;
(*jsontree)["revision"] = mRevision; jsontree->put("version", mVersion);
(*jsontree)["objects"] = nlohmann::json::array(); jsontree->put("revision", mRevision);
jsontree->add_child("objects", objs);
} }
void MetadataFile::printKPIs() void MetadataFile::printKPIs()
@@ -217,11 +219,11 @@ size_t MetadataFile::getLength() const
{ {
size_t totalSize = 0; size_t totalSize = 0;
auto &objects = (*jsontree)["objects"]; auto& objects = jsontree->get_child("objects");
if (!objects.empty()) if (!objects.empty())
{ {
auto& lastObject = objects.back(); auto& lastObject = objects.back().second;
totalSize = lastObject["offset"].get<off_t>() + lastObject["length"].get<size_t>(); totalSize = lastObject.get<off_t>("offset") + lastObject.get<size_t>("length");
} }
return totalSize; return totalSize;
} }
@@ -241,9 +243,10 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length) c
rather than write a new alg. rather than write a new alg.
*/ */
set<metadataObject> mObjects; set<metadataObject> mObjects;
for(const auto &v : (*jsontree)["objects"]) BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
{ {
mObjects.insert(metadataObject(v["offset"], v["length"], v["key"])); mObjects.insert(metadataObject(v.second.get<uint64_t>("offset"), v.second.get<uint64_t>("length"),
v.second.get<string>("key")));
} }
if (mObjects.size() == 0) if (mObjects.size() == 0)
@@ -285,20 +288,20 @@ metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& fi
// //
metadataObject addObject; metadataObject addObject;
auto& objects = (*jsontree)["objects"]; auto& objects = jsontree->get_child("objects");
if (!objects.empty()) if (!objects.empty())
{ {
auto& lastObject = objects.back(); auto& lastObject = objects.back().second;
addObject.offset = lastObject["offset"].get<off_t>() + mpConfig->mObjectSize; addObject.offset = lastObject.get<off_t>("offset") + mpConfig->mObjectSize;
} }
addObject.length = length; addObject.length = length;
addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length); addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length);
nlohmann::json object = nlohmann::json::object(); boost::property_tree::ptree object;
object["offset"] = addObject.offset; object.put("offset", addObject.offset);
object["length"] = addObject.length; object.put("length", addObject.length);
object["key"] = addObject.key; object.put("key", addObject.key);
objects.push_back(object); objects.push_back(make_pair("", object));
return addObject; return addObject;
} }
@@ -309,8 +312,7 @@ int MetadataFile::writeMetadata()
if (!boost::filesystem::exists(mFilename.parent_path())) if (!boost::filesystem::exists(mFilename.parent_path()))
boost::filesystem::create_directories(mFilename.parent_path()); boost::filesystem::create_directories(mFilename.parent_path());
std::ofstream o(mFilename.c_str()); write_json(mFilename.string(), *jsontree);
o << *jsontree;
_exists = true; _exists = true;
boost::unique_lock<boost::mutex> s(jsonCache.getMutex()); boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
@@ -322,13 +324,13 @@ int MetadataFile::writeMetadata()
bool MetadataFile::getEntry(off_t offset, metadataObject* out) const bool MetadataFile::getEntry(off_t offset, metadataObject* out) const
{ {
metadataObject addObject; metadataObject addObject;
for(auto &v: (*jsontree)["objects"]) BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
{ {
if (v["offset"].get<off_t>() == offset) if (v.second.get<off_t>("offset") == offset)
{ {
out->offset = offset; out->offset = offset;
out->length = v["length"].get<size_t>(); out->length = v.second.get<size_t>("length");
out->key = v["key"]; out->key = v.second.get<string>("key");
return true; return true;
} }
} }
@@ -337,10 +339,10 @@ bool MetadataFile::getEntry(off_t offset, metadataObject* out) const
void MetadataFile::removeEntry(off_t offset) void MetadataFile::removeEntry(off_t offset)
{ {
auto& objects = (*jsontree)["objects"]; bpt::ptree& objects = jsontree->get_child("objects");
for (auto it = objects.begin(); it != objects.end(); ++it) for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it)
{ {
if ((*it)["offset"].get<off_t>() == offset) if (it->second.get<off_t>("offset") == offset)
{ {
objects.erase(it); objects.erase(it);
break; break;
@@ -350,7 +352,7 @@ void MetadataFile::removeEntry(off_t offset)
void MetadataFile::removeAllEntries() void MetadataFile::removeAllEntries()
{ {
(*jsontree)["objects"] = nlohmann::json::array(); jsontree->get_child("objects").clear();
} }
void MetadataFile::deletedMeta(const bf::path& p) void MetadataFile::deletedMeta(const bf::path& p)
@@ -454,21 +456,21 @@ void MetadataFile::setLengthInKey(string& key, size_t newLength)
void MetadataFile::printObjects() const void MetadataFile::printObjects() const
{ {
for (auto& v : (*jsontree)["objects"]) BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
{ {
printf("Name: %s Length: %zu Offset: %lld\n", v["key"].get<std::string>().c_str(), printf("Name: %s Length: %zu Offset: %lld\n", v.second.get<string>("key").c_str(),
v["length"].get<size_t>(), (long long)v["offset"].get<off_t>()); v.second.get<size_t>("length"), (long long)v.second.get<off_t>("offset"));
} }
} }
void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLength) void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLength)
{ {
for (auto& v : (*jsontree)["objects"]) for (auto& v : jsontree->get_child("objects"))
{ {
if (v["offset"].get<off_t>() == offset) if (v.second.get<off_t>("offset") == offset)
{ {
v["key"] = newName; v.second.put("key", newName);
v["length"] = newLength; v.second.put("length", newLength);
return; return;
} }
} }
@@ -480,11 +482,11 @@ void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLe
void MetadataFile::updateEntryLength(off_t offset, size_t newLength) void MetadataFile::updateEntryLength(off_t offset, size_t newLength)
{ {
for (auto& v : (*jsontree)["objects"]) for (auto& v : jsontree->get_child("objects"))
{ {
if (v["offset"].get<off_t>() == offset) if (v.second.get<off_t>("offset") == offset)
{ {
v["length"] = newLength; v.second.put("length", newLength);
return; return;
} }
} }
@@ -496,12 +498,11 @@ void MetadataFile::updateEntryLength(off_t offset, size_t newLength)
off_t MetadataFile::getMetadataNewObjectOffset() off_t MetadataFile::getMetadataNewObjectOffset()
{ {
auto& objects = (*jsontree)["objects"]; auto& objects = jsontree->get_child("objects");
if (objects.empty()) if (objects.empty())
return 0; return 0;
auto& lastObject = jsontree->get_child("objects").back().second;
auto& lastObject = objects.back(); return lastObject.get<off_t>("offset") + lastObject.get<size_t>("length");
return lastObject["offset"].get<off_t>() + lastObject["length"].get<size_t>();
} }
metadataObject::metadataObject() : offset(0), length(0) metadataObject::metadataObject() : offset(0), length(0)

4
storage-manager/src/MetadataFile.h
View File

@@ -28,8 +28,6 @@
#include <unordered_map> #include <unordered_map>
#include <boost/filesystem/path.hpp> #include <boost/filesystem/path.hpp>
#include <utils/json/json.hpp>
namespace storagemanager namespace storagemanager
{ {
struct metadataObject struct metadataObject
@@ -112,7 +110,7 @@ class MetadataFile
static void printKPIs(); static void printKPIs();
typedef boost::shared_ptr<nlohmann::json> Jsontree_t; typedef boost::shared_ptr<boost::property_tree::ptree> Jsontree_t;
private: private:
MetadataConfig* mpConfig; MetadataConfig* mpConfig;

16
storage-manager/src/Replicator.cpp
View File

@@ -27,12 +27,12 @@
#include <errno.h> #include <errno.h>
#include <sys/sendfile.h> #include <sys/sendfile.h>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#define BOOST_SPIRIT_THREADSAFE
#include <boost/property_tree/json_parser.hpp>
#include <boost/shared_array.hpp> #include <boost/shared_array.hpp>
#include <boost/format.hpp> #include <boost/format.hpp>
#include <iostream> #include <iostream>
#include <utils/json/json.hpp>
using namespace std; using namespace std;
namespace namespace
@@ -279,14 +279,12 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u
stringstream ss; stringstream ss;
ss << headertxt.get(); ss << headertxt.get();
headerRollback = headertxt.get(); headerRollback = headertxt.get();
nlohmann::json header; boost::property_tree::ptree header;
try try
{ {
header = nlohmann::json::parse(ss); boost::property_tree::json_parser::read_json(ss, header);
} }
catch (const boost::property_tree::json_parser::json_parser_error& e)
catch (const nlohmann::json::exception& e)
{ {
mpLogger->log(LOG_CRIT, "%s", e.what()); mpLogger->log(LOG_CRIT, "%s", e.what());
errno = EIO; errno = EIO;
@@ -298,8 +296,8 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u
errno = EIO; errno = EIO;
return -1; return -1;
} }
assert(header["version"] == 1); assert(header.get<int>("version") == 1);
uint64_t currentMaxOffset = header["max_offset"]; uint64_t currentMaxOffset = header.get<uint64_t>("max_offset");
if (thisEntryMaxOffset > currentMaxOffset) if (thisEntryMaxOffset > currentMaxOffset)
{ {
bHeaderChanged = true; bHeaderChanged = true;

18
storage-manager/src/S3Storage.cpp
View File

@@ -26,9 +26,9 @@
#include <boost/uuid/uuid.hpp> #include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp> #include <boost/uuid/uuid_io.hpp>
#include <boost/uuid/random_generator.hpp> #include <boost/uuid/random_generator.hpp>
#define BOOST_SPIRIT_THREADSAFE
#include "utils/json/json.hpp" #include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include "Utilities.h" #include "Utilities.h"
using namespace std; using namespace std;
@@ -258,12 +258,12 @@ bool S3Storage::getCredentialsFromMetadataEC2()
logger->log(LOG_ERR, "CURL fail %u", curl_res); logger->log(LOG_ERR, "CURL fail %u", curl_res);
return false; return false;
} }
stringstream credentials(readBuffer);
nlohmann::json pt = nlohmann::json::parse(readBuffer); boost::property_tree::ptree pt;
key = pt["AccessKeyId"]; boost::property_tree::read_json(credentials, pt);
secret = pt["SecretAccessKey"]; key = pt.get<string>("AccessKeyId");
token = pt["Token"]; secret = pt.get<string>("SecretAccessKey");
token = pt.get<string>("Token");
// logger->log(LOG_INFO, "S3Storage: key = %s secret = %s token = // logger->log(LOG_INFO, "S3Storage: key = %s secret = %s token =
// %s",key.c_str(),secret.c_str(),token.c_str()); // %s",key.c_str(),secret.c_str(),token.c_str());