diff --git a/storage-manager/src/Config.cpp b/storage-manager/src/Config.cpp index fa9873484..f634ececb 100644 --- a/storage-manager/src/Config.cpp +++ b/storage-manager/src/Config.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -30,7 +31,6 @@ #include #include #include -#include #include "SMLogging.h" @@ -165,13 +165,13 @@ bool Config::reload() return rtn; } -string use_envvar(const std::smatch& envvar) +string use_envvar(const boost::smatch& envvar) { char* env = getenv(envvar[1].str().c_str()); return (env ? env : ""); } -string expand_numbers(const std::smatch& match) +string expand_numbers(const boost::smatch& match) { long long num = stol(match[1].str()); char suffix = (char)::tolower(match[2].str()[0]); @@ -187,20 +187,6 @@ string expand_numbers(const std::smatch& match) return ::to_string(num); } -std::string regex_replace_with_format(const std::string& input, - const std::regex& regex, - std::function format) -{ - - std::ostringstream output; - std::sregex_iterator begin(input.begin(), input.end(), regex), end; - for(; begin != end; begin++){ - output << begin->prefix() << format(*begin); - } - output << input.substr(input.size() - begin->position()); - return output.str(); -} - string Config::getValue(const string& section, const string& key) const { // if we care, move this envvar substition stuff to where the file is loaded @@ -216,15 +202,15 @@ string Config::getValue(const string& section, const string& key) const } s.unlock(); - std::regex re("\\$\\{(.+)\\}"); + boost::regex re("\\$\\{(.+)\\}"); - ret = regex_replace_with_format(ret, re, use_envvar); + ret = boost::regex_replace(ret, re, use_envvar); // do the numeric substitutions. ex, the suffixes m, k, g // ehhhhh. going to end up turning a string to a number, to a string, and then to a number again // don't like that. OTOH who cares. - std::regex num_re("^([[:digit:]]+)([mMkKgG])$", std::regex::extended); - ret = regex_replace_with_format(ret, num_re, expand_numbers); + boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended); + ret = boost::regex_replace(ret, num_re, expand_numbers); return ret; } diff --git a/storage-manager/src/IOCoordinator.cpp b/storage-manager/src/IOCoordinator.cpp index 52c1567f6..b17d502b2 100644 --- a/storage-manager/src/IOCoordinator.cpp +++ b/storage-manager/src/IOCoordinator.cpp @@ -25,6 +25,8 @@ #include #include #include +#define BOOST_SPIRIT_THREADSAFE +#include #include #include "checks.h" #include "vlarray.h" @@ -1264,10 +1266,9 @@ boost::shared_array IOCoordinator::mergeJournal(const char* object, con boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - - nlohmann::json header = nlohmann::json::parse(ss); - - assert(header["version"] == 1); + boost::property_tree::ptree header; + boost::property_tree::json_parser::read_json(ss, header); + assert(header.get("version") == 1); // start processing the entries while (1) @@ -1352,9 +1353,9 @@ int IOCoordinator::mergeJournalInMem(boost::shared_array& objData, size boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - - nlohmann::json header = nlohmann::json::parse(ss); - assert(header["version"] == 1); + boost::property_tree::ptree header; + boost::property_tree::json_parser::read_json(ss, header); + assert(header.get("version") == 1); // read the journal file into memory size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead; @@ -1432,9 +1433,9 @@ int IOCoordinator::mergeJournalInMem_bigJ(boost::shared_array& objData, boost::shared_array headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead); stringstream ss; ss << headertxt.get(); - - nlohmann::json header = nlohmann::json::parse(ss); - assert(header["version"] == 1); + boost::property_tree::ptree header; + boost::property_tree::json_parser::read_json(ss, header); + assert(header.get("version") == 1); // start processing the entries while (1) diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index 5386beae5..e45c95173 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -20,17 +20,20 @@ */ #include "MetadataFile.h" #include +#define BOOST_SPIRIT_THREADSAFE +#include +#include +#include #include #include #include -#include #include -#include #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) using namespace std; +namespace bpt = boost::property_tree; namespace bf = boost::filesystem; namespace @@ -117,13 +120,12 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename) { if (boost::filesystem::exists(mFilename)) { - std::ifstream i(mFilename.string()); - jsontree.reset(new nlohmann::json); - i >> *jsontree; + jsontree.reset(new bpt::ptree()); + boost::property_tree::read_json(mFilename.string(), *jsontree); jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; - mRevision = (*jsontree)["revision"]; + mRevision = jsontree->get("revision"); } else { @@ -138,7 +140,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename) { s.unlock(); mVersion = 1; - mRevision = (*jsontree)["revision"];; + mRevision = jsontree->get("revision"); } ++metadataFilesAccessed; } @@ -160,13 +162,12 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, if (boost::filesystem::exists(mFilename)) { _exists = true; - jsontree.reset(new nlohmann::json); - std::ifstream i(mFilename.string()); - i >> *jsontree; + jsontree.reset(new bpt::ptree()); + boost::property_tree::read_json(mFilename.string(), *jsontree); jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; - mRevision = (*jsontree)["revision"]; + mRevision = jsontree->get("revision"); } else { @@ -181,7 +182,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, s.unlock(); _exists = true; mVersion = 1; - mRevision = (*jsontree)["revision"]; + mRevision = jsontree->get("revision"); } ++metadataFilesAccessed; } @@ -192,10 +193,11 @@ MetadataFile::~MetadataFile() void MetadataFile::makeEmptyJsonTree() { - jsontree.reset(new nlohmann::json); - (*jsontree)["version"] = mVersion; - (*jsontree)["revision"] = mRevision; - (*jsontree)["objects"] = nlohmann::json::array(); + jsontree.reset(new bpt::ptree()); + boost::property_tree::ptree objs; + jsontree->put("version", mVersion); + jsontree->put("revision", mRevision); + jsontree->add_child("objects", objs); } void MetadataFile::printKPIs() @@ -217,11 +219,11 @@ size_t MetadataFile::getLength() const { size_t totalSize = 0; - auto &objects = (*jsontree)["objects"]; + auto& objects = jsontree->get_child("objects"); if (!objects.empty()) { - auto& lastObject = objects.back(); - totalSize = lastObject["offset"].get() + lastObject["length"].get(); + auto& lastObject = objects.back().second; + totalSize = lastObject.get("offset") + lastObject.get("length"); } return totalSize; } @@ -241,9 +243,10 @@ vector MetadataFile::metadataRead(off_t offset, size_t length) c rather than write a new alg. */ set mObjects; - for(const auto &v : (*jsontree)["objects"]) + BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) { - mObjects.insert(metadataObject(v["offset"], v["length"], v["key"])); + mObjects.insert(metadataObject(v.second.get("offset"), v.second.get("length"), + v.second.get("key"))); } if (mObjects.size() == 0) @@ -285,20 +288,20 @@ metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& fi // metadataObject addObject; - auto& objects = (*jsontree)["objects"]; + auto& objects = jsontree->get_child("objects"); if (!objects.empty()) { - auto& lastObject = objects.back(); - addObject.offset = lastObject["offset"].get() + mpConfig->mObjectSize; + auto& lastObject = objects.back().second; + addObject.offset = lastObject.get("offset") + mpConfig->mObjectSize; } addObject.length = length; addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length); - nlohmann::json object = nlohmann::json::object(); - object["offset"] = addObject.offset; - object["length"] = addObject.length; - object["key"] = addObject.key; - objects.push_back(object); + boost::property_tree::ptree object; + object.put("offset", addObject.offset); + object.put("length", addObject.length); + object.put("key", addObject.key); + objects.push_back(make_pair("", object)); return addObject; } @@ -309,8 +312,7 @@ int MetadataFile::writeMetadata() if (!boost::filesystem::exists(mFilename.parent_path())) boost::filesystem::create_directories(mFilename.parent_path()); - std::ofstream o(mFilename.c_str()); - o << *jsontree; + write_json(mFilename.string(), *jsontree); _exists = true; boost::unique_lock s(jsonCache.getMutex()); @@ -322,13 +324,13 @@ int MetadataFile::writeMetadata() bool MetadataFile::getEntry(off_t offset, metadataObject* out) const { metadataObject addObject; - for(auto &v: (*jsontree)["objects"]) + BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) { - if (v["offset"].get() == offset) + if (v.second.get("offset") == offset) { out->offset = offset; - out->length = v["length"].get(); - out->key = v["key"]; + out->length = v.second.get("length"); + out->key = v.second.get("key"); return true; } } @@ -337,10 +339,10 @@ bool MetadataFile::getEntry(off_t offset, metadataObject* out) const void MetadataFile::removeEntry(off_t offset) { - auto& objects = (*jsontree)["objects"]; - for (auto it = objects.begin(); it != objects.end(); ++it) + bpt::ptree& objects = jsontree->get_child("objects"); + for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it) { - if ((*it)["offset"].get() == offset) + if (it->second.get("offset") == offset) { objects.erase(it); break; @@ -350,7 +352,7 @@ void MetadataFile::removeEntry(off_t offset) void MetadataFile::removeAllEntries() { - (*jsontree)["objects"] = nlohmann::json::array(); + jsontree->get_child("objects").clear(); } void MetadataFile::deletedMeta(const bf::path& p) @@ -454,21 +456,21 @@ void MetadataFile::setLengthInKey(string& key, size_t newLength) void MetadataFile::printObjects() const { - for (auto& v : (*jsontree)["objects"]) + BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects")) { - printf("Name: %s Length: %zu Offset: %lld\n", v["key"].get().c_str(), - v["length"].get(), (long long)v["offset"].get()); + printf("Name: %s Length: %zu Offset: %lld\n", v.second.get("key").c_str(), + v.second.get("length"), (long long)v.second.get("offset")); } } void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLength) { - for (auto& v : (*jsontree)["objects"]) + for (auto& v : jsontree->get_child("objects")) { - if (v["offset"].get() == offset) + if (v.second.get("offset") == offset) { - v["key"] = newName; - v["length"] = newLength; + v.second.put("key", newName); + v.second.put("length", newLength); return; } } @@ -480,11 +482,11 @@ void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLe void MetadataFile::updateEntryLength(off_t offset, size_t newLength) { - for (auto& v : (*jsontree)["objects"]) + for (auto& v : jsontree->get_child("objects")) { - if (v["offset"].get() == offset) + if (v.second.get("offset") == offset) { - v["length"] = newLength; + v.second.put("length", newLength); return; } } @@ -496,12 +498,11 @@ void MetadataFile::updateEntryLength(off_t offset, size_t newLength) off_t MetadataFile::getMetadataNewObjectOffset() { - auto& objects = (*jsontree)["objects"]; + auto& objects = jsontree->get_child("objects"); if (objects.empty()) return 0; - - auto& lastObject = objects.back(); - return lastObject["offset"].get() + lastObject["length"].get(); + auto& lastObject = jsontree->get_child("objects").back().second; + return lastObject.get("offset") + lastObject.get("length"); } metadataObject::metadataObject() : offset(0), length(0) diff --git a/storage-manager/src/MetadataFile.h b/storage-manager/src/MetadataFile.h index 94d65c404..5c302c618 100644 --- a/storage-manager/src/MetadataFile.h +++ b/storage-manager/src/MetadataFile.h @@ -28,8 +28,6 @@ #include #include -#include - namespace storagemanager { struct metadataObject @@ -112,7 +110,7 @@ class MetadataFile static void printKPIs(); - typedef boost::shared_ptr Jsontree_t; + typedef boost::shared_ptr Jsontree_t; private: MetadataConfig* mpConfig; diff --git a/storage-manager/src/Replicator.cpp b/storage-manager/src/Replicator.cpp index 2a9241306..10fc07002 100644 --- a/storage-manager/src/Replicator.cpp +++ b/storage-manager/src/Replicator.cpp @@ -27,12 +27,12 @@ #include #include #include +#define BOOST_SPIRIT_THREADSAFE +#include #include #include #include -#include - using namespace std; namespace @@ -279,14 +279,12 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u stringstream ss; ss << headertxt.get(); headerRollback = headertxt.get(); - nlohmann::json header; - + boost::property_tree::ptree header; try { - header = nlohmann::json::parse(ss); + boost::property_tree::json_parser::read_json(ss, header); } - - catch (const nlohmann::json::exception& e) + catch (const boost::property_tree::json_parser::json_parser_error& e) { mpLogger->log(LOG_CRIT, "%s", e.what()); errno = EIO; @@ -298,8 +296,8 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u errno = EIO; return -1; } - assert(header["version"] == 1); - uint64_t currentMaxOffset = header["max_offset"]; + assert(header.get("version") == 1); + uint64_t currentMaxOffset = header.get("max_offset"); if (thisEntryMaxOffset > currentMaxOffset) { bHeaderChanged = true; diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index d8b4de33a..22560b41a 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -26,9 +26,9 @@ #include #include #include - -#include "utils/json/json.hpp" - +#define BOOST_SPIRIT_THREADSAFE +#include +#include #include "Utilities.h" using namespace std; @@ -258,12 +258,12 @@ bool S3Storage::getCredentialsFromMetadataEC2() logger->log(LOG_ERR, "CURL fail %u", curl_res); return false; } - - nlohmann::json pt = nlohmann::json::parse(readBuffer); - key = pt["AccessKeyId"]; - secret = pt["SecretAccessKey"]; - token = pt["Token"]; - + stringstream credentials(readBuffer); + boost::property_tree::ptree pt; + boost::property_tree::read_json(credentials, pt); + key = pt.get("AccessKeyId"); + secret = pt.get("SecretAccessKey"); + token = pt.get("Token"); // logger->log(LOG_INFO, "S3Storage: key = %s secret = %s token = // %s",key.c_str(),secret.c_str(),token.c_str());