C++20 fixes

storage-manager/src/Config.cpp

@@ -23,7 +23,6 @@
 #include <boost/thread/mutex.hpp>
 #include <boost/property_tree/ini_parser.hpp>
 #include <boost/filesystem.hpp>
-#include <boost/regex.hpp>
 
 #include <stdlib.h>
 #include <vector>
@@ -31,6 +30,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <iostream>
+#include <regex>
 
 #include "SMLogging.h"
 
@@ -165,13 +165,13 @@ bool Config::reload()
   return rtn;
 }
 
-string use_envvar(const boost::smatch& envvar)
+string use_envvar(const std::smatch& envvar)
 {
   char* env = getenv(envvar[1].str().c_str());
   return (env ? env : "");
 }
 
-string expand_numbers(const boost::smatch& match)
+string expand_numbers(const std::smatch& match)
 {
   long long num = stol(match[1].str());
   char suffix = (char)::tolower(match[2].str()[0]);
@@ -187,6 +187,20 @@ string expand_numbers(const boost::smatch& match)
   return ::to_string(num);
 }
 
+std::string regex_replace_with_format(const std::string& input,
+                                      const std::regex& regex,
+                                      std::function<std::string(std::smatch const& match)> format)
+{
+
+    std::ostringstream output;
+    std::sregex_iterator begin(input.begin(), input.end(), regex), end;
+    for(; begin != end; begin++){
+        output << begin->prefix() << format(*begin);
+    }
+    output << input.substr(input.size() - begin->position());
+    return output.str();
+}
+
 string Config::getValue(const string& section, const string& key) const
 {
   // if we care, move this envvar substition stuff to where the file is loaded
@@ -202,15 +216,15 @@ string Config::getValue(const string& section, const string& key) const
   }
   s.unlock();
 
-  boost::regex re("\\$\\{(.+)\\}");
+  std::regex re("\\$\\{(.+)\\}");
 
-  ret = boost::regex_replace(ret, re, use_envvar);
+  ret = regex_replace_with_format(ret, re, use_envvar);
 
   // do the numeric substitutions.  ex, the suffixes m, k, g
   // ehhhhh.  going to end up turning a string to a number, to a string, and then to a number again
   // don't like that.  OTOH who cares.
-  boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended);
-  ret = boost::regex_replace(ret, num_re, expand_numbers);
+  std::regex num_re("^([[:digit:]]+)([mMkKgG])$", std::regex::extended);
+  ret = regex_replace_with_format(ret, num_re, expand_numbers);
 
   return ret;
 }

storage-manager/src/IOCoordinator.cpp

@@ -25,8 +25,6 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <boost/filesystem.hpp>
-#define BOOST_SPIRIT_THREADSAFE
-#include <boost/property_tree/json_parser.hpp>
 #include <iostream>
 #include "checks.h"
 #include "vlarray.h"
@@ -1266,9 +1264,10 @@ boost::shared_array<uint8_t> IOCoordinator::mergeJournal(const char* object, con
   boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead);
   stringstream ss;
   ss << headertxt.get();
-  boost::property_tree::ptree header;
-  boost::property_tree::json_parser::read_json(ss, header);
-  assert(header.get<int>("version") == 1);
+
+  nlohmann::json header = nlohmann::json::parse(ss);
+
+  assert(header["version"] == 1);
 
   // start processing the entries
   while (1)
@@ -1353,9 +1352,9 @@ int IOCoordinator::mergeJournalInMem(boost::shared_array<uint8_t>& objData, size
   boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead);
   stringstream ss;
   ss << headertxt.get();
-  boost::property_tree::ptree header;
-  boost::property_tree::json_parser::read_json(ss, header);
-  assert(header.get<int>("version") == 1);
+
+  nlohmann::json header = nlohmann::json::parse(ss);
+  assert(header["version"] == 1);
 
   // read the journal file into memory
   size_t journalBytes = ::lseek(journalFD, 0, SEEK_END) - l_bytesRead;
@@ -1433,9 +1432,9 @@ int IOCoordinator::mergeJournalInMem_bigJ(boost::shared_array<uint8_t>& objData,
   boost::shared_array<char> headertxt = seekToEndOfHeader1(journalFD, &l_bytesRead);
   stringstream ss;
   ss << headertxt.get();
-  boost::property_tree::ptree header;
-  boost::property_tree::json_parser::read_json(ss, header);
-  assert(header.get<int>("version") == 1);
+
+  nlohmann::json header = nlohmann::json::parse(ss);
+  assert(header["version"] == 1);
 
   // start processing the entries
   while (1)

storage-manager/src/MetadataFile.cpp

@@ -20,20 +20,17 @@
  */
 #include "MetadataFile.h"
 #include <boost/filesystem.hpp>
-#define BOOST_SPIRIT_THREADSAFE
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/json_parser.hpp>
-#include <boost/foreach.hpp>
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/uuid_io.hpp>
 #include <boost/uuid/random_generator.hpp>
+#include <cstdio>
 #include <unistd.h>
+#include <fstream>
 
 #define max(x, y) (x > y ? x : y)
 #define min(x, y) (x < y ? x : y)
 
 using namespace std;
-namespace bpt = boost::property_tree;
 namespace bf = boost::filesystem;
 
 namespace
@@ -120,12 +117,13 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename)
   {
     if (boost::filesystem::exists(mFilename))
     {
-      jsontree.reset(new bpt::ptree());
-      boost::property_tree::read_json(mFilename.string(), *jsontree);
+      std::ifstream i(mFilename.string());
+      jsontree.reset(new nlohmann::json);
+      i >> *jsontree;
       jsonCache.put(mFilename, jsontree);
       s.unlock();
       mVersion = 1;
-      mRevision = jsontree->get<int>("revision");
+      mRevision = (*jsontree)["revision"];
     }
     else
     {
@@ -140,7 +138,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename)
   {
     s.unlock();
     mVersion = 1;
-    mRevision = jsontree->get<int>("revision");
+    mRevision = (*jsontree)["revision"];;
   }
   ++metadataFilesAccessed;
 }
@@ -162,12 +160,13 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t,
     if (boost::filesystem::exists(mFilename))
     {
       _exists = true;
-      jsontree.reset(new bpt::ptree());
-      boost::property_tree::read_json(mFilename.string(), *jsontree);
+      jsontree.reset(new nlohmann::json);
+      std::ifstream i(mFilename.string());
+      i >> *jsontree;
       jsonCache.put(mFilename, jsontree);
       s.unlock();
       mVersion = 1;
-      mRevision = jsontree->get<int>("revision");
+      mRevision = (*jsontree)["revision"];
     }
     else
     {
@@ -182,7 +181,7 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t,
     s.unlock();
     _exists = true;
     mVersion = 1;
-    mRevision = jsontree->get<int>("revision");
+    mRevision = (*jsontree)["revision"];
   }
   ++metadataFilesAccessed;
 }
@@ -193,11 +192,10 @@ MetadataFile::~MetadataFile()
 
 void MetadataFile::makeEmptyJsonTree()
 {
-  jsontree.reset(new bpt::ptree());
-  boost::property_tree::ptree objs;
-  jsontree->put("version", mVersion);
-  jsontree->put("revision", mRevision);
-  jsontree->add_child("objects", objs);
+  jsontree.reset(new nlohmann::json);
+  (*jsontree)["version"] = mVersion;
+  (*jsontree)["revision"] = mRevision;
+  (*jsontree)["objects"] = nlohmann::json::array();
 }
 
 void MetadataFile::printKPIs()
@@ -219,11 +217,11 @@ size_t MetadataFile::getLength() const
 {
   size_t totalSize = 0;
 
-  auto& objects = jsontree->get_child("objects");
+  auto &objects = (*jsontree)["objects"];
   if (!objects.empty())
   {
-    auto& lastObject = objects.back().second;
-    totalSize = lastObject.get<off_t>("offset") + lastObject.get<size_t>("length");
+    auto& lastObject = objects.back();
+    totalSize = lastObject["offset"].get<off_t>() + lastObject["length"].get<size_t>();
   }
   return totalSize;
 }
@@ -243,10 +241,9 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length) c
      rather than write a new alg.
   */
   set<metadataObject> mObjects;
-  BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
+  for(const auto &v : (*jsontree)["objects"])
   {
-    mObjects.insert(metadataObject(v.second.get<uint64_t>("offset"), v.second.get<uint64_t>("length"),
-                                   v.second.get<string>("key")));
+    mObjects.insert(metadataObject(v["offset"], v["length"], v["key"]));
   }
 
   if (mObjects.size() == 0)
@@ -288,20 +285,20 @@ metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& fi
   //
 
   metadataObject addObject;
-  auto& objects = jsontree->get_child("objects");
+  auto& objects = (*jsontree)["objects"];
   if (!objects.empty())
   {
-    auto& lastObject = objects.back().second;
-    addObject.offset = lastObject.get<off_t>("offset") + mpConfig->mObjectSize;
+    auto& lastObject = objects.back();
+    addObject.offset = lastObject["offset"].get<off_t>() + mpConfig->mObjectSize;
   }
 
   addObject.length = length;
   addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length);
-  boost::property_tree::ptree object;
-  object.put("offset", addObject.offset);
-  object.put("length", addObject.length);
-  object.put("key", addObject.key);
-  objects.push_back(make_pair("", object));
+  nlohmann::json object = nlohmann::json::object();
+  object["offset"] = addObject.offset;
+  object["length"] = addObject.length;
+  object["key"] = addObject.key;
+  objects.push_back(object);
 
   return addObject;
 }
@@ -312,7 +309,8 @@ int MetadataFile::writeMetadata()
   if (!boost::filesystem::exists(mFilename.parent_path()))
     boost::filesystem::create_directories(mFilename.parent_path());
 
-  write_json(mFilename.string(), *jsontree);
+  std::ofstream o(mFilename.c_str());
+  o << *jsontree;
   _exists = true;
 
   boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
@@ -324,13 +322,13 @@ int MetadataFile::writeMetadata()
 bool MetadataFile::getEntry(off_t offset, metadataObject* out) const
 {
   metadataObject addObject;
-  BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
+  for(auto &v: (*jsontree)["objects"])
   {
-    if (v.second.get<off_t>("offset") == offset)
+    if (v["offset"].get<off_t>() == offset)
     {
       out->offset = offset;
-      out->length = v.second.get<size_t>("length");
-      out->key = v.second.get<string>("key");
+      out->length = v["length"].get<size_t>();
+      out->key = v["key"];
       return true;
     }
   }
@@ -339,10 +337,10 @@ bool MetadataFile::getEntry(off_t offset, metadataObject* out) const
 
 void MetadataFile::removeEntry(off_t offset)
 {
-  bpt::ptree& objects = jsontree->get_child("objects");
-  for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it)
+  auto& objects = (*jsontree)["objects"];
+  for (auto it = objects.begin(); it != objects.end(); ++it)
   {
-    if (it->second.get<off_t>("offset") == offset)
+    if ((*it)["offset"].get<off_t>() == offset)
     {
       objects.erase(it);
       break;
@@ -352,7 +350,7 @@ void MetadataFile::removeEntry(off_t offset)
 
 void MetadataFile::removeAllEntries()
 {
-  jsontree->get_child("objects").clear();
+  (*jsontree)["objects"] = nlohmann::json::array();
 }
 
 void MetadataFile::deletedMeta(const bf::path& p)
@@ -456,21 +454,21 @@ void MetadataFile::setLengthInKey(string& key, size_t newLength)
 
 void MetadataFile::printObjects() const
 {
-  BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
+  for (auto& v : (*jsontree)["objects"])
   {
-    printf("Name: %s Length: %zu Offset: %lld\n", v.second.get<string>("key").c_str(),
-           v.second.get<size_t>("length"), (long long)v.second.get<off_t>("offset"));
+    printf("Name: %s Length: %zu Offset: %lld\n", v["key"].get<std::string>().c_str(),
+           v["length"].get<size_t>(), (long long)v["offset"].get<off_t>());
   }
 }
 
 void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLength)
 {
-  for (auto& v : jsontree->get_child("objects"))
+  for (auto& v : (*jsontree)["objects"])
   {
-    if (v.second.get<off_t>("offset") == offset)
+    if (v["offset"].get<off_t>() == offset)
     {
-      v.second.put("key", newName);
-      v.second.put("length", newLength);
+      v["key"] = newName;
+      v["length"] = newLength;
       return;
     }
   }
@@ -482,11 +480,11 @@ void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLe
 
 void MetadataFile::updateEntryLength(off_t offset, size_t newLength)
 {
-  for (auto& v : jsontree->get_child("objects"))
+  for (auto& v : (*jsontree)["objects"])
   {
-    if (v.second.get<off_t>("offset") == offset)
+    if (v["offset"].get<off_t>() == offset)
     {
-      v.second.put("length", newLength);
+      v["length"] = newLength;
       return;
     }
   }
@@ -498,11 +496,12 @@ void MetadataFile::updateEntryLength(off_t offset, size_t newLength)
 
 off_t MetadataFile::getMetadataNewObjectOffset()
 {
-  auto& objects = jsontree->get_child("objects");
+  auto& objects = (*jsontree)["objects"];
   if (objects.empty())
     return 0;
-  auto& lastObject = jsontree->get_child("objects").back().second;
-  return lastObject.get<off_t>("offset") + lastObject.get<size_t>("length");
+
+  auto& lastObject = objects.back();
+  return lastObject["offset"].get<off_t>() + lastObject["length"].get<size_t>();
 }
 
 metadataObject::metadataObject() : offset(0), length(0)

storage-manager/src/MetadataFile.h

@@ -28,6 +28,8 @@
 #include <unordered_map>
 #include <boost/filesystem/path.hpp>
 
+#include <utils/json/json.hpp>
+
 namespace storagemanager
 {
 struct metadataObject
@@ -110,7 +112,7 @@ class MetadataFile
 
   static void printKPIs();
 
-  typedef boost::shared_ptr<boost::property_tree::ptree> Jsontree_t;
+  typedef boost::shared_ptr<nlohmann::json> Jsontree_t;
 
  private:
   MetadataConfig* mpConfig;

storage-manager/src/Replicator.cpp

@@ -27,12 +27,12 @@
 #include <errno.h>
 #include <sys/sendfile.h>
 #include <boost/filesystem.hpp>
-#define BOOST_SPIRIT_THREADSAFE
-#include <boost/property_tree/json_parser.hpp>
 #include <boost/shared_array.hpp>
 #include <boost/format.hpp>
 #include <iostream>
 
+#include <utils/json/json.hpp>
+
 using namespace std;
 
 namespace
@@ -279,12 +279,14 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u
     stringstream ss;
     ss << headertxt.get();
     headerRollback = headertxt.get();
-    boost::property_tree::ptree header;
+    nlohmann::json header;
+
     try
     {
-      boost::property_tree::json_parser::read_json(ss, header);
+      header = nlohmann::json::parse(ss);
     }
-    catch (const boost::property_tree::json_parser::json_parser_error& e)
+
+    catch (const nlohmann::json::exception& e)
     {
       mpLogger->log(LOG_CRIT, "%s", e.what());
       errno = EIO;
@@ -296,8 +298,8 @@ int Replicator::addJournalEntry(const boost::filesystem::path& filename, const u
       errno = EIO;
       return -1;
     }
-    assert(header.get<int>("version") == 1);
-    uint64_t currentMaxOffset = header.get<uint64_t>("max_offset");
+    assert(header["version"] == 1);
+    uint64_t currentMaxOffset = header["max_offset"];
     if (thisEntryMaxOffset > currentMaxOffset)
     {
       bHeaderChanged = true;

storage-manager/src/S3Storage.cpp

@@ -26,9 +26,9 @@
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/uuid_io.hpp>
 #include <boost/uuid/random_generator.hpp>
-#define BOOST_SPIRIT_THREADSAFE
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/json_parser.hpp>
+
+#include "utils/json/json.hpp"
+
 #include "Utilities.h"
 
 using namespace std;
@@ -258,12 +258,12 @@ bool S3Storage::getCredentialsFromMetadataEC2()
     logger->log(LOG_ERR, "CURL fail %u", curl_res);
     return false;
   }
-  stringstream credentials(readBuffer);
-  boost::property_tree::ptree pt;
-  boost::property_tree::read_json(credentials, pt);
-  key = pt.get<string>("AccessKeyId");
-  secret = pt.get<string>("SecretAccessKey");
-  token = pt.get<string>("Token");
+
+  nlohmann::json pt = nlohmann::json::parse(readBuffer);
+  key = pt["AccessKeyId"];
+  secret = pt["SecretAccessKey"];
+  token = pt["Token"];
+
   // logger->log(LOG_INFO, "S3Storage: key = %s secret = %s token =
   // %s",key.c_str(),secret.c_str(),token.c_str());
 
@@ -626,7 +626,7 @@ int S3Storage::copyObject(const string& _sourceKey, const string& _destKey)
 
 #if 0
     // no s3-s3 copy yet.  get & put for now.
-    
+
     int err;
     boost::shared_array<uint8_t> data;
     size_t len;