feat(SM,FDB): MCOL-5720 Move meta to FDB. (#3369)

storage-manager/src/IOCoordinator.cpp

@@ -1021,17 +1021,6 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2)
   int err;
   char errbuf[80];
 
-  if (!bf::exists(metaFile1))
-  {
-    errno = ENOENT;
-    return -1;
-  }
-  if (bf::exists(metaFile2))
-  {
-    deleteMetaFile(metaFile2);
-    ++filesDeleted;
-  }
-
   // since we don't implement mkdir(), assume the caller did that and
   // create any necessary parent dirs for filename2
   try
@@ -1046,12 +1035,18 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2)
     return -1;
   }
 
-  vector<pair<string, size_t> > newJournalEntries;
+  vector<pair<string, size_t>> newJournalEntries;
   ScopedReadLock lock(this, filename1);
   ScopedWriteLock lock2(this, filename2);
   MetadataFile meta1(metaFile1, MetadataFile::no_create_t(), false);
   MetadataFile meta2(metaFile2, MetadataFile::no_create_t(), false);
   vector<metadataObject> objects = meta1.metadataRead(0, meta1.getLength());
+
+  if (!meta1.exists())
+  {
+    errno = ENOENT;
+    return -1;
+  }
   bytesCopied += meta1.getLength();
 
   if (meta2.exists())
@@ -1197,9 +1192,8 @@ int IOCoordinator::mergeJournal(int objFD, int journalFD, uint8_t* buf, off_t of
   throw runtime_error("IOCoordinator::mergeJournal(int, int, etc) is not implemented yet.");
 }
 
-std::shared_ptr<uint8_t[]> IOCoordinator::mergeJournal(const char* object, const char* journal,
+std::shared_ptr<uint8_t[]> IOCoordinator::mergeJournal(const char* object, const char* journal, off_t offset,
-                                                         off_t offset, size_t len,
+                                                       size_t len, size_t* _bytesReadOut) const
-                                                         size_t* _bytesReadOut) const
 {
   int objFD, journalFD;
   std::shared_ptr<uint8_t[]> ret;
@@ -1335,8 +1329,8 @@ out:
 
 // MergeJournalInMem is a specialized version of mergeJournal().  This is currently only used by Synchronizer
 // and mergeJournal(), and only for merging the whole object with the whole journal.
-int IOCoordinator::mergeJournalInMem(std::shared_ptr<uint8_t[]>& objData, size_t len,
+int IOCoordinator::mergeJournalInMem(std::shared_ptr<uint8_t[]>& objData, size_t len, const char* journalPath,
-                                     const char* journalPath, size_t* _bytesReadOut) const
+                                     size_t* _bytesReadOut) const
 {
   // if the journal is over some size threshold (100MB for now why not),
   // use the original low-mem-usage version

storage-manager/src/MetadataFile.cpp

@@ -19,18 +19,24 @@
  * MetadataFile.cpp
  */
 #include "MetadataFile.h"
+#include "KVStorageInitializer.h"
 #include <set>
 #include <boost/filesystem.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <boost/uuid/uuid_io.hpp>
+#include <boost/uuid/random_generator.hpp>
+#include <boost/lexical_cast.hpp>
+#include <filesystem>
 #define BOOST_SPIRIT_THREADSAFE
 #ifndef __clang__
-  #pragma GCC diagnostic push
+#pragma GCC diagnostic push
-  #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
 #endif
 
 #include <boost/property_tree/ptree.hpp>
 
 #ifndef __clang__
-  #pragma GCC diagnostic pop
+#pragma GCC diagnostic pop
 #endif
 #include <boost/property_tree/json_parser.hpp>
 #include <boost/foreach.hpp>
@@ -38,6 +44,8 @@
 #include <boost/uuid/uuid_io.hpp>
 #include <boost/uuid/random_generator.hpp>
 #include <unistd.h>
+#include "fdbcs.hpp"
+#include "KVPrefixes.hpp"
 
 #define max(x, y) (x > y ? x : y)
 #define min(x, y) (x < y ? x : y)
@@ -55,6 +63,12 @@ uint64_t metadataFilesAccessed = 0;
 
 namespace storagemanager
 {
+
+inline std::string getKeyName(const std::string& fileName)
+{
+  return KVPrefixes[static_cast<uint32_t>(KVPrefixId::SM_META)] + fileName;
+}
+
 MetadataFile::MetadataConfig* MetadataFile::MetadataConfig::get()
 {
   if (inst)
@@ -123,15 +137,21 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename)
   _exists = true;
 
   mFilename = mpConfig->msMetadataPath / (filename.string() + ".meta");
+  metaKVName_ = getKeyName(mFilename.string());
 
   boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
   jsontree = jsonCache.get(mFilename);
   if (!jsontree)
   {
-    if (boost::filesystem::exists(mFilename))
+    auto kvStorage = KVStorageInitializer::getStorageInstance();
+    auto keyGen = std::make_shared<FDBCS::BoostUIDKeyGenerator>();
+    FDBCS::BlobHandler blobReader(keyGen);
+    auto rPair = blobReader.readBlob(kvStorage, metaKVName_);
+    if (rPair.first)
     {
       jsontree.reset(new bpt::ptree());
-      boost::property_tree::read_json(mFilename.string(), *jsontree);
+      stringstream stream(rPair.second);
+      boost::property_tree::read_json(stream, *jsontree);
       jsonCache.put(mFilename, jsontree);
       s.unlock();
       mVersion = 1;
@@ -161,19 +181,24 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t,
   mpLogger = SMLogging::get();
 
   mFilename = filename;
-
   if (appendExt)
     mFilename = mpConfig->msMetadataPath / (mFilename.string() + ".meta");
 
+  metaKVName_ = getKeyName(mFilename.string());
   boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
   jsontree = jsonCache.get(mFilename);
   if (!jsontree)
   {
-    if (boost::filesystem::exists(mFilename))
+    auto kvStorage = KVStorageInitializer::getStorageInstance();
+    auto keyGen = std::make_shared<FDBCS::BoostUIDKeyGenerator>();
+    FDBCS::BlobHandler blobReader(keyGen);
+    auto rPair = blobReader.readBlob(kvStorage, metaKVName_);
+    if (rPair.first)
     {
       _exists = true;
       jsontree.reset(new bpt::ptree());
-      boost::property_tree::read_json(mFilename.string(), *jsontree);
+      stringstream stream(rPair.second);
+      boost::property_tree::read_json(stream, *jsontree);
       jsonCache.put(mFilename, jsontree);
       s.unlock();
       mVersion = 1;
@@ -215,16 +240,55 @@ void MetadataFile::printKPIs()
   cout << "Metadata files accessed = " << metadataFilesAccessed << endl;
 }
 
-int MetadataFile::stat(struct stat* out) const
+// FIXME: This one increases IO load, we create a new file if the `stat` is not cached,
+// but I'm not currently sure how to generate a `stat` info without a file.
+// Could we make it smarter? Need some research on this area.
+int MetadataFile::generateStatStructInfo(struct stat* out)
 {
-  int err = ::stat(mFilename.c_str(), out);
+  try
-  if (err)
+  {
-    return err;
+    const std::string statFileName =
+        mpConfig->msMetadataPath.string() + "/" + boost::to_string(boost::uuids::random_generator()());
+    std::ofstream statStream(statFileName);
+    statStream.close();
+
+    int err = ::stat(statFileName.c_str(), out);
+    if (err)
+      return -1;
+
+    statCache.resize(sizeof(struct stat));
+    std::memcpy(&statCache[0], out, sizeof(struct stat));
+    statCached = true;
+    std::filesystem::remove(statFileName);
+    out->st_size = getLength();
+  }
+  catch (const std::exception& ex)
+  {
+    SMLogging::get()->log(LOG_CRIT, "Metadatafile::stat() failed with error: %s", ex.what());
+    return -1;
+  }
 
-  out->st_size = getLength();
   return 0;
 }
 
+int MetadataFile::stat(struct stat* out)
+{
+  auto kvStorage = KVStorageInitializer::getStorageInstance();
+  auto tnx = kvStorage->createTransaction();
+  auto rPair = tnx->get(metaKVName_);
+  if (rPair.first)
+  {
+    if (statCached)
+    {
+      std::memcpy(out, (uint8_t*)&statCache[0], sizeof(struct stat));
+      out->st_size = getLength();
+      return 0;
+    }
+    return generateStatStructInfo(out);
+  }
+  return -1;
+}
+
 size_t MetadataFile::getLength() const
 {
   size_t totalSize = 0;
@@ -319,10 +383,20 @@ metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& fi
 // TODO: Error handling...s
 int MetadataFile::writeMetadata()
 {
-  if (!boost::filesystem::exists(mFilename.parent_path()))
+  {
-    boost::filesystem::create_directories(mFilename.parent_path());
+    auto kvStorage = KVStorageInitializer::getStorageInstance();
+    auto keyGen = std::make_shared<FDBCS::BoostUIDKeyGenerator>();
+    FDBCS::BlobHandler blobWriter(keyGen);
+    stringstream stream;
+    write_json(stream, *jsontree);
+
+    if (!blobWriter.writeBlob(kvStorage, metaKVName_, stream.str()))
+    {
+      SMLogging::get()->log(LOG_CRIT, "Metadatafile: cannot commit tnx set().");
+      throw runtime_error("Metadatafile: cannot commit tnx set().");
+    }
+  }
 
-  write_json(mFilename.string(), *jsontree);
   _exists = true;
 
   boost::unique_lock<boost::mutex> s(jsonCache.getMutex());

storage-manager/src/MetadataFile.h

@@ -63,7 +63,7 @@ class MetadataFile
 
   bool exists() const;
   void printObjects() const;
-  int stat(struct stat*) const;
+  int stat(struct stat*);
   size_t getLength() const;
   // returns the objects needed to update
   std::vector<metadataObject> metadataRead(off_t offset, size_t length) const;
@@ -118,10 +118,14 @@ class MetadataFile
   int mVersion;
   int mRevision;
   boost::filesystem::path mFilename;
+  std::string metaKVName_;
   Jsontree_t jsontree;
   // std::set<metadataObject> mObjects;
   bool _exists;
   void makeEmptyJsonTree();
+  int generateStatStructInfo(struct stat *);
+  std::vector<uint8_t> statCache;
+  bool statCached{false};
 
   class MetadataCache
   {

storage-manager/src/Ownership.cpp

@@ -242,10 +242,6 @@ void Ownership::_takeOwnership(const bf::path& p)
 
 void Ownership::takeOwnership(const bf::path& p)
 {
-  // If the prefix doesn't exist, ownership isn't possible yet.
-  if (!bf::is_directory(metadataPrefix / p))
-    return;
-
   boost::unique_lock<boost::mutex> s(mutex);
 
   auto it = ownedPrefixes.find(p);