diff --git a/storage-manager/src/IOCoordinator.cpp b/storage-manager/src/IOCoordinator.cpp index 347899db4..5a6adaf9a 100644 --- a/storage-manager/src/IOCoordinator.cpp +++ b/storage-manager/src/IOCoordinator.cpp @@ -1021,17 +1021,6 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) int err; char errbuf[80]; - if (!bf::exists(metaFile1)) - { - errno = ENOENT; - return -1; - } - if (bf::exists(metaFile2)) - { - deleteMetaFile(metaFile2); - ++filesDeleted; - } - // since we don't implement mkdir(), assume the caller did that and // create any necessary parent dirs for filename2 try @@ -1046,12 +1035,18 @@ int IOCoordinator::copyFile(const char* _filename1, const char* _filename2) return -1; } - vector > newJournalEntries; + vector> newJournalEntries; ScopedReadLock lock(this, filename1); ScopedWriteLock lock2(this, filename2); MetadataFile meta1(metaFile1, MetadataFile::no_create_t(), false); MetadataFile meta2(metaFile2, MetadataFile::no_create_t(), false); vector objects = meta1.metadataRead(0, meta1.getLength()); + + if (!meta1.exists()) + { + errno = ENOENT; + return -1; + } bytesCopied += meta1.getLength(); if (meta2.exists()) @@ -1197,9 +1192,8 @@ int IOCoordinator::mergeJournal(int objFD, int journalFD, uint8_t* buf, off_t of throw runtime_error("IOCoordinator::mergeJournal(int, int, etc) is not implemented yet."); } -std::shared_ptr IOCoordinator::mergeJournal(const char* object, const char* journal, - off_t offset, size_t len, - size_t* _bytesReadOut) const +std::shared_ptr IOCoordinator::mergeJournal(const char* object, const char* journal, off_t offset, + size_t len, size_t* _bytesReadOut) const { int objFD, journalFD; std::shared_ptr ret; @@ -1335,8 +1329,8 @@ out: // MergeJournalInMem is a specialized version of mergeJournal(). This is currently only used by Synchronizer // and mergeJournal(), and only for merging the whole object with the whole journal. -int IOCoordinator::mergeJournalInMem(std::shared_ptr& objData, size_t len, - const char* journalPath, size_t* _bytesReadOut) const +int IOCoordinator::mergeJournalInMem(std::shared_ptr& objData, size_t len, const char* journalPath, + size_t* _bytesReadOut) const { // if the journal is over some size threshold (100MB for now why not), // use the original low-mem-usage version diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index b4fb327f8..8cf964c69 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -19,18 +19,24 @@ * MetadataFile.cpp */ #include "MetadataFile.h" +#include "KVStorageInitializer.h" #include #include +#include +#include +#include +#include +#include #define BOOST_SPIRIT_THREADSAFE #ifndef __clang__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif #include #ifndef __clang__ - #pragma GCC diagnostic pop +#pragma GCC diagnostic pop #endif #include #include @@ -38,6 +44,8 @@ #include #include #include +#include "fdbcs.hpp" +#include "KVPrefixes.hpp" #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) @@ -55,6 +63,12 @@ uint64_t metadataFilesAccessed = 0; namespace storagemanager { + +inline std::string getKeyName(const std::string& fileName) +{ + return KVPrefixes[static_cast(KVPrefixId::SM_META)] + fileName; +} + MetadataFile::MetadataConfig* MetadataFile::MetadataConfig::get() { if (inst) @@ -123,15 +137,21 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename) _exists = true; mFilename = mpConfig->msMetadataPath / (filename.string() + ".meta"); + metaKVName_ = getKeyName(mFilename.string()); boost::unique_lock s(jsonCache.getMutex()); jsontree = jsonCache.get(mFilename); if (!jsontree) { - if (boost::filesystem::exists(mFilename)) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler blobReader(keyGen); + auto rPair = blobReader.readBlob(kvStorage, metaKVName_); + if (rPair.first) { jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); + stringstream stream(rPair.second); + boost::property_tree::read_json(stream, *jsontree); jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; @@ -161,19 +181,24 @@ MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, mpLogger = SMLogging::get(); mFilename = filename; - if (appendExt) mFilename = mpConfig->msMetadataPath / (mFilename.string() + ".meta"); + metaKVName_ = getKeyName(mFilename.string()); boost::unique_lock s(jsonCache.getMutex()); jsontree = jsonCache.get(mFilename); if (!jsontree) { - if (boost::filesystem::exists(mFilename)) + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler blobReader(keyGen); + auto rPair = blobReader.readBlob(kvStorage, metaKVName_); + if (rPair.first) { _exists = true; jsontree.reset(new bpt::ptree()); - boost::property_tree::read_json(mFilename.string(), *jsontree); + stringstream stream(rPair.second); + boost::property_tree::read_json(stream, *jsontree); jsonCache.put(mFilename, jsontree); s.unlock(); mVersion = 1; @@ -215,16 +240,55 @@ void MetadataFile::printKPIs() cout << "Metadata files accessed = " << metadataFilesAccessed << endl; } -int MetadataFile::stat(struct stat* out) const +// FIXME: This one increases IO load, we create a new file if the `stat` is not cached, +// but I'm not currently sure how to generate a `stat` info without a file. +// Could we make it smarter? Need some research on this area. +int MetadataFile::generateStatStructInfo(struct stat* out) { - int err = ::stat(mFilename.c_str(), out); - if (err) - return err; + try + { + const std::string statFileName = + mpConfig->msMetadataPath.string() + "/" + boost::to_string(boost::uuids::random_generator()()); + std::ofstream statStream(statFileName); + statStream.close(); + + int err = ::stat(statFileName.c_str(), out); + if (err) + return -1; + + statCache.resize(sizeof(struct stat)); + std::memcpy(&statCache[0], out, sizeof(struct stat)); + statCached = true; + std::filesystem::remove(statFileName); + out->st_size = getLength(); + } + catch (const std::exception& ex) + { + SMLogging::get()->log(LOG_CRIT, "Metadatafile::stat() failed with error: %s", ex.what()); + return -1; + } - out->st_size = getLength(); return 0; } +int MetadataFile::stat(struct stat* out) +{ + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + auto rPair = tnx->get(metaKVName_); + if (rPair.first) + { + if (statCached) + { + std::memcpy(out, (uint8_t*)&statCache[0], sizeof(struct stat)); + out->st_size = getLength(); + return 0; + } + return generateStatStructInfo(out); + } + return -1; +} + size_t MetadataFile::getLength() const { size_t totalSize = 0; @@ -319,10 +383,20 @@ metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& fi // TODO: Error handling...s int MetadataFile::writeMetadata() { - if (!boost::filesystem::exists(mFilename.parent_path())) - boost::filesystem::create_directories(mFilename.parent_path()); + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto keyGen = std::make_shared(); + FDBCS::BlobHandler blobWriter(keyGen); + stringstream stream; + write_json(stream, *jsontree); + + if (!blobWriter.writeBlob(kvStorage, metaKVName_, stream.str())) + { + SMLogging::get()->log(LOG_CRIT, "Metadatafile: cannot commit tnx set()."); + throw runtime_error("Metadatafile: cannot commit tnx set()."); + } + } - write_json(mFilename.string(), *jsontree); _exists = true; boost::unique_lock s(jsonCache.getMutex()); diff --git a/storage-manager/src/MetadataFile.h b/storage-manager/src/MetadataFile.h index 5c302c618..f8115ae09 100644 --- a/storage-manager/src/MetadataFile.h +++ b/storage-manager/src/MetadataFile.h @@ -63,7 +63,7 @@ class MetadataFile bool exists() const; void printObjects() const; - int stat(struct stat*) const; + int stat(struct stat*); size_t getLength() const; // returns the objects needed to update std::vector metadataRead(off_t offset, size_t length) const; @@ -118,10 +118,14 @@ class MetadataFile int mVersion; int mRevision; boost::filesystem::path mFilename; + std::string metaKVName_; Jsontree_t jsontree; // std::set mObjects; bool _exists; void makeEmptyJsonTree(); + int generateStatStructInfo(struct stat *); + std::vector statCache; + bool statCached{false}; class MetadataCache { diff --git a/storage-manager/src/Ownership.cpp b/storage-manager/src/Ownership.cpp index 0fe670b76..35c8a19b4 100644 --- a/storage-manager/src/Ownership.cpp +++ b/storage-manager/src/Ownership.cpp @@ -242,10 +242,6 @@ void Ownership::_takeOwnership(const bf::path& p) void Ownership::takeOwnership(const bf::path& p) { - // If the prefix doesn't exist, ownership isn't possible yet. - if (!bf::is_directory(metadataPrefix / p)) - return; - boost::unique_lock s(mutex); auto it = ownedPrefixes.find(p);