From 8d68ff3847d11faeecf846a407dd3e15d995d3d3 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Wed, 24 Jul 2019 10:06:47 -0500 Subject: [PATCH] Checkpoint of Path changes MCOL-3365 --- src/IOCoordinator.cpp | 65 +++++++++++++++++++++---------------------- src/IOCoordinator.h | 2 +- src/MetadataFile.cpp | 13 +++++---- src/MetadataFile.h | 12 ++++---- src/Synchronizer.cpp | 4 +-- 5 files changed, 47 insertions(+), 49 deletions(-) diff --git a/src/IOCoordinator.cpp b/src/IOCoordinator.cpp index 738c32e4d..006f84832 100644 --- a/src/IOCoordinator.cpp +++ b/src/IOCoordinator.cpp @@ -158,12 +158,12 @@ ssize_t IOCoordinator::read(const char *_filename, uint8_t *data, off_t offset, release read lock put together the response in data */ - bf::path p = ownership.get(_filename); - const bf::path firstDir = *(p.begin()); - const char *filename = p.string().c_str(); + bf::path filename = ownership.get(_filename); + const bf::path firstDir = *(filename.begin()); + //const char *filename = p.string().c_str(); - ScopedReadLock fileLock(this, filename); - MetadataFile meta(filename, MetadataFile::no_create_t()); + ScopedReadLock fileLock(this, filename.string()); + MetadataFile meta(filename, MetadataFile::no_create_t(),true); if (!meta.exists()) { @@ -283,18 +283,17 @@ out: ssize_t IOCoordinator::write(const char *_filename, const uint8_t *data, off_t offset, size_t length) { - bf::path p = ownership.get(_filename); - const bf::path firstDir = *(p.begin()); - const char *filename = p.string().c_str(); + bf::path filename = ownership.get(_filename); + const bf::path firstDir = *(filename.begin()); bytesWritten += length; - ScopedWriteLock lock(this, filename); + ScopedWriteLock lock(this, filename.string()); int ret = _write(filename, data, offset, length, firstDir); lock.unlock(); cache->doneWriting(firstDir); return ret; } -ssize_t IOCoordinator::_write(const char *filename, const uint8_t *data, off_t offset, size_t length, +ssize_t IOCoordinator::_write(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length, const bf::path &firstDir) { int err = 0; @@ -306,7 +305,7 @@ ssize_t IOCoordinator::_write(const char *filename, const uint8_t *data, off_t o vector newObjectKeys; Synchronizer *synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency... - MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t()); + MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t(),true); if (!metadata.exists()) { errno = ENOENT; @@ -421,9 +420,9 @@ ssize_t IOCoordinator::_write(const char *filename, const uint8_t *data, off_t o ssize_t IOCoordinator::append(const char *_filename, const uint8_t *data, size_t length) { - bf::path p = ownership.get(_filename); - const bf::path firstDir = *(p.begin()); - const char *filename = p.string().c_str(); + bf::path filename = ownership.get(_filename); + const bf::path firstDir = *(filename.begin()); + //const char *filename = p.string().c_str(); int err; ssize_t count = 0; @@ -433,9 +432,9 @@ ssize_t IOCoordinator::append(const char *_filename, const uint8_t *data, size_t vector newObjectKeys; Synchronizer *synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency... - ScopedWriteLock lock(this, filename); + ScopedWriteLock lock(this, filename.string()); - MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t()); + MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t(),true); if (!metadata.exists()) { errno = EBADF; @@ -530,23 +529,23 @@ out: // TODO: might need to support more open flags, ex: O_EXCL int IOCoordinator::open(const char *_filename, int openmode, struct stat *out) { - bf::path p = ownership.get(_filename); - const char *filename = p.string().c_str(); + bf::path filename = ownership.get(_filename); + //const char *filename = p.string().c_str(); boost::scoped_ptr s; if (openmode & O_CREAT || openmode | O_TRUNC) - s.reset(new ScopedWriteLock(this, filename)); + s.reset(new ScopedWriteLock(this, filename.string())); else - s.reset(new ScopedReadLock(this, filename)); + s.reset(new ScopedReadLock(this, filename.string())); - MetadataFile meta(filename, MetadataFile::no_create_t()); + MetadataFile meta(filename, MetadataFile::no_create_t(),true); if ((openmode & O_CREAT) && !meta.exists()) { ++filesCreated; replicator->updateMetadata(filename, meta); // this will end up creating filename } if ((openmode & O_TRUNC) && meta.exists()) - _truncate(p, 0, s.get()); + _truncate(filename, 0, s.get()); ++filesOpened; return meta.stat(out); @@ -582,14 +581,13 @@ int IOCoordinator::listDirectory(const char *dirname, vector *listing) int IOCoordinator::stat(const char *_path, struct stat *out) { - bf::path p = ownership.get(_path); - const char *path = p.string().c_str(); + bf::path filename = ownership.get(_path); - if (bf::is_directory(metaPath/p)) - return ::stat((metaPath/p).string().c_str(), out); + if (bf::is_directory(metaPath/filename)) + return ::stat((metaPath/filename).string().c_str(), out); - ScopedReadLock s(this, path); - MetadataFile meta(path, MetadataFile::no_create_t()); + ScopedReadLock s(this, filename.string()); + MetadataFile meta(filename, MetadataFile::no_create_t(),true); return meta.stat(out); } @@ -617,12 +615,11 @@ int IOCoordinator::_truncate(const bf::path &bfpath, size_t newSize, ScopedFileL tell synchronizer they were deleted */ const bf::path firstDir = *(bfpath.begin()); - const char *path = bfpath.string().c_str(); Synchronizer *synchronizer = Synchronizer::get(); // needs to init sync here to break circular dependency... int err; - MetadataFile meta(path, MetadataFile::no_create_t()); + MetadataFile meta(bfpath, MetadataFile::no_create_t(),true); if (!meta.exists()) { errno = ENOENT; @@ -637,7 +634,7 @@ int IOCoordinator::_truncate(const bf::path &bfpath, size_t newSize, ScopedFileL if (filesize < newSize) { uint8_t zero = 0; - err = _write(path, &zero, newSize - 1, 1, firstDir); + err = _write(bfpath, &zero, newSize - 1, 1, firstDir); lock->unlock(); cache->doneWriting(firstDir); if (err < 0) @@ -658,7 +655,7 @@ int IOCoordinator::_truncate(const bf::path &bfpath, size_t newSize, ScopedFileL for (uint i = 1; i < objects.size(); i++) meta.removeEntry(objects[i].offset); - err = replicator->updateMetadata(path, meta); + err = replicator->updateMetadata(filename.string().c_str(), meta); if (err) return err; //lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call @@ -703,7 +700,7 @@ void IOCoordinator::deleteMetaFile(const bf::path &file) ScopedWriteLock lock(this, pita); //cout << "file is " << file.string() << " locked on " << pita << endl; - MetadataFile meta(file); + MetadataFile meta(file,MetadataFile::no_create_t(),false); replicator->remove(file); //lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call @@ -858,7 +855,7 @@ int IOCoordinator::copyFile(const char *_filename1, const char *_filename2) ScopedReadLock lock(this, filename1); ScopedWriteLock lock2(this, filename2); MetadataFile meta1(metaFile1); - MetadataFile meta2(metaFile2.string().c_str(), MetadataFile::no_create_t()); + MetadataFile meta2(metaFile2, MetadataFile::no_create_t(),false); vector objects = meta1.metadataRead(0, meta1.getLength()); bytesCopied += meta1.getLength(); diff --git a/src/IOCoordinator.h b/src/IOCoordinator.h index 88adde532..6b6c8192f 100644 --- a/src/IOCoordinator.h +++ b/src/IOCoordinator.h @@ -92,7 +92,7 @@ class IOCoordinator : public boost::noncopyable void deleteMetaFile(const boost::filesystem::path &file); int _truncate(const boost::filesystem::path &path, size_t newsize, ScopedFileLock *lock); - ssize_t _write(const char *filename, const uint8_t *data, off_t offset, size_t length, + ssize_t _write(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length, const boost::filesystem::path &firstDir); int loadObjectAndJournal(const char *objFilename, const char *journalFilename, diff --git a/src/MetadataFile.cpp b/src/MetadataFile.cpp index 8adb3e66a..12f166e87 100644 --- a/src/MetadataFile.cpp +++ b/src/MetadataFile.cpp @@ -90,7 +90,7 @@ MetadataFile::MetadataFile() } -MetadataFile::MetadataFile(const char* filename) +MetadataFile::MetadataFile(const boost::filesystem::path &filename) { mpConfig = MetadataConfig::get(); mpLogger = SMLogging::get(); @@ -124,12 +124,13 @@ MetadataFile::MetadataFile(const char* filename) ++metadataFilesAccessed; } -MetadataFile::MetadataFile(const char* filename, no_create_t) +MetadataFile::MetadataFile(const boost::filesystem::path &filename, no_create_t,bool appendExt) { mpConfig = MetadataConfig::get(); mpLogger = SMLogging::get(); - mFilename = mpConfig->msMetadataPath + "/" + string(filename) + ".meta"; + if(appendExt) + mFilename = mpConfig->msMetadataPath / string(filename) + ".meta"; if (boost::filesystem::exists(mFilename)) { _exists = true; @@ -262,7 +263,7 @@ vector MetadataFile::metadataRead(off_t offset, size_t length) c return ret; } -metadataObject MetadataFile::addMetadataObject(const char *filename, size_t length) +metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path &filename, size_t length) { // this needs to handle if data write is beyond the end of the last object // but not at start of new object @@ -286,11 +287,11 @@ metadataObject MetadataFile::addMetadataObject(const char *filename, size_t leng } -int MetadataFile::writeMetadata(const char *filename) +int MetadataFile::writeMetadata(const boost::filesystem::path &filename) { int error=0; - string metadataFilename = mpConfig->msMetadataPath + "/" + string(filename) + ".meta"; + string metadataFilename = mpConfig->msMetadataPath / filename + ".meta"; boost::filesystem::path pMetadataFilename = metadataFilename; boost::property_tree::ptree jsontree; boost::property_tree::ptree objs; diff --git a/src/MetadataFile.h b/src/MetadataFile.h index fdcbe3323..2ca349b49 100644 --- a/src/MetadataFile.h +++ b/src/MetadataFile.h @@ -32,13 +32,13 @@ class MetadataFile public: struct no_create_t {}; MetadataFile(); - MetadataFile(const char* filename); - MetadataFile(const char* filename, no_create_t); // this one won't create it if it doesn't exist + MetadataFile(const boost::filesystem::path &filename); + MetadataFile(const boost::filesystem::path &path, no_create_t,bool appendExt); // this one won't create it if it doesn't exist // this ctor is 'special'. It will take an absolute path, and it will assume it points to a metafile // meaning, that it doesn't need the metadata prefix prepended, or the .meta extension appended. // aside from that, it will behave like the no_create ctor variant above - MetadataFile(const boost::filesystem::path &path); + //MetadataFile(const boost::filesystem::path &path); ~MetadataFile(); bool exists() const; @@ -48,12 +48,12 @@ class MetadataFile // returns the objects needed to update std::vector metadataRead(off_t offset, size_t length) const; // updates the metadatafile with new object - int writeMetadata(const char *filename); + int writeMetadata(const boost::filesystem::path &filename); // updates the name and length fields of an entry, given the offset void updateEntry(off_t offset, const std::string &newName, size_t newLength); void updateEntryLength(off_t offset, size_t newLength); - metadataObject addMetadataObject(const char *filename, size_t length); + metadataObject addMetadataObject(const boost::filesystem::path &filename, size_t length); bool getEntry(off_t offset, const metadataObject **out) const; void removeEntry(off_t offset); void removeAllEntries(); @@ -92,7 +92,7 @@ class MetadataFile SMLogging *mpLogger; int mVersion; int mRevision; - std::string mFilename; + boost::filesystem::path mFilename; std::set mObjects; bool _exists; }; diff --git a/src/Synchronizer.cpp b/src/Synchronizer.cpp index b8db133c2..1662f9d09 100644 --- a/src/Synchronizer.cpp +++ b/src/Synchronizer.cpp @@ -423,7 +423,7 @@ void Synchronizer::synchronize(const string &sourceFile, list::iterator char buf[80]; bool exists = false; int err; - MetadataFile md(sourceFile.c_str(), MetadataFile::no_create_t()); + MetadataFile md(sourceFile.c_str(), MetadataFile::no_create_t(),true); if (!md.exists()) { @@ -481,7 +481,7 @@ void Synchronizer::synchronizeWithJournal(const string &sourceFile, list bf::path prefix = key.substr(0, pos); string cloudKey = key.substr(pos + 1); - MetadataFile md(sourceFile.c_str(), MetadataFile::no_create_t()); + MetadataFile md(sourceFile.c_str(), MetadataFile::no_create_t(),true); if (!md.exists()) {