From e164cbc36410c78f1024fbb71ecb0a83034fa84a Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 28 Mar 2019 14:13:21 -0500 Subject: [PATCH] Got a basic unit test to work for IOC::read() --- libmarias3 | 2 +- src/Cache.cpp | 5 ++- src/IOCoordinator.cpp | 21 ++++++++--- src/MetadataFile.cpp | 73 ++++++++++++++++++++++++++++++++++++- src/MetadataFile.h | 4 ++ src/Utilities.cpp | 4 +- src/unit_tests.cpp | 85 ++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 180 insertions(+), 14 deletions(-) diff --git a/libmarias3 b/libmarias3 index aef6eaca1..fa4c00da1 160000 --- a/libmarias3 +++ b/libmarias3 @@ -1 +1 @@ -Subproject commit aef6eaca141f1754c1839bdd4707c5ae2c1bf061 +Subproject commit fa4c00da11781ad8bd37778f66ad35704827d04b diff --git a/src/Cache.cpp b/src/Cache.cpp index 85fd6f2bd..accfa1118 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -85,7 +85,7 @@ Cache::Cache() : currentCacheSize(0) } catch (exception &e) { - syslog(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what()); + logger->log(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what()); throw e; } //cout << "Cache got prefix " << prefix << endl; @@ -209,7 +209,8 @@ void Cache::read(const vector &keys) if (mit != m_lru.end()) { lru.splice(lru.end(), lru, mit->lit); - removeFromDNE(lru.end()); + LRU_t::iterator lit = lru.end(); + removeFromDNE(--lit); } else if (dl_errnos[i] == 0) // successful download { diff --git a/src/IOCoordinator.cpp b/src/IOCoordinator.cpp index 5a67ff764..9bc59bf08 100755 --- a/src/IOCoordinator.cpp +++ b/src/IOCoordinator.cpp @@ -44,6 +44,7 @@ IOCoordinator::IOCoordinator() cerr << "ObjectStorage/object_size must be set to a numeric value" << endl; throw; } + cachePath = cache->getCachePath(); journalPath = cache->getJournalPath(); } @@ -93,6 +94,7 @@ int IOCoordinator::loadObject(int fd, uint8_t *data, off_t offset, size_t length } count += err; } + return 0; } int IOCoordinator::loadObjectAndJournal(const char *objFilename, const char *journalFilename, @@ -124,9 +126,16 @@ int IOCoordinator::read(const char *filename, uint8_t *data, off_t offset, size_ release read lock put together the response in data */ - + ScopedReadLock fileLock(this, filename); - MetadataFile meta(filename); + MetadataFile meta(filename, MetadataFile::no_create_t()); + + if (!meta.exists()) + { + errno = ENOENT; + return -1; + } + vector relevants = meta.metadataRead(offset, length); map journalFDs, objectFDs; map keyToJournalName, keyToObjectName; @@ -153,10 +162,10 @@ int IOCoordinator::read(const char *filename, uint8_t *data, off_t offset, size_ if (fd >= 0) { keyToJournalName[key] = filename; - journalFDs[filename] = fd; + journalFDs[key] = fd; fdMinders.push_back(SharedCloser(fd)); } - else if (errno != EEXIST) + else if (errno != ENOENT) { int l_errno = errno; logger->log(LOG_CRIT, "IOCoordinator::read(): Got an unexpected error opening %s, error was '%s'", @@ -196,9 +205,9 @@ int IOCoordinator::read(const char *filename, uint8_t *data, off_t offset, size_ // if this is the last object, the length of the read is length - count, // otherwise it is the length of the object size_t thisLength = min(object.length, length - count); - if (jit == journalFDs.end()) + if (jit == journalFDs.end()) err = loadObject(objectFDs[object.key], &data[count], thisOffset, thisLength); - else + else err = loadObjectAndJournal(keyToObjectName[object.key].c_str(), keyToJournalName[object.key].c_str(), &data[count], thisOffset, thisLength); if (err) diff --git a/src/MetadataFile.cpp b/src/MetadataFile.cpp index 65b22aec8..d5ec29607 100755 --- a/src/MetadataFile.cpp +++ b/src/MetadataFile.cpp @@ -47,18 +47,19 @@ MetadataFile::MetadataFile() mpLogger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set"); throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); } - boost::filesystem::create_directories(msMetadataPath); + try { boost::filesystem::create_directories(msMetadataPath); } catch (exception &e) { - syslog(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what()); + mpLogger->log(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what()); throw e; } mVersion=1; mRevision=1; + _exists = false; } @@ -67,6 +68,7 @@ MetadataFile::MetadataFile(const char* filename) mpConfig = Config::get(); mpLogger = SMLogging::get(); mObjectSize = 5 * (1<<20); + _exists = true; try { mObjectSize = stoul(mpConfig->getValue("ObjectStorage", "object_size")); @@ -122,11 +124,77 @@ MetadataFile::MetadataFile(const char* filename) } } +MetadataFile::MetadataFile(const char* filename, no_create_t) +{ + mpConfig = Config::get(); + mpLogger = SMLogging::get(); + mObjectSize = 5 * (1<<20); + try + { + mObjectSize = stoul(mpConfig->getValue("ObjectStorage", "object_size")); + } + catch (...) + { + cerr << "ObjectStorage/object_size must be set to a numeric value" << endl; + throw; + } + try + { + msMetadataPath = mpConfig->getValue("ObjectStorage", "metadata_path"); + } + catch (...) + { + mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file."); + throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); + } + boost::filesystem::create_directories(msMetadataPath); + try + { + boost::filesystem::create_directories(msMetadataPath); + } + catch (exception &e) + { + syslog(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what()); + throw e; + } + string metadataFilename = msMetadataPath + "/" + string(filename) + ".meta"; + if (boost::filesystem::exists(metadataFilename)) + { + _exists = true; + boost::property_tree::ptree jsontree; + boost::property_tree::read_json(metadataFilename, jsontree); + metadataObject newObject; + //try catch + mVersion = jsontree.get("version"); + mRevision = jsontree.get("revision"); + + BOOST_FOREACH(const boost::property_tree::ptree::value_type &v, jsontree.get_child("objects")) + { + metadataObject newObject; + newObject.offset = v.second.get("offset"); + newObject.length = v.second.get("length"); + newObject.key = v.second.get("key"); + mObjects.insert(newObject); + } + } + else + { + mVersion = 1; + mRevision = 1; + _exists = false; + } +} + MetadataFile::~MetadataFile() { } +bool MetadataFile::exists() +{ + return _exists; +} + vector MetadataFile::metadataRead(off_t offset, size_t length) { // this version assumes mObjects is sorted by offset, and there are no gaps between objects @@ -224,6 +292,7 @@ int MetadataFile::writeMetadata(const char *filename) } jsontree.add_child("objects", objs); write_json(metadataFilename, jsontree); + _exists = true; return error; } diff --git a/src/MetadataFile.h b/src/MetadataFile.h index ca9bf270e..fa9979bbe 100755 --- a/src/MetadataFile.h +++ b/src/MetadataFile.h @@ -26,10 +26,13 @@ struct metadataObject { class MetadataFile { public: + struct no_create_t {}; MetadataFile(); MetadataFile(const char* filename); + MetadataFile(const char* filename, no_create_t); // this one won't create it if it doesn't exist ~MetadataFile(); + bool exists(); void printObjects(); // returns the objects needed to update std::vector metadataRead(off_t offset, size_t length); @@ -59,6 +62,7 @@ class MetadataFile size_t mObjectSize; std::string msMetadataPath; std::set mObjects; + bool _exists; //vector mObjects; }; diff --git a/src/Utilities.cpp b/src/Utilities.cpp index 7f58c6dc1..ee9000d73 100644 --- a/src/Utilities.cpp +++ b/src/Utilities.cpp @@ -4,7 +4,7 @@ namespace storagemanager { -ScopedReadLock::ScopedReadLock(IOCoordinator *i, const std::string &k) : ioc(i), key(k) +ScopedReadLock::ScopedReadLock(IOCoordinator *i, const std::string &k) : ioc(i), key(k), locked(false) { lock(); } @@ -29,7 +29,7 @@ void ScopedReadLock::unlock() } } -ScopedWriteLock::ScopedWriteLock(IOCoordinator *i, const std::string &k) : ioc(i), key(k) +ScopedWriteLock::ScopedWriteLock(IOCoordinator *i, const std::string &k) : ioc(i), key(k), locked(false) { lock(); } diff --git a/src/unit_tests.cpp b/src/unit_tests.cpp index 3286e742a..1d1fcaacc 100755 --- a/src/unit_tests.cpp +++ b/src/unit_tests.cpp @@ -638,6 +638,7 @@ bool cacheTest1() cout << "cache test 1 OK" << endl; } +// (ints) 0 1 2 3 ... 2048 void makeTestObject(const char *dest) { int objFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600); @@ -665,7 +666,8 @@ void makeTestJournal(const char *dest) assert(write(journalFD, &i, 4) == 4); } -const char *testObjKey = "12345_0_8192_test-object"; +const char *testObjKey = "12345_0_8192_test-file"; +const char *testFile = "test-file"; void makeTestMetadata(const char *dest) { @@ -964,6 +966,86 @@ void s3storageTest1() cout << "S3Storage Test 1 OK" << endl; } +void IOCReadTest1() +{ + /* Generate the test object & metadata + read it, verify result + + Generate the journal object + read it, verify the merged result + + TODO: do partial reads with an offset similar to what the mergeJournal tests do + TODO: some error path testing + */ + Cache *cache = Cache::get(); + CloudStorage *cs = CloudStorage::get(); + IOCoordinator *ioc = IOCoordinator::get(); + Config *config = Config::get(); + LocalStorage *ls = dynamic_cast(cs); + if (!ls) + { + cout << "IOC read test 1 requires LocalStorage for now." << endl; + return; + } + + bf::path storagePath = ls->getPrefix(); + bf::path cachePath = cache->getCachePath(); + bf::path journalPath = cache->getJournalPath(); + bf::path metaPath = config->getValue("ObjectStorage", "metadata_path"); + assert(!metaPath.empty()); + bf::create_directories(metaPath); + + string objFilename = (storagePath/testObjKey).string(); + string journalFilename = (journalPath/testObjKey).string() + ".journal"; + string metaFilename = (metaPath/testFile).string() + ".meta"; + + cache->reset(); + bf::remove(objFilename); + bf::remove(journalFilename); + bf::remove(metaFilename); + + int err; + boost::scoped_array data(new uint8_t[1<<20]); + memset(data.get(), 0, 1<<20); + err = ioc->read(testFile, data.get(), 0, 1<<20); + assert(err < 0); + assert(errno == ENOENT); + + makeTestObject(objFilename.c_str()); + makeTestMetadata(metaFilename.c_str()); + size_t objSize = bf::file_size(objFilename); + err = ioc->read(testFile, data.get(), 0, 1<<20); + assert(err == objSize); + + // verify the data + int *data32 = (int *) data.get(); + int i; + for (i = 0; i < 2048; i++) + assert(data32[i] == i); + for (; i < (1<<20)/4; i++) + assert(data32[i] == 0); + + makeTestJournal(journalFilename.c_str()); + + err = ioc->read(testFile, data.get(), 0, 1<<20); + assert(err == objSize); + for (i = 0; i < 5; i++) + assert(data32[i] == i); + for (; i < 10; i++) + assert(data32[i] == i-5); + for (; i < 2048; i++) + assert(data32[i] == i); + for (; i < (1<<20)/4; i++) + assert(data32[i] == 0); + + cache->reset(); + bf::remove(objFilename); + bf::remove(journalFilename); + bf::remove(metaFilename); + + cout << "IOC read test 1 OK" << endl; +} + int main() { std::size_t sizeKB = 1024; @@ -1003,6 +1085,7 @@ int main() syncTest1(); s3storageTest1(); + IOCReadTest1(); return 0; }