diff --git a/src/MetadataFile.cpp b/src/MetadataFile.cpp index 25c8ea77e..d2fc69ebe 100755 --- a/src/MetadataFile.cpp +++ b/src/MetadataFile.cpp @@ -33,6 +33,30 @@ MetadataFile::MetadataFile() cerr << "ObjectStorage/object_size must be set to a numeric value" << endl; throw; } + try + { + msMetadataPath = mpConfig->getValue("ObjectStorage", "metadata_path"); + if (msMetadataPath.empty()) + { + mpLogger->log(LOG_CRIT, "ObjectStorage/journal_path is not set"); + throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); + } + } + catch (...) + { + mpLogger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set"); + throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); + } + boost::filesystem::create_directories(msMetadataPath); + try + { + boost::filesystem::create_directories(msMetadataPath); + } + catch (exception &e) + { + syslog(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what()); + throw e; + } mVersion=1; mRevision=1; } @@ -52,7 +76,26 @@ MetadataFile::MetadataFile(const char* filename) cerr << "ObjectStorage/object_size must be set to a numeric value" << endl; throw; } - string metadataFilename = string(filename) + ".meta"; + try + { + msMetadataPath = mpConfig->getValue("ObjectStorage", "metadata_path"); + } + catch (...) + { + mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file."); + throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); + } + boost::filesystem::create_directories(msMetadataPath); + try + { + boost::filesystem::create_directories(msMetadataPath); + } + catch (exception &e) + { + syslog(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what()); + throw e; + } + string metadataFilename = msMetadataPath + "/" + string(filename) + ".meta"; if (boost::filesystem::exists(metadataFilename)) { boost::property_tree::ptree jsontree; @@ -89,7 +132,6 @@ vector MetadataFile::metadataRead(off_t offset, size_t length) vector returnObjs; uint64_t startData = offset; uint64_t endData = offset + length; - uint64_t dataRemaining = length; bool foundStart = false; for (std::set::iterator i = mObjects.begin(); i != mObjects.end(); ++i) { @@ -143,7 +185,9 @@ metadataObject MetadataFile::addMetadataObject(const char *filename, size_t leng int MetadataFile::writeMetadata(const char *filename) { - string metadataFilename = string(filename) + ".meta"; + int error=0; + + string metadataFilename = msMetadataPath + "/" + string(filename) + ".meta"; boost::property_tree::ptree jsontree; boost::property_tree::ptree objs; jsontree.put("version",mVersion); @@ -158,6 +202,8 @@ int MetadataFile::writeMetadata(const char *filename) } jsontree.add_child("objects", objs); write_json(metadataFilename, jsontree); + + return error; } string MetadataFile::getNewKeyFromOldKey(const string &key, size_t length) diff --git a/src/MetadataFile.h b/src/MetadataFile.h index 7ab9a466a..ca9bf270e 100755 --- a/src/MetadataFile.h +++ b/src/MetadataFile.h @@ -57,6 +57,7 @@ class MetadataFile int mVersion; int mRevision; size_t mObjectSize; + std::string msMetadataPath; std::set mObjects; //vector mObjects; }; diff --git a/src/Replicator.cpp b/src/Replicator.cpp index 40fb79fee..4a75f4c23 100755 --- a/src/Replicator.cpp +++ b/src/Replicator.cpp @@ -26,6 +26,32 @@ namespace storagemanager Replicator::Replicator() { + mpConfig = Config::get(); + mpLogger = SMLogging::get(); + try + { + msJournalPath = mpConfig->getValue("ObjectStorage", "journal_path"); + if (msJournalPath.empty()) + { + mpLogger->log(LOG_CRIT, "ObjectStorage/journal_path is not set"); + throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file"); + } + } + catch (...) + { + mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file."); + throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file"); + } + boost::filesystem::create_directories(msJournalPath); + try + { + boost::filesystem::create_directories(msJournalPath); + } + catch (exception &e) + { + syslog(LOG_CRIT, "Failed to create %s, got: %s", msJournalPath.c_str(), e.what()); + throw e; + } } Replicator::~Replicator() @@ -84,7 +110,7 @@ int Replicator::addJournalEntry(const char *filename, const uint8_t *data, off_t uint64_t offlen[] = {offset,length}; size_t count = 0; int version = 1; - string journalFilename = string(filename) + ".journal"; + string journalFilename = msJournalPath + "/" + string(filename) + ".journal"; uint64_t thisEntryMaxOffset = (offset + length - 1); if (!boost::filesystem::exists(journalFilename)) { diff --git a/src/Replicator.h b/src/Replicator.h index e78603b7e..8e05d9f4f 100755 --- a/src/Replicator.h +++ b/src/Replicator.h @@ -34,6 +34,9 @@ class Replicator private: Replicator(); + Config *mpConfig; + SMLogging *mpLogger; + std::string msJournalPath; //ThreadPool threadPool; }; diff --git a/src/Synchronizer.cpp b/src/Synchronizer.cpp index ed5503387..ad5a28129 100644 --- a/src/Synchronizer.cpp +++ b/src/Synchronizer.cpp @@ -422,10 +422,7 @@ void Synchronizer::synchronizeWithJournal(const string &sourceFile, list } // update the metadata for the source file - // note: a temporary fix. Metadatafile needs the full path to the metadata atm. Fix this - // once MDF knows where to look. - string metaPrefix = Config::get()->getValue("ObjectStorage", "metadata_path"); - MetadataFile md((metaPrefix + "/" + sourceFile).c_str()); + MetadataFile md(sourceFile.c_str()); md.updateEntry(MetadataFile::getOffsetFromKey(key), newKey, size); replicator->updateMetadata(sourceFile.c_str(), md); diff --git a/src/unit_tests.cpp b/src/unit_tests.cpp index 8b166d3b5..3286e742a 100755 --- a/src/unit_tests.cpp +++ b/src/unit_tests.cpp @@ -142,17 +142,20 @@ bool opentask() bool replicatorTest() { + Config* config = Config::get(); + string metaPath = config->getValue("ObjectStorage", "metadata_path"); + string journalPath = config->getValue("ObjectStorage", "journal_path"); Replicator *repli = Replicator::get(); int err,fd; const char *newobject = "newobjectTest"; const char *newobjectJournal = "newobjectTest.journal"; + string newObjectJournalFullPath = journalPath + "/" + "newobjectTest.journal"; uint8_t buf[1024]; uint8_t data[1024]; int version = 1; uint64_t max_offset = 0; memcpy(data,"1234567890",10); string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % max_offset).str(); - ::pwrite(fd, header.c_str(), header.length() + 1,0); // test newObject repli->newObject(newobject,data,10); @@ -169,7 +172,7 @@ bool replicatorTest() // test addJournalEntry repli->addJournalEntry(newobject,data,0,10); - fd = ::open(newobjectJournal, O_RDONLY); + fd = ::open(newObjectJournalFullPath.c_str(), O_RDONLY); err = ::read(fd, buf, 1024); assert(err == (header.length() + 1 + 16 + 10)); buf[err] = 0; @@ -178,7 +181,7 @@ bool replicatorTest() ::close(fd); repli->remove(newobject); - repli->remove(newobjectJournal); + repli->remove(newObjectJournalFullPath.c_str()); assert(!boost::filesystem::exists(newobject)); cout << "replicator remove OK" << endl; return true; @@ -228,18 +231,22 @@ bool metadataJournalTest(std::size_t size, off_t offset) void metadataJournalTestCleanup(std::size_t size) { + Config* config = Config::get(); + string metaPath = config->getValue("ObjectStorage", "metadata_path"); + string journalPath = config->getValue("ObjectStorage", "journal_path"); const char *filename = "metadataJournalTest"; MetadataFile mdfTest(filename); std::vector objects = mdfTest.metadataRead(0,size); for (std::vector::const_iterator i = objects.begin(); i != objects.end(); ++i) { - string keyJournal = i->key + ".journal"; + string keyJournal = journalPath + "/" + i->key + ".journal"; if(boost::filesystem::exists(i->key.c_str())) ::unlink(i->key.c_str()); if(boost::filesystem::exists(keyJournal.c_str())) ::unlink(keyJournal.c_str()); } - ::unlink("metadataJournalTest.meta"); + string mdfFile = metaPath + "/" + "metadataJournalTest.meta"; + ::unlink(mdfFile.c_str()); } bool writetask() @@ -802,7 +809,7 @@ bool syncTest1() // check that the original objects no longer exist assert(!cache->exists(key)); - assert(!bf::exists(journalPath / journalName)); + assert(!bf::exists(journalPath/journalName)); // Replicator doesn't implement all of its functionality yet, need to delete key from the cache manually for now bf::remove(cachePath/key); @@ -857,6 +864,7 @@ bool syncTest1() for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir) keys.push_back(dir->path().filename().string()); sync->deletedObjects(keys); + ::unlink((metaPath/"test-file.meta").string().c_str()); cout << "Sync test 1 OK" << endl; return true; @@ -864,6 +872,8 @@ bool syncTest1() void metadataUpdateTest() { + Config* config = Config::get(); + string metaPath = config->getValue("ObjectStorage", "metadata_path"); MetadataFile mdfTest("metadataUpdateTest"); mdfTest.addMetadataObject("metadataUpdateTest",100); mdfTest.printObjects(); @@ -871,7 +881,8 @@ void metadataUpdateTest() mdfTest.printObjects(); //mdfTest.updateEntryLength(0,100); //mdfTest.printObjects(); - ::unlink("metadataUpdateTest.meta"); + string metaFilePath = metaPath + "/" + "metadataUpdateTest.meta"; + ::unlink(metaFilePath.c_str()); }