Update Metadata and Replicator to use configure file paths. Update unit_tests to work and cleanup as expected.

src/MetadataFile.cpp

@@ -33,6 +33,30 @@ MetadataFile::MetadataFile()
         cerr << "ObjectStorage/object_size must be set to a numeric value" << endl;
         throw;
     }
+    try
+    {
+        msMetadataPath = mpConfig->getValue("ObjectStorage", "metadata_path");
+        if (msMetadataPath.empty())
+        {
+            mpLogger->log(LOG_CRIT, "ObjectStorage/journal_path is not set");
+            throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file");
+        }
+    }
+    catch (...)
+    {
+        mpLogger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set");
+        throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
+    }
+    boost::filesystem::create_directories(msMetadataPath);
+    try
+    {
+        boost::filesystem::create_directories(msMetadataPath);
+    }
+    catch (exception &e)
+    {
+        syslog(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what());
+        throw e;
+    }
     mVersion=1;
     mRevision=1;
 }
@@ -52,7 +76,26 @@ MetadataFile::MetadataFile(const char* filename)
         cerr << "ObjectStorage/object_size must be set to a numeric value" << endl;
         throw;
     }
-    string metadataFilename = string(filename) + ".meta";
+    try
+    {
+        msMetadataPath = mpConfig->getValue("ObjectStorage", "metadata_path");
+    }
+    catch (...)
+    {
+        mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file.");
+        throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
+    }
+    boost::filesystem::create_directories(msMetadataPath);
+    try
+    {
+        boost::filesystem::create_directories(msMetadataPath);
+    }
+    catch (exception &e)
+    {
+        syslog(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what());
+        throw e;
+    }
+    string metadataFilename = msMetadataPath + "/" + string(filename) + ".meta";
     if (boost::filesystem::exists(metadataFilename))
     {
         boost::property_tree::ptree jsontree;
@@ -89,7 +132,6 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length)
     vector<metadataObject> returnObjs;
     uint64_t startData = offset;
     uint64_t endData = offset + length;
-    uint64_t dataRemaining = length;
     bool foundStart = false;
     for (std::set<metadataObject>::iterator i = mObjects.begin(); i != mObjects.end(); ++i)
     {
@@ -143,7 +185,9 @@ metadataObject MetadataFile::addMetadataObject(const char *filename, size_t leng
 
 int MetadataFile::writeMetadata(const char *filename)
 {
-    string metadataFilename = string(filename) + ".meta";
+    int error=0;
+
+    string metadataFilename = msMetadataPath + "/" + string(filename) + ".meta";
     boost::property_tree::ptree jsontree;
     boost::property_tree::ptree objs;
     jsontree.put("version",mVersion);
@@ -158,6 +202,8 @@ int MetadataFile::writeMetadata(const char *filename)
     }
     jsontree.add_child("objects", objs);
     write_json(metadataFilename, jsontree);
+
+    return error;
 }
 
 string MetadataFile::getNewKeyFromOldKey(const string &key, size_t length)

src/MetadataFile.h

@@ -57,6 +57,7 @@ class MetadataFile
         int mVersion;
         int mRevision;
         size_t mObjectSize;
+        std::string msMetadataPath;
         std::set<metadataObject> mObjects;
         //vector<metadataObject> mObjects;
 };

src/Replicator.cpp

@@ -26,6 +26,32 @@ namespace storagemanager
 
 Replicator::Replicator()
 {
+    mpConfig = Config::get();
+    mpLogger = SMLogging::get();
+    try
+    {
+        msJournalPath = mpConfig->getValue("ObjectStorage", "journal_path");
+        if (msJournalPath.empty())
+        {
+            mpLogger->log(LOG_CRIT, "ObjectStorage/journal_path is not set");
+            throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file");
+        }
+    }
+    catch (...)
+    {
+        mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file.");
+        throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
+    }
+    boost::filesystem::create_directories(msJournalPath);
+    try
+    {
+        boost::filesystem::create_directories(msJournalPath);
+    }
+    catch (exception &e)
+    {
+        syslog(LOG_CRIT, "Failed to create %s, got: %s", msJournalPath.c_str(), e.what());
+        throw e;
+    }
 }
 
 Replicator::~Replicator()
@@ -84,7 +110,7 @@ int Replicator::addJournalEntry(const char *filename, const uint8_t *data, off_t
     uint64_t offlen[] = {offset,length};
     size_t count = 0;
     int version = 1;
-    string journalFilename = string(filename) + ".journal";
+    string journalFilename = msJournalPath + "/" + string(filename) + ".journal";
     uint64_t thisEntryMaxOffset = (offset + length - 1);
     if (!boost::filesystem::exists(journalFilename))
     {

src/Replicator.h

@@ -34,6 +34,9 @@ class Replicator
 
     private:
         Replicator();
+        Config *mpConfig;
+        SMLogging *mpLogger;
+        std::string msJournalPath;
         //ThreadPool threadPool;
 };
 

src/Synchronizer.cpp

@@ -422,10 +422,7 @@ void Synchronizer::synchronizeWithJournal(const string &sourceFile, list<string>
     }
     
     // update the metadata for the source file
-    // note: a temporary fix.  Metadatafile needs the full path to the metadata atm.  Fix this
-    // once MDF knows where to look.
-    string metaPrefix = Config::get()->getValue("ObjectStorage", "metadata_path");
-    MetadataFile md((metaPrefix + "/" + sourceFile).c_str());
+    MetadataFile md(sourceFile.c_str());
     md.updateEntry(MetadataFile::getOffsetFromKey(key), newKey, size);
     replicator->updateMetadata(sourceFile.c_str(), md);
 

src/unit_tests.cpp

@@ -142,17 +142,20 @@ bool opentask()
 
 bool replicatorTest()
 {
+    Config* config = Config::get();
+    string metaPath = config->getValue("ObjectStorage", "metadata_path");
+    string journalPath = config->getValue("ObjectStorage", "journal_path");
     Replicator *repli = Replicator::get();
     int err,fd;
     const char *newobject = "newobjectTest";
     const char *newobjectJournal = "newobjectTest.journal";
+    string newObjectJournalFullPath = journalPath + "/" + "newobjectTest.journal";
     uint8_t buf[1024];
     uint8_t data[1024];
     int version = 1;
     uint64_t max_offset = 0;
     memcpy(data,"1234567890",10);
     string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % max_offset).str();
-    ::pwrite(fd, header.c_str(), header.length() + 1,0);
 
     // test newObject
     repli->newObject(newobject,data,10);
@@ -169,7 +172,7 @@ bool replicatorTest()
     // test addJournalEntry
     repli->addJournalEntry(newobject,data,0,10);
 
-    fd = ::open(newobjectJournal, O_RDONLY);
+    fd = ::open(newObjectJournalFullPath.c_str(), O_RDONLY);
     err = ::read(fd, buf, 1024);
     assert(err == (header.length() + 1 + 16 + 10));
     buf[err] = 0;
@@ -178,7 +181,7 @@ bool replicatorTest()
     ::close(fd);
 
     repli->remove(newobject);
-    repli->remove(newobjectJournal);
+    repli->remove(newObjectJournalFullPath.c_str());
     assert(!boost::filesystem::exists(newobject));
     cout << "replicator remove OK" << endl;
     return true;
@@ -228,18 +231,22 @@ bool metadataJournalTest(std::size_t size, off_t offset)
 
 void metadataJournalTestCleanup(std::size_t size)
 {
+    Config* config = Config::get();
+    string metaPath = config->getValue("ObjectStorage", "metadata_path");
+    string journalPath = config->getValue("ObjectStorage", "journal_path");
     const char *filename = "metadataJournalTest";
     MetadataFile mdfTest(filename);
     std::vector<metadataObject> objects = mdfTest.metadataRead(0,size);
     for (std::vector<metadataObject>::const_iterator i = objects.begin(); i != objects.end(); ++i)
     {
-        string keyJournal = i->key + ".journal";
+        string keyJournal = journalPath + "/" + i->key + ".journal";
         if(boost::filesystem::exists(i->key.c_str()))
             ::unlink(i->key.c_str());
         if(boost::filesystem::exists(keyJournal.c_str()))
             ::unlink(keyJournal.c_str());
     }
-    ::unlink("metadataJournalTest.meta");
+    string mdfFile = metaPath + "/" + "metadataJournalTest.meta";
+    ::unlink(mdfFile.c_str());
 }
 
 bool writetask()
@@ -802,7 +809,7 @@ bool syncTest1()
     
     // check that the original objects no longer exist
     assert(!cache->exists(key));
-    assert(!bf::exists(journalPath / journalName));
+    assert(!bf::exists(journalPath/journalName));
     
     // Replicator doesn't implement all of its functionality yet, need to delete key from the cache manually for now
     bf::remove(cachePath/key);
@@ -857,6 +864,7 @@ bool syncTest1()
     for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir)
         keys.push_back(dir->path().filename().string());
     sync->deletedObjects(keys);
+    ::unlink((metaPath/"test-file.meta").string().c_str());
     
     cout << "Sync test 1 OK" << endl;
     return true;
@@ -864,6 +872,8 @@ bool syncTest1()
 
 void metadataUpdateTest()
 {
+    Config* config = Config::get();
+    string metaPath = config->getValue("ObjectStorage", "metadata_path");
     MetadataFile mdfTest("metadataUpdateTest");
     mdfTest.addMetadataObject("metadataUpdateTest",100);
     mdfTest.printObjects();
@@ -871,7 +881,8 @@ void metadataUpdateTest()
     mdfTest.printObjects();
     //mdfTest.updateEntryLength(0,100);
     //mdfTest.printObjects();
-    ::unlink("metadataUpdateTest.meta");
+    string metaFilePath = metaPath + "/" + "metadataUpdateTest.meta";
+    ::unlink(metaFilePath.c_str());
 
 }