Got a basic unit test to work for IOC::read()

src/Cache.cpp

@@ -85,7 +85,7 @@ Cache::Cache() : currentCacheSize(0)
     }
     catch (exception &e)
     {
-        syslog(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what());
+        logger->log(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what());
         throw e;
     }
     //cout << "Cache got prefix " << prefix << endl;
@@ -209,7 +209,8 @@ void Cache::read(const vector<string> &keys)
         if (mit != m_lru.end())
         {
             lru.splice(lru.end(), lru, mit->lit);
-            removeFromDNE(lru.end());
+            LRU_t::iterator lit = lru.end();
+            removeFromDNE(--lit);
         }
         else if (dl_errnos[i] == 0)   // successful download
         {

src/IOCoordinator.cpp

@@ -44,6 +44,7 @@ IOCoordinator::IOCoordinator()
         cerr << "ObjectStorage/object_size must be set to a numeric value" << endl;
         throw;
     }
+
     cachePath = cache->getCachePath();
     journalPath = cache->getJournalPath();
 }
@@ -93,6 +94,7 @@ int IOCoordinator::loadObject(int fd, uint8_t *data, off_t offset, size_t length
         }
         count += err;
     }
+    return 0;
 }
 
 int IOCoordinator::loadObjectAndJournal(const char *objFilename, const char *journalFilename, 
@@ -124,9 +126,16 @@ int IOCoordinator::read(const char *filename, uint8_t *data, off_t offset, size_
         release read lock
         put together the response in data
     */
-
+    
     ScopedReadLock fileLock(this, filename);
-    MetadataFile meta(filename);
+    MetadataFile meta(filename, MetadataFile::no_create_t());
+    
+    if (!meta.exists())
+    {
+        errno = ENOENT;
+        return -1;
+    }
+    
     vector<metadataObject> relevants = meta.metadataRead(offset, length);
     map<string, int> journalFDs, objectFDs;
     map<string, string> keyToJournalName, keyToObjectName;
@@ -153,10 +162,10 @@ int IOCoordinator::read(const char *filename, uint8_t *data, off_t offset, size_
         if (fd >= 0)
         {
             keyToJournalName[key] = filename;
-            journalFDs[filename] = fd;
+            journalFDs[key] = fd;
             fdMinders.push_back(SharedCloser(fd));
         }
-        else if (errno != EEXIST)
+        else if (errno != ENOENT)
         {
             int l_errno = errno;
             logger->log(LOG_CRIT, "IOCoordinator::read(): Got an unexpected error opening %s, error was '%s'",
@@ -196,9 +205,9 @@ int IOCoordinator::read(const char *filename, uint8_t *data, off_t offset, size_
         // if this is the last object, the length of the read is length - count,
         // otherwise it is the length of the object
         size_t thisLength = min(object.length, length - count);
-        if (jit == journalFDs.end())
+        if (jit == journalFDs.end()) 
             err = loadObject(objectFDs[object.key], &data[count], thisOffset, thisLength);
-        else
+        else 
             err = loadObjectAndJournal(keyToObjectName[object.key].c_str(), keyToJournalName[object.key].c_str(), 
                 &data[count], thisOffset, thisLength);
         if (err) 

src/MetadataFile.cpp

@@ -47,18 +47,19 @@ MetadataFile::MetadataFile()
         mpLogger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set");
         throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
     }
-    boost::filesystem::create_directories(msMetadataPath);
+
     try
     {
         boost::filesystem::create_directories(msMetadataPath);
     }
     catch (exception &e)
     {
-        syslog(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what());
+        mpLogger->log(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what());
         throw e;
     }
     mVersion=1;
     mRevision=1;
+    _exists = false;
 }
 
 
@@ -67,6 +68,7 @@ MetadataFile::MetadataFile(const char* filename)
     mpConfig = Config::get();
     mpLogger = SMLogging::get();
     mObjectSize = 5 * (1<<20);
+    _exists = true;
     try
     {
         mObjectSize = stoul(mpConfig->getValue("ObjectStorage", "object_size"));
@@ -122,11 +124,77 @@ MetadataFile::MetadataFile(const char* filename)
     }
 }
 
+MetadataFile::MetadataFile(const char* filename, no_create_t)
+{
+    mpConfig = Config::get();
+    mpLogger = SMLogging::get();
+    mObjectSize = 5 * (1<<20);
+    try
+    {
+        mObjectSize = stoul(mpConfig->getValue("ObjectStorage", "object_size"));
+    }
+    catch (...)
+    {
+        cerr << "ObjectStorage/object_size must be set to a numeric value" << endl;
+        throw;
+    }
+    try
+    {
+        msMetadataPath = mpConfig->getValue("ObjectStorage", "metadata_path");
+    }
+    catch (...)
+    {
+        mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file.");
+        throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
+    }
+    boost::filesystem::create_directories(msMetadataPath);
+    try
+    {
+        boost::filesystem::create_directories(msMetadataPath);
+    }
+    catch (exception &e)
+    {
+        syslog(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what());
+        throw e;
+    }
+    string metadataFilename = msMetadataPath + "/" + string(filename) + ".meta";
+    if (boost::filesystem::exists(metadataFilename))
+    {
+        _exists = true;
+        boost::property_tree::ptree jsontree;
+        boost::property_tree::read_json(metadataFilename, jsontree);
+        metadataObject newObject;
+        //try catch
+        mVersion = jsontree.get<int>("version");
+        mRevision = jsontree.get<int>("revision");
+
+        BOOST_FOREACH(const boost::property_tree::ptree::value_type &v, jsontree.get_child("objects"))
+        {
+            metadataObject newObject;
+            newObject.offset = v.second.get<uint64_t>("offset");
+            newObject.length = v.second.get<uint64_t>("length");
+            newObject.key = v.second.get<string>("key");
+            mObjects.insert(newObject);
+        }
+    }
+    else
+    {
+        mVersion = 1;
+        mRevision = 1;
+        _exists = false;
+    }
+}
+
 MetadataFile::~MetadataFile()
 {
 
 }
 
+bool MetadataFile::exists()
+{
+    return _exists;
+}
+
 vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length)
 {
     // this version assumes mObjects is sorted by offset, and there are no gaps between objects
@@ -224,6 +292,7 @@ int MetadataFile::writeMetadata(const char *filename)
     }
     jsontree.add_child("objects", objs);
     write_json(metadataFilename, jsontree);
+    _exists = true;
 
     return error;
 }

src/MetadataFile.h

@@ -26,10 +26,13 @@ struct metadataObject {
 class MetadataFile
 {
     public:
+        struct no_create_t {};
         MetadataFile();
         MetadataFile(const char* filename);
+        MetadataFile(const char* filename, no_create_t);   // this one won't create it if it doesn't exist
         ~MetadataFile();
 
+        bool exists();
         void printObjects();
         // returns the objects needed to update
         std::vector<metadataObject> metadataRead(off_t offset, size_t length);
@@ -59,6 +62,7 @@ class MetadataFile
         size_t mObjectSize;
         std::string msMetadataPath;
         std::set<metadataObject> mObjects;
+        bool _exists;
         //vector<metadataObject> mObjects;
 };
 

src/Utilities.cpp

@@ -4,7 +4,7 @@
 namespace storagemanager
 {
 
-ScopedReadLock::ScopedReadLock(IOCoordinator *i, const std::string &k) : ioc(i), key(k)
+ScopedReadLock::ScopedReadLock(IOCoordinator *i, const std::string &k) : ioc(i), key(k), locked(false)
 {
     lock();
 }
@@ -29,7 +29,7 @@ void ScopedReadLock::unlock()
     }
 }
 
-ScopedWriteLock::ScopedWriteLock(IOCoordinator *i, const std::string &k) : ioc(i), key(k)
+ScopedWriteLock::ScopedWriteLock(IOCoordinator *i, const std::string &k) : ioc(i), key(k), locked(false)
 {
     lock();
 }

src/unit_tests.cpp

@@ -638,6 +638,7 @@ bool cacheTest1()
     cout << "cache test 1 OK" << endl;
 }
 
+// (ints) 0 1 2 3 ... 2048
 void makeTestObject(const char *dest)
 {
     int objFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600);
@@ -665,7 +666,8 @@ void makeTestJournal(const char *dest)
         assert(write(journalFD, &i, 4) == 4);
 }
 
-const char *testObjKey = "12345_0_8192_test-object";
+const char *testObjKey = "12345_0_8192_test-file";
+const char *testFile = "test-file";
 
 void makeTestMetadata(const char *dest)
 {
@@ -964,6 +966,86 @@ void s3storageTest1()
     cout << "S3Storage Test 1 OK" << endl;
 }
 
+void IOCReadTest1()
+{
+    /*  Generate the test object & metadata
+        read it, verify result
+        
+        Generate the journal object
+        read it, verify the merged result
+        
+        TODO: do partial reads with an offset similar to what the mergeJournal tests do
+        TODO: some error path testing
+    */
+    Cache *cache = Cache::get();
+    CloudStorage *cs = CloudStorage::get();
+    IOCoordinator *ioc = IOCoordinator::get();
+    Config *config = Config::get();
+    LocalStorage *ls = dynamic_cast<LocalStorage *>(cs);
+    if (!ls)
+    {
+        cout << "IOC read test 1 requires LocalStorage for now." << endl;
+        return;
+    }
+        
+    bf::path storagePath = ls->getPrefix();
+    bf::path cachePath = cache->getCachePath();
+    bf::path journalPath = cache->getJournalPath();
+    bf::path metaPath = config->getValue("ObjectStorage", "metadata_path");
+    assert(!metaPath.empty());
+    bf::create_directories(metaPath);
+    
+    string objFilename = (storagePath/testObjKey).string();
+    string journalFilename = (journalPath/testObjKey).string() + ".journal";
+    string metaFilename = (metaPath/testFile).string() + ".meta";
+    
+    cache->reset();
+    bf::remove(objFilename);
+    bf::remove(journalFilename);
+    bf::remove(metaFilename);
+    
+    int err;
+    boost::scoped_array<uint8_t> data(new uint8_t[1<<20]);
+    memset(data.get(), 0, 1<<20);
+    err = ioc->read(testFile, data.get(), 0, 1<<20);
+    assert(err < 0);
+    assert(errno == ENOENT);
+    
+    makeTestObject(objFilename.c_str());
+    makeTestMetadata(metaFilename.c_str());
+    size_t objSize = bf::file_size(objFilename);
+    err = ioc->read(testFile, data.get(), 0, 1<<20);
+    assert(err == objSize);
+    
+    // verify the data
+    int *data32 = (int *) data.get();
+    int i;
+    for (i = 0; i < 2048; i++)
+        assert(data32[i] == i);
+    for (; i < (1<<20)/4; i++)
+        assert(data32[i] == 0);
+        
+    makeTestJournal(journalFilename.c_str());
+    
+    err = ioc->read(testFile, data.get(), 0, 1<<20);
+    assert(err == objSize);
+    for (i = 0; i < 5; i++)
+        assert(data32[i] == i);
+    for (; i < 10; i++)
+        assert(data32[i] == i-5);
+    for (; i < 2048; i++)
+        assert(data32[i] == i);
+    for (; i < (1<<20)/4; i++)
+        assert(data32[i] == 0);
+    
+    cache->reset();
+    bf::remove(objFilename);
+    bf::remove(journalFilename);
+    bf::remove(metaFilename);
+    
+    cout << "IOC read test 1 OK" << endl;
+}
+
 int main()
 {
     std::size_t sizeKB = 1024;
@@ -1003,6 +1085,7 @@ int main()
     syncTest1();
     
     s3storageTest1();
+    IOCReadTest1();
 
     return 0;
 }