Changes to MetadataFile for updating entries. changes references to metadata object name to key. enhanced the metadataJournal file unit_test.

src/IOCoordinator.cpp

@@ -105,10 +105,9 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset
 {
     int err = 0;
     uint64_t count = 0;
-    uint64_t writelength = 0;
+    uint64_t writeLength = 0;
     uint64_t dataRemaining = length;
     uint64_t journalOffset = 0;
-    bool updateMeta = false;
     vector<metadataObject> objects;
 
     //writeLock(filename);
@@ -129,20 +128,26 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset
                 // first object in the list so start at offset and
                 // write to end of oject or all the data
                 journalOffset = offset - i->offset;
-                writelength = min((objectSize - journalOffset),dataRemaining);
+                writeLength = min((objectSize - journalOffset),dataRemaining);
             }
             else
             {
                 // starting at beginning of next object write the rest of data
                 // or until object length is reached
-                writelength = min(objectSize,dataRemaining);
+                writeLength = min(objectSize,dataRemaining);
                 journalOffset = 0;
             }
-            err = replicator->addJournalEntry(i->name.c_str(),&data[count],journalOffset,writelength);
+            err = replicator->addJournalEntry(i->key.c_str(),&data[count],journalOffset,writeLength);
             if (err <= 0)
             {
-                //log error and abort
+                metadata.updateEntryLength(i->offset, count);
+                metadata.writeMetadata(filename);
+                logger->log(LOG_ERR,"IOCoordinator::write(): newObject failed to complete write, %u of %u bytes written.",count,length);
+                return count;
             }
+            if ((writeLength + journalOffset) > i->length)
+                metadata.updateEntryLength(i->offset, (writeLength + journalOffset));
+
             count += err;
             dataRemaining -= err;
         }
@@ -154,14 +159,19 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset
     while (dataRemaining > 0 && err >= 0)
     {
         //add a new metaDataObject
-        writelength = min(objectSize,dataRemaining);
+        writeLength = min(objectSize,dataRemaining);
         //cache.makeSpace(size)
-        // add a new metadata object, this will get a new objectKey
+        // add a new metadata object, this will get a new objectKey NOTE: probably needs offset too
-        metadataObject newObject = metadata.addMetadataObject(filename,writelength);
+        metadataObject newObject = metadata.addMetadataObject(filename,writeLength);
         // write the new object
-        err = replicator->newObject(newObject.name.c_str(),data,writelength);
+        err = replicator->newObject(newObject.key.c_str(),data,writeLength);
         if (err <= 0)
         {
+            // update metadataObject length to reflect what awas actually written
+            metadata.updateEntryLength(newObject.offset, count);
+            metadata.writeMetadata(filename);
+            logger->log(LOG_ERR,"IOCoordinator::write(): newObject failed to complete write, %u of %u bytes written.",count,length);
+            return count;
             //log error and abort
         }
         // sync
@@ -170,7 +180,7 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset
         dataRemaining -= err;
     }
 
-    metadata.updateMetadata(filename);
+    metadata.writeMetadata(filename);
     
     //writeUnlock(filename);
 

src/MetadataFile.cpp

@@ -14,6 +14,8 @@
 #define max(x, y) (x > y ? x : y)
 #define min(x, y) (x < y ? x : y)
 
+using namespace std;
+
 namespace storagemanager
 {
 
@@ -56,6 +58,7 @@ MetadataFile::MetadataFile(const char* filename)
         boost::property_tree::ptree jsontree;
         boost::property_tree::read_json(metadataFilename, jsontree);
         metadataObject newObject;
+        //try catch
         mVersion = jsontree.get<int>("version");
         mRevision = jsontree.get<int>("revision");
 
@@ -64,15 +67,15 @@ MetadataFile::MetadataFile(const char* filename)
             metadataObject newObject;
             newObject.offset = v.second.get<uint64_t>("offset");
             newObject.length = v.second.get<uint64_t>("length");
-            newObject.name = v.second.get<string>("name");
+            newObject.key = v.second.get<string>("key");
-            mObjects.push_back(newObject);
+            mObjects.insert(newObject);
         }
     }
     else
     {
         mVersion = 1;
         mRevision = 1;
-        updateMetadata(filename);
+        writeMetadata(filename);
     }
 }
 
@@ -88,7 +91,7 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length)
     uint64_t endData = offset + length;
     uint64_t dataRemaining = length;
     bool foundStart = false;
-    for (std::vector<metadataObject>::iterator i = mObjects.begin(); i != mObjects.end(); ++i)
+    for (std::set<metadataObject>::iterator i = mObjects.begin(); i != mObjects.end(); ++i)
     {
         uint64_t startObject = i->offset;
         uint64_t endObject = i->offset + i->length;
@@ -97,22 +100,12 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length)
         if (startData >= startObject && (startData < endObject || startData < maxEndObject))
         {
             returnObjs.push_back(*i);
-            if (startData >= endObject)
-            {
-                // data starts and the end of current object and can atleast partially fit here update length
-                i->length += min((maxEndObject-startData),dataRemaining);
-            }
             foundStart = true;
         }
         else if (endData >= startObject && (endData < endObject || endData < maxEndObject))
         {
             // data ends in this object
             returnObjs.push_back(*i);
-            if (endData >= endObject)
-            {
-                // data end is beyond old length
-                i->length += (endData - endObject);
-            }
         }
         else if (endData >= startObject && foundStart)
         {
@@ -126,11 +119,14 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length)
 
 metadataObject MetadataFile::addMetadataObject(const char *filename, size_t length)
 {
-    metadataObject addObject,lastObject;
+    // this needs to handle if data write is beyond the end of the last object
+    // but not at start of new object
+    // 
+    metadataObject addObject;
     if (!mObjects.empty())
     {
-        metadataObject lastObject = mObjects.back();
+        std::set<metadataObject>::reverse_iterator iLastObject = mObjects.rbegin();
-        addObject.offset = lastObject.offset + lastObject.length;
+        addObject.offset = iLastObject->offset + iLastObject->length;
     }
     else
     {
@@ -138,26 +134,26 @@ metadataObject MetadataFile::addMetadataObject(const char *filename, size_t leng
     }
     addObject.length = length;
     string newObjectKey = getNewKey(filename, addObject.offset, addObject.length);
-    addObject.name = string(newObjectKey);
+    addObject.key = string(newObjectKey);
-    mObjects.push_back(addObject);
+    mObjects.insert(addObject);
 
     return addObject;
 }
 
 
-int MetadataFile::updateMetadata(const char *filename)
+int MetadataFile::writeMetadata(const char *filename)
 {
     string metadataFilename = string(filename) + ".meta";
     boost::property_tree::ptree jsontree;
     boost::property_tree::ptree objs;
     jsontree.put("version",mVersion);
     jsontree.put("revision",mRevision);
-    for (std::vector<metadataObject>::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i)
+    for (std::set<metadataObject>::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i)
     {
         boost::property_tree::ptree object;
         object.put("offset",i->offset);
         object.put("length",i->length);
-        object.put("name",i->name);
+        object.put("key",i->key);
         objs.push_back(std::make_pair("", object));
     }
     jsontree.add_child("objects", objs);
@@ -234,14 +230,35 @@ void MetadataFile::setLengthInKey(string &key, size_t newLength)
 void MetadataFile::printObjects()
 {
     printf("Version: %i Revision: %i\n",mVersion,mRevision);
-    for (std::vector<metadataObject>::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i)
+    for (std::set<metadataObject>::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i)
     {
-        printf("Name: %s Length: %lu Offset: %lu\n",i->name.c_str(),i->length,i->offset);
+        printf("Name: %s Length: %lu Offset: %lu\n",i->key.c_str(),i->length,i->offset);
     }
 }
 
 void MetadataFile::updateEntry(off_t offset, const string &newName, size_t newLength)
 {
+    metadataObject lookup;
+    lookup.offset = offset;
+    set<metadataObject>::iterator updateObj = mObjects.find(lookup);
+    if (updateObj == mObjects.end())
+    {
+        //throw
+    }
+    updateObj->key = newName;
+    updateObj->length = newLength;
+}
+
+void MetadataFile::updateEntryLength(off_t offset, size_t newLength)
+{
+    metadataObject lookup;
+    lookup.offset = offset;
+    set<metadataObject>::iterator updateObj = mObjects.find(lookup);
+    if (updateObj == mObjects.end())
+    {
+        //throw
+    }
+    updateObj->length = newLength;
 }
 
 }

src/MetadataFile.h

@@ -13,15 +13,13 @@
 #include <iostream>
 #include <set>
 
-using namespace std;
-
 namespace storagemanager
 {
 
 struct metadataObject {
     uint64_t offset;
-    uint64_t length;
+    mutable uint64_t length;
-    string name;
+    mutable std::string key;
     bool operator < (const metadataObject &b) const { return offset < b.offset; }
 };
 
@@ -34,12 +32,13 @@ class MetadataFile
 
         void printObjects();
         // returns the objects needed to update
-        vector<metadataObject> metadataRead(off_t offset, size_t length);
+        std::vector<metadataObject> metadataRead(off_t offset, size_t length);
         // updates the metadatafile with new object
-        int updateMetadata(const char *filename);
+        int writeMetadata(const char *filename);
         
         // updates the name and length fields of an entry, given the offset
         void updateEntry(off_t offset, const std::string &newName, size_t newLength);
+        void updateEntryLength(off_t offset, size_t newLength);
         metadataObject addMetadataObject(const char *filename, size_t length);
 
         // TBD: this may have to go; there may be no use case where only the uuid needs to change.
@@ -58,8 +57,8 @@ class MetadataFile
         int mVersion;
         int mRevision;
         size_t mObjectSize;
-        //set<metadataObject> mObjects;
+        std::set<metadataObject> mObjects;
-        vector<metadataObject> mObjects;
+        //vector<metadataObject> mObjects;
 };
 
 }

src/unit_tests.cpp

@@ -184,13 +184,62 @@ bool replicatorTest()
     return true;
 }
 
-::vector<uint64_t> GenerateData(std::size_t bytes)
+bool metadataJournalTest(std::size_t size, off_t offset)
 {
-    assert(bytes % sizeof(uint64_t) == 0);
+    // make an empty file to write to
-    ::vector<uint64_t> data(bytes / sizeof(uint64_t));
+    const char *filename = "metadataJournalTest";
-    ::iota(data.begin(), data.end(), 0);
+    uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)];
-    ::shuffle(data.begin(), data.end(), std::mt19937{ std::random_device{}() });
+    uint64_t *data;
-    return data;
+
+    sm_msg_header *hdr = (sm_msg_header *) buf;
+    write_cmd *cmd = (write_cmd *) &hdr[1];
+    cmd->opcode = WRITE;
+    cmd->offset = offset;
+    cmd->count = size;
+    cmd->flen = std::strlen(filename);
+    memcpy(&cmd->filename, filename, cmd->flen);
+    data = (uint64_t *) &cmd->filename[cmd->flen];
+    int count = 0;
+    for (uint64_t i = 0; i < (size/sizeof(uint64_t)); i++)
+    {
+        data[i] = i;
+        count++;
+    }
+    hdr->type = SM_MSG_START;
+    hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count;
+    WriteTask w(clientSock, hdr->payloadLen);
+    int error = ::write(sessionSock, cmd, hdr->payloadLen);
+
+    w.run();
+
+    // verify response
+    int err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT);
+    sm_response *resp = (sm_response *) buf;
+    assert(err == sizeof(*resp));
+    assert(resp->header.type == SM_MSG_START);
+    assert(resp->header.payloadLen == 4);
+    assert(resp->header.flags == 0);
+    assert(resp->returnCode == size);
+
+    MetadataFile mdfTest(filename);
+    mdfTest.printObjects();
+
+}
+
+void metadataJournalTestCleanup(std::size_t size)
+{
+    const char *filename = "metadataJournalTest";
+    MetadataFile mdfTest(filename);
+    std::vector<metadataObject> objects = mdfTest.metadataRead(0,size);
+    for (std::vector<metadataObject>::const_iterator i = objects.begin(); i != objects.end(); ++i)
+    {
+        string keyJournal = i->key + ".journal";
+        if(boost::filesystem::exists(i->key.c_str()))
+            ::unlink(i->key.c_str());
+        if(boost::filesystem::exists(keyJournal.c_str()))
+            ::unlink(keyJournal.c_str());
+    }
+    ::unlink("metadataJournalTest.meta");
 }
 
 bool writetask()
@@ -202,46 +251,41 @@ bool writetask()
     assert(fd > 0);
     scoped_closer f(fd);
  
-    std::size_t writeSize = (10 * 1024);
+    uint8_t buf[1024];
-    std::vector<uint64_t> writeData = GenerateData(writeSize);
+    sm_msg_header *hdr = (sm_msg_header *) buf;
-    off_t nextOffset = 0;
+    write_cmd *cmd = (write_cmd *) &hdr[1];
+    uint8_t *data;
 
-    for (std::size_t size = writeSize; size <= (5 * writeSize); size += writeSize)
+    cmd->opcode = WRITE;
-    {
+    cmd->offset = 0;
-        uint8_t buf[(1024 + writeSize)];
+    cmd->count = 9;
-        uint8_t *data;
+    cmd->flen = 10;
-        sm_msg_header *hdr = (sm_msg_header *) buf;
+    memcpy(&cmd->filename, filename, cmd->flen);
-        write_cmd *cmd = (write_cmd *) &hdr[1];
+    data = (uint8_t *) &cmd->filename[cmd->flen];
-        cmd->opcode = WRITE;
+    memcpy(data, "123456789", cmd->count);
-        cmd->offset = nextOffset;
-        cmd->count = writeSize;
-        cmd->flen = 10;
-        memcpy(&cmd->filename, filename, cmd->flen);
 
-        data = (uint8_t *) &cmd->filename[cmd->flen];
+    hdr->type = SM_MSG_START;
-        memcpy(data, &writeData, writeSize);
+    hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count;
 
-        hdr->type = SM_MSG_START;
+    WriteTask w(clientSock, hdr->payloadLen);
-        hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count;
+    ::write(sessionSock, cmd, hdr->payloadLen);
+    w.run();
     
-        WriteTask w(clientSock, hdr->payloadLen);
+    // verify response
-        ::write(sessionSock, cmd, hdr->payloadLen);
+    int err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT);
-        w.run();
+    sm_response *resp = (sm_response *) buf;
+    assert(err == sizeof(*resp));
+    assert(resp->header.type == SM_MSG_START);
+    assert(resp->header.payloadLen == 4);
+    assert(resp->header.flags == 0);
+    assert(resp->returnCode == 9);
     
-        // verify response
+    //check file contents
-        int err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT);
+    err = ::read(fd, buf, 1024);
-        sm_response *resp = (sm_response *) buf;
+    assert(err == 9);
-        assert(err == sizeof(*resp));
+    buf[9] = 0;
-        assert(resp->header.type == SM_MSG_START);
+    assert(!strcmp("123456789", (const char *) buf));
-        assert(resp->header.payloadLen == 4);
+    ::unlink(filename);
-        assert(resp->header.flags == 0);
-        assert(resp->returnCode == writeSize);
-        nextOffset += (writeSize);
-    }
-    // This leaves behind object journal and metadata files currently
-    
-    MetadataFile mdf("./writetest1");
-    mdf.printObjects();
     cout << "write task OK" << endl;
     return true;
 }
@@ -814,14 +858,43 @@ bool syncTest1()
     return true;
 }
 
+void metadataUpdateTest()
+{
+    MetadataFile mdfTest("metadataUpdateTest");
+    mdfTest.addMetadataObject("metadataUpdateTest",100);
+    mdfTest.printObjects();
+    mdfTest.updateEntryLength(0,200);
+    mdfTest.printObjects();
+    //mdfTest.updateEntryLength(0,100);
+    //mdfTest.printObjects();
+    ::unlink("metadataUpdateTest.meta");
+
+    }    
+
 int main()
 {
+    std::size_t sizeKB = 1024;
     cout << "connecting" << endl;
     makeConnection();
     cout << "connected" << endl;
     scoped_closer sc1(serverSock), sc2(sessionSock), sc3(clientSock);
     opentask();
-    writetask();
+    metadataUpdateTest();
+    // requires 8K object size to test boundries
+    //Case 1 new write that spans full object
+    metadataJournalTest((10*sizeKB),0);
+    //Case 2 write data beyond end of data in object 2 that still ends in object 2
+    metadataJournalTest((4*sizeKB),(8*sizeKB));
+    //Case 3 write spans 2 journal objects
+    metadataJournalTest((8*sizeKB),(4*sizeKB));
+    //Case 4 write starts object1 ends object3
+    metadataJournalTest((10*sizeKB),(7*sizeKB));
+    //Case 5 write starts in new object at offset >0
+    //TODO add zero padding to writes in this scenario
+    //metadataJournalTest((8*sizeKB),4*sizeKB);
+    metadataJournalTestCleanup(17*sizeKB);
+
+    //writetask();
     appendtask();
     unlinktask();
     stattask();
@@ -835,5 +908,6 @@ int main()
     mergeJournalTest();
     replicatorTest();
     syncTest1();
+
     return 0;
 }