diff --git a/src/IOCoordinator.cpp b/src/IOCoordinator.cpp index 33a37e9ff..82bb69cf2 100755 --- a/src/IOCoordinator.cpp +++ b/src/IOCoordinator.cpp @@ -105,10 +105,9 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset { int err = 0; uint64_t count = 0; - uint64_t writelength = 0; + uint64_t writeLength = 0; uint64_t dataRemaining = length; uint64_t journalOffset = 0; - bool updateMeta = false; vector objects; //writeLock(filename); @@ -129,20 +128,26 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset // first object in the list so start at offset and // write to end of oject or all the data journalOffset = offset - i->offset; - writelength = min((objectSize - journalOffset),dataRemaining); + writeLength = min((objectSize - journalOffset),dataRemaining); } else { // starting at beginning of next object write the rest of data // or until object length is reached - writelength = min(objectSize,dataRemaining); + writeLength = min(objectSize,dataRemaining); journalOffset = 0; } - err = replicator->addJournalEntry(i->name.c_str(),&data[count],journalOffset,writelength); + err = replicator->addJournalEntry(i->key.c_str(),&data[count],journalOffset,writeLength); if (err <= 0) { - //log error and abort + metadata.updateEntryLength(i->offset, count); + metadata.writeMetadata(filename); + logger->log(LOG_ERR,"IOCoordinator::write(): newObject failed to complete write, %u of %u bytes written.",count,length); + return count; } + if ((writeLength + journalOffset) > i->length) + metadata.updateEntryLength(i->offset, (writeLength + journalOffset)); + count += err; dataRemaining -= err; } @@ -154,14 +159,19 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset while (dataRemaining > 0 && err >= 0) { //add a new metaDataObject - writelength = min(objectSize,dataRemaining); + writeLength = min(objectSize,dataRemaining); //cache.makeSpace(size) - // add a new metadata object, this will get a new objectKey - metadataObject newObject = metadata.addMetadataObject(filename,writelength); + // add a new metadata object, this will get a new objectKey NOTE: probably needs offset too + metadataObject newObject = metadata.addMetadataObject(filename,writeLength); // write the new object - err = replicator->newObject(newObject.name.c_str(),data,writelength); + err = replicator->newObject(newObject.key.c_str(),data,writeLength); if (err <= 0) { + // update metadataObject length to reflect what awas actually written + metadata.updateEntryLength(newObject.offset, count); + metadata.writeMetadata(filename); + logger->log(LOG_ERR,"IOCoordinator::write(): newObject failed to complete write, %u of %u bytes written.",count,length); + return count; //log error and abort } // sync @@ -170,7 +180,7 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset dataRemaining -= err; } - metadata.updateMetadata(filename); + metadata.writeMetadata(filename); //writeUnlock(filename); diff --git a/src/MetadataFile.cpp b/src/MetadataFile.cpp index 02bd1f865..e88edce5d 100755 --- a/src/MetadataFile.cpp +++ b/src/MetadataFile.cpp @@ -14,6 +14,8 @@ #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) +using namespace std; + namespace storagemanager { @@ -56,6 +58,7 @@ MetadataFile::MetadataFile(const char* filename) boost::property_tree::ptree jsontree; boost::property_tree::read_json(metadataFilename, jsontree); metadataObject newObject; + //try catch mVersion = jsontree.get("version"); mRevision = jsontree.get("revision"); @@ -64,15 +67,15 @@ MetadataFile::MetadataFile(const char* filename) metadataObject newObject; newObject.offset = v.second.get("offset"); newObject.length = v.second.get("length"); - newObject.name = v.second.get("name"); - mObjects.push_back(newObject); + newObject.key = v.second.get("key"); + mObjects.insert(newObject); } } else { mVersion = 1; mRevision = 1; - updateMetadata(filename); + writeMetadata(filename); } } @@ -88,7 +91,7 @@ vector MetadataFile::metadataRead(off_t offset, size_t length) uint64_t endData = offset + length; uint64_t dataRemaining = length; bool foundStart = false; - for (std::vector::iterator i = mObjects.begin(); i != mObjects.end(); ++i) + for (std::set::iterator i = mObjects.begin(); i != mObjects.end(); ++i) { uint64_t startObject = i->offset; uint64_t endObject = i->offset + i->length; @@ -97,22 +100,12 @@ vector MetadataFile::metadataRead(off_t offset, size_t length) if (startData >= startObject && (startData < endObject || startData < maxEndObject)) { returnObjs.push_back(*i); - if (startData >= endObject) - { - // data starts and the end of current object and can atleast partially fit here update length - i->length += min((maxEndObject-startData),dataRemaining); - } foundStart = true; } else if (endData >= startObject && (endData < endObject || endData < maxEndObject)) { // data ends in this object returnObjs.push_back(*i); - if (endData >= endObject) - { - // data end is beyond old length - i->length += (endData - endObject); - } } else if (endData >= startObject && foundStart) { @@ -126,11 +119,14 @@ vector MetadataFile::metadataRead(off_t offset, size_t length) metadataObject MetadataFile::addMetadataObject(const char *filename, size_t length) { - metadataObject addObject,lastObject; + // this needs to handle if data write is beyond the end of the last object + // but not at start of new object + // + metadataObject addObject; if (!mObjects.empty()) { - metadataObject lastObject = mObjects.back(); - addObject.offset = lastObject.offset + lastObject.length; + std::set::reverse_iterator iLastObject = mObjects.rbegin(); + addObject.offset = iLastObject->offset + iLastObject->length; } else { @@ -138,26 +134,26 @@ metadataObject MetadataFile::addMetadataObject(const char *filename, size_t leng } addObject.length = length; string newObjectKey = getNewKey(filename, addObject.offset, addObject.length); - addObject.name = string(newObjectKey); - mObjects.push_back(addObject); + addObject.key = string(newObjectKey); + mObjects.insert(addObject); return addObject; } -int MetadataFile::updateMetadata(const char *filename) +int MetadataFile::writeMetadata(const char *filename) { string metadataFilename = string(filename) + ".meta"; boost::property_tree::ptree jsontree; boost::property_tree::ptree objs; jsontree.put("version",mVersion); jsontree.put("revision",mRevision); - for (std::vector::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i) + for (std::set::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i) { boost::property_tree::ptree object; object.put("offset",i->offset); object.put("length",i->length); - object.put("name",i->name); + object.put("key",i->key); objs.push_back(std::make_pair("", object)); } jsontree.add_child("objects", objs); @@ -234,14 +230,35 @@ void MetadataFile::setLengthInKey(string &key, size_t newLength) void MetadataFile::printObjects() { printf("Version: %i Revision: %i\n",mVersion,mRevision); - for (std::vector::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i) + for (std::set::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i) { - printf("Name: %s Length: %lu Offset: %lu\n",i->name.c_str(),i->length,i->offset); + printf("Name: %s Length: %lu Offset: %lu\n",i->key.c_str(),i->length,i->offset); } } void MetadataFile::updateEntry(off_t offset, const string &newName, size_t newLength) { + metadataObject lookup; + lookup.offset = offset; + set::iterator updateObj = mObjects.find(lookup); + if (updateObj == mObjects.end()) + { + //throw + } + updateObj->key = newName; + updateObj->length = newLength; +} + +void MetadataFile::updateEntryLength(off_t offset, size_t newLength) +{ + metadataObject lookup; + lookup.offset = offset; + set::iterator updateObj = mObjects.find(lookup); + if (updateObj == mObjects.end()) + { + //throw + } + updateObj->length = newLength; } } diff --git a/src/MetadataFile.h b/src/MetadataFile.h index d0190c716..0091e16b5 100755 --- a/src/MetadataFile.h +++ b/src/MetadataFile.h @@ -13,15 +13,13 @@ #include #include -using namespace std; - namespace storagemanager { struct metadataObject { uint64_t offset; - uint64_t length; - string name; + mutable uint64_t length; + mutable std::string key; bool operator < (const metadataObject &b) const { return offset < b.offset; } }; @@ -34,12 +32,13 @@ class MetadataFile void printObjects(); // returns the objects needed to update - vector metadataRead(off_t offset, size_t length); + std::vector metadataRead(off_t offset, size_t length); // updates the metadatafile with new object - int updateMetadata(const char *filename); + int writeMetadata(const char *filename); // updates the name and length fields of an entry, given the offset - void updateEntry(off_t offset, const std::string &newName, size_t newLength); + void updateEntry(off_t offset, const std::string &newName, size_t newLength); + void updateEntryLength(off_t offset, size_t newLength); metadataObject addMetadataObject(const char *filename, size_t length); // TBD: this may have to go; there may be no use case where only the uuid needs to change. @@ -58,8 +57,8 @@ class MetadataFile int mVersion; int mRevision; size_t mObjectSize; - //set mObjects; - vector mObjects; + std::set mObjects; + //vector mObjects; }; } diff --git a/src/unit_tests.cpp b/src/unit_tests.cpp index 1b56d8131..fc2060aab 100755 --- a/src/unit_tests.cpp +++ b/src/unit_tests.cpp @@ -184,13 +184,62 @@ bool replicatorTest() return true; } -::vector GenerateData(std::size_t bytes) +bool metadataJournalTest(std::size_t size, off_t offset) { - assert(bytes % sizeof(uint64_t) == 0); - ::vector data(bytes / sizeof(uint64_t)); - ::iota(data.begin(), data.end(), 0); - ::shuffle(data.begin(), data.end(), std::mt19937{ std::random_device{}() }); - return data; + // make an empty file to write to + const char *filename = "metadataJournalTest"; + uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)]; + uint64_t *data; + + sm_msg_header *hdr = (sm_msg_header *) buf; + write_cmd *cmd = (write_cmd *) &hdr[1]; + cmd->opcode = WRITE; + cmd->offset = offset; + cmd->count = size; + cmd->flen = std::strlen(filename); + memcpy(&cmd->filename, filename, cmd->flen); + data = (uint64_t *) &cmd->filename[cmd->flen]; + int count = 0; + for (uint64_t i = 0; i < (size/sizeof(uint64_t)); i++) + { + data[i] = i; + count++; + } + hdr->type = SM_MSG_START; + hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; + WriteTask w(clientSock, hdr->payloadLen); + int error = ::write(sessionSock, cmd, hdr->payloadLen); + + w.run(); + + // verify response + int err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + sm_response *resp = (sm_response *) buf; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == 4); + assert(resp->header.flags == 0); + assert(resp->returnCode == size); + + MetadataFile mdfTest(filename); + mdfTest.printObjects(); + +} + +void metadataJournalTestCleanup(std::size_t size) +{ + const char *filename = "metadataJournalTest"; + MetadataFile mdfTest(filename); + std::vector objects = mdfTest.metadataRead(0,size); + for (std::vector::const_iterator i = objects.begin(); i != objects.end(); ++i) + { + string keyJournal = i->key + ".journal"; + if(boost::filesystem::exists(i->key.c_str())) + ::unlink(i->key.c_str()); + if(boost::filesystem::exists(keyJournal.c_str())) + ::unlink(keyJournal.c_str()); + } + ::unlink("metadataJournalTest.meta"); } bool writetask() @@ -202,46 +251,41 @@ bool writetask() assert(fd > 0); scoped_closer f(fd); - std::size_t writeSize = (10 * 1024); - std::vector writeData = GenerateData(writeSize); - off_t nextOffset = 0; + uint8_t buf[1024]; + sm_msg_header *hdr = (sm_msg_header *) buf; + write_cmd *cmd = (write_cmd *) &hdr[1]; + uint8_t *data; - for (std::size_t size = writeSize; size <= (5 * writeSize); size += writeSize) - { - uint8_t buf[(1024 + writeSize)]; - uint8_t *data; - sm_msg_header *hdr = (sm_msg_header *) buf; - write_cmd *cmd = (write_cmd *) &hdr[1]; - cmd->opcode = WRITE; - cmd->offset = nextOffset; - cmd->count = writeSize; - cmd->flen = 10; - memcpy(&cmd->filename, filename, cmd->flen); + cmd->opcode = WRITE; + cmd->offset = 0; + cmd->count = 9; + cmd->flen = 10; + memcpy(&cmd->filename, filename, cmd->flen); + data = (uint8_t *) &cmd->filename[cmd->flen]; + memcpy(data, "123456789", cmd->count); - data = (uint8_t *) &cmd->filename[cmd->flen]; - memcpy(data, &writeData, writeSize); + hdr->type = SM_MSG_START; + hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; + + WriteTask w(clientSock, hdr->payloadLen); + ::write(sessionSock, cmd, hdr->payloadLen); + w.run(); - hdr->type = SM_MSG_START; - hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; - - WriteTask w(clientSock, hdr->payloadLen); - ::write(sessionSock, cmd, hdr->payloadLen); - w.run(); - - // verify response - int err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == 4); - assert(resp->header.flags == 0); - assert(resp->returnCode == writeSize); - nextOffset += (writeSize); - } - // This leaves behind object journal and metadata files currently + // verify response + int err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + sm_response *resp = (sm_response *) buf; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == 4); + assert(resp->header.flags == 0); + assert(resp->returnCode == 9); - MetadataFile mdf("./writetest1"); - mdf.printObjects(); + //check file contents + err = ::read(fd, buf, 1024); + assert(err == 9); + buf[9] = 0; + assert(!strcmp("123456789", (const char *) buf)); + ::unlink(filename); cout << "write task OK" << endl; return true; } @@ -814,14 +858,43 @@ bool syncTest1() return true; } +void metadataUpdateTest() +{ + MetadataFile mdfTest("metadataUpdateTest"); + mdfTest.addMetadataObject("metadataUpdateTest",100); + mdfTest.printObjects(); + mdfTest.updateEntryLength(0,200); + mdfTest.printObjects(); + //mdfTest.updateEntryLength(0,100); + //mdfTest.printObjects(); + ::unlink("metadataUpdateTest.meta"); + + } + int main() { + std::size_t sizeKB = 1024; cout << "connecting" << endl; makeConnection(); cout << "connected" << endl; scoped_closer sc1(serverSock), sc2(sessionSock), sc3(clientSock); opentask(); - writetask(); + metadataUpdateTest(); + // requires 8K object size to test boundries + //Case 1 new write that spans full object + metadataJournalTest((10*sizeKB),0); + //Case 2 write data beyond end of data in object 2 that still ends in object 2 + metadataJournalTest((4*sizeKB),(8*sizeKB)); + //Case 3 write spans 2 journal objects + metadataJournalTest((8*sizeKB),(4*sizeKB)); + //Case 4 write starts object1 ends object3 + metadataJournalTest((10*sizeKB),(7*sizeKB)); + //Case 5 write starts in new object at offset >0 + //TODO add zero padding to writes in this scenario + //metadataJournalTest((8*sizeKB),4*sizeKB); + metadataJournalTestCleanup(17*sizeKB); + + //writetask(); appendtask(); unlinktask(); stattask(); @@ -835,5 +908,6 @@ int main() mergeJournalTest(); replicatorTest(); syncTest1(); + return 0; }