database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-12-15 12:09:09 +03:00
Code Activity

Changes to MetadataFile for updating entries. changes references to metadata object name to key. enhanced the metadataJournal file unit_test.

Browse Source
This commit is contained in:
Ben Thompson
2019-03-25 17:23:22 -05:00
parent bb5a3d6127
commit e87e9fd5ae
4 changed files with 187 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/IOCoordinator.cpp
View File

@@ -105,10 +105,9 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset
{
int err = 0;
uint64_t count = 0;
uint64_t writelength = 0;
uint64_t writeLength = 0;
uint64_t dataRemaining = length;
uint64_t journalOffset = 0;
bool updateMeta = false;
vector<metadataObject> objects;
//writeLock(filename);
@@ -129,20 +128,26 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset
// first object in the list so start at offset and
// write to end of oject or all the data
journalOffset = offset - i->offset;
writelength = min((objectSize - journalOffset),dataRemaining);
writeLength = min((objectSize - journalOffset),dataRemaining);
}
else
{
// starting at beginning of next object write the rest of data
// or until object length is reached
writelength = min(objectSize,dataRemaining);
writeLength = min(objectSize,dataRemaining);
journalOffset = 0;
}
err = replicator->addJournalEntry(i->name.c_str(),&data[count],journalOffset,writelength);
err = replicator->addJournalEntry(i->key.c_str(),&data[count],journalOffset,writeLength);
if (err <= 0)
{
//log error and abort
metadata.updateEntryLength(i->offset, count);
metadata.writeMetadata(filename);
logger->log(LOG_ERR,"IOCoordinator::write(): newObject failed to complete write, %u of %u bytes written.",count,length);
return count;
}
if ((writeLength + journalOffset) > i->length)
metadata.updateEntryLength(i->offset, (writeLength + journalOffset));
count += err;
dataRemaining -= err;
}
@@ -154,14 +159,19 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset
while (dataRemaining > 0 && err >= 0)
{
//add a new metaDataObject
writelength = min(objectSize,dataRemaining);
writeLength = min(objectSize,dataRemaining);
//cache.makeSpace(size)
// add a new metadata object, this will get a new objectKey
metadataObject newObject = metadata.addMetadataObject(filename,writelength);
// add a new metadata object, this will get a new objectKey NOTE: probably needs offset too
metadataObject newObject = metadata.addMetadataObject(filename,writeLength);
// write the new object
err = replicator->newObject(newObject.name.c_str(),data,writelength);
err = replicator->newObject(newObject.key.c_str(),data,writeLength);
if (err <= 0)
{
// update metadataObject length to reflect what awas actually written
metadata.updateEntryLength(newObject.offset, count);
metadata.writeMetadata(filename);
logger->log(LOG_ERR,"IOCoordinator::write(): newObject failed to complete write, %u of %u bytes written.",count,length);
return count;
//log error and abort
}
// sync
@@ -170,7 +180,7 @@ int IOCoordinator::write(const char *filename, const uint8_t *data, off_t offset
dataRemaining -= err;
}
metadata.updateMetadata(filename);
metadata.writeMetadata(filename);
//writeUnlock(filename);

65
src/MetadataFile.cpp
View File

@@ -14,6 +14,8 @@
#define max(x, y) (x > y ? x : y)
#define min(x, y) (x < y ? x : y)
using namespace std;
namespace storagemanager
{
@@ -56,6 +58,7 @@ MetadataFile::MetadataFile(const char* filename)
boost::property_tree::ptree jsontree;
boost::property_tree::read_json(metadataFilename, jsontree);
metadataObject newObject;
//try catch
mVersion = jsontree.get<int>("version");
mRevision = jsontree.get<int>("revision");
@@ -64,15 +67,15 @@ MetadataFile::MetadataFile(const char* filename)
metadataObject newObject;
newObject.offset = v.second.get<uint64_t>("offset");
newObject.length = v.second.get<uint64_t>("length");
newObject.name = v.second.get<string>("name");
mObjects.push_back(newObject);
newObject.key = v.second.get<string>("key");
mObjects.insert(newObject);
}
}
else
{
mVersion = 1;
mRevision = 1;
updateMetadata(filename);
writeMetadata(filename);
}
}
@@ -88,7 +91,7 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length)
uint64_t endData = offset + length;
uint64_t dataRemaining = length;
bool foundStart = false;
for (std::vector<metadataObject>::iterator i = mObjects.begin(); i != mObjects.end(); ++i)
for (std::set<metadataObject>::iterator i = mObjects.begin(); i != mObjects.end(); ++i)
{
uint64_t startObject = i->offset;
uint64_t endObject = i->offset + i->length;
@@ -97,22 +100,12 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length)
if (startData >= startObject && (startData < endObject || startData < maxEndObject))
{
returnObjs.push_back(*i);
if (startData >= endObject)
{
// data starts and the end of current object and can atleast partially fit here update length
i->length += min((maxEndObject-startData),dataRemaining);
}
foundStart = true;
}
else if (endData >= startObject && (endData < endObject || endData < maxEndObject))
{
// data ends in this object
returnObjs.push_back(*i);
if (endData >= endObject)
{
// data end is beyond old length
i->length += (endData - endObject);
}
}
else if (endData >= startObject && foundStart)
{
@@ -126,11 +119,14 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length)
metadataObject MetadataFile::addMetadataObject(const char *filename, size_t length)
{
metadataObject addObject,lastObject;
// this needs to handle if data write is beyond the end of the last object
// but not at start of new object
//
metadataObject addObject;
if (!mObjects.empty())
{
metadataObject lastObject = mObjects.back();
addObject.offset = lastObject.offset + lastObject.length;
std::set<metadataObject>::reverse_iterator iLastObject = mObjects.rbegin();
addObject.offset = iLastObject->offset + iLastObject->length;
}
else
{
@@ -138,26 +134,26 @@ metadataObject MetadataFile::addMetadataObject(const char *filename, size_t leng
}
addObject.length = length;
string newObjectKey = getNewKey(filename, addObject.offset, addObject.length);
addObject.name = string(newObjectKey);
mObjects.push_back(addObject);
addObject.key = string(newObjectKey);
mObjects.insert(addObject);
return addObject;
}
int MetadataFile::updateMetadata(const char *filename)
int MetadataFile::writeMetadata(const char *filename)
{
string metadataFilename = string(filename) + ".meta";
boost::property_tree::ptree jsontree;
boost::property_tree::ptree objs;
jsontree.put("version",mVersion);
jsontree.put("revision",mRevision);
for (std::vector<metadataObject>::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i)
for (std::set<metadataObject>::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i)
{
boost::property_tree::ptree object;
object.put("offset",i->offset);
object.put("length",i->length);
object.put("name",i->name);
object.put("key",i->key);
objs.push_back(std::make_pair("", object));
}
jsontree.add_child("objects", objs);
@@ -234,14 +230,35 @@ void MetadataFile::setLengthInKey(string &key, size_t newLength)
void MetadataFile::printObjects()
{
printf("Version: %i Revision: %i\n",mVersion,mRevision);
for (std::vector<metadataObject>::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i)
for (std::set<metadataObject>::const_iterator i = mObjects.begin(); i != mObjects.end(); ++i)
{
printf("Name: %s Length: %lu Offset: %lu\n",i->name.c_str(),i->length,i->offset);
printf("Name: %s Length: %lu Offset: %lu\n",i->key.c_str(),i->length,i->offset);
}
}
void MetadataFile::updateEntry(off_t offset, const string &newName, size_t newLength)
{
metadataObject lookup;
lookup.offset = offset;
set<metadataObject>::iterator updateObj = mObjects.find(lookup);
if (updateObj == mObjects.end())
{
//throw
}
updateObj->key = newName;
updateObj->length = newLength;
}
void MetadataFile::updateEntryLength(off_t offset, size_t newLength)
{
metadataObject lookup;
lookup.offset = offset;
set<metadataObject>::iterator updateObj = mObjects.find(lookup);
if (updateObj == mObjects.end())
{
//throw
}
updateObj->length = newLength;
}
}

15
src/MetadataFile.h
View File

@@ -13,15 +13,13 @@
#include <iostream>
#include <set>
using namespace std;
namespace storagemanager
{
struct metadataObject {
uint64_t offset;
uint64_t length;
string name;
mutable uint64_t length;
mutable std::string key;
bool operator < (const metadataObject &b) const { return offset < b.offset; }
};
@@ -34,12 +32,13 @@ class MetadataFile
void printObjects();
// returns the objects needed to update
vector<metadataObject> metadataRead(off_t offset, size_t length);
std::vector<metadataObject> metadataRead(off_t offset, size_t length);
// updates the metadatafile with new object
int updateMetadata(const char *filename);
int writeMetadata(const char *filename);
// updates the name and length fields of an entry, given the offset
void updateEntry(off_t offset, const std::string &newName, size_t newLength);
void updateEntryLength(off_t offset, size_t newLength);
metadataObject addMetadataObject(const char *filename, size_t length);
// TBD: this may have to go; there may be no use case where only the uuid needs to change.
@@ -58,8 +57,8 @@ class MetadataFile
int mVersion;
int mRevision;
size_t mObjectSize;
//set<metadataObject> mObjects;
vector<metadataObject> mObjects;
std::set<metadataObject> mObjects;
//vector<metadataObject> mObjects;
};
}

124
src/unit_tests.cpp
View File

@@ -184,13 +184,62 @@ bool replicatorTest()
return true;
}
::vector<uint64_t> GenerateData(std::size_t bytes)
bool metadataJournalTest(std::size_t size, off_t offset)
{
assert(bytes % sizeof(uint64_t) == 0);
::vector<uint64_t> data(bytes / sizeof(uint64_t));
::iota(data.begin(), data.end(), 0);
::shuffle(data.begin(), data.end(), std::mt19937{ std::random_device{}() });
return data;
// make an empty file to write to
const char *filename = "metadataJournalTest";
uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)];
uint64_t *data;
sm_msg_header *hdr = (sm_msg_header *) buf;
write_cmd *cmd = (write_cmd *) &hdr[1];
cmd->opcode = WRITE;
cmd->offset = offset;
cmd->count = size;
cmd->flen = std::strlen(filename);
memcpy(&cmd->filename, filename, cmd->flen);
data = (uint64_t *) &cmd->filename[cmd->flen];
int count = 0;
for (uint64_t i = 0; i < (size/sizeof(uint64_t)); i++)
{
data[i] = i;
count++;
}
hdr->type = SM_MSG_START;
hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count;
WriteTask w(clientSock, hdr->payloadLen);
int error = ::write(sessionSock, cmd, hdr->payloadLen);
w.run();
// verify response
int err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT);
sm_response *resp = (sm_response *) buf;
assert(err == sizeof(*resp));
assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4);
assert(resp->header.flags == 0);
assert(resp->returnCode == size);
MetadataFile mdfTest(filename);
mdfTest.printObjects();
}
void metadataJournalTestCleanup(std::size_t size)
{
const char *filename = "metadataJournalTest";
MetadataFile mdfTest(filename);
std::vector<metadataObject> objects = mdfTest.metadataRead(0,size);
for (std::vector<metadataObject>::const_iterator i = objects.begin(); i != objects.end(); ++i)
{
string keyJournal = i->key + ".journal";
if(boost::filesystem::exists(i->key.c_str()))
::unlink(i->key.c_str());
if(boost::filesystem::exists(keyJournal.c_str()))
::unlink(keyJournal.c_str());
}
::unlink("metadataJournalTest.meta");
}
bool writetask()
@@ -202,24 +251,18 @@ bool writetask()
assert(fd > 0);
scoped_closer f(fd);
std::size_t writeSize = (10 * 1024);
std::vector<uint64_t> writeData = GenerateData(writeSize);
off_t nextOffset = 0;
for (std::size_t size = writeSize; size <= (5 * writeSize); size += writeSize)
{
uint8_t buf[(1024 + writeSize)];
uint8_t *data;
uint8_t buf[1024];
sm_msg_header *hdr = (sm_msg_header *) buf;
write_cmd *cmd = (write_cmd *) &hdr[1];
uint8_t *data;
cmd->opcode = WRITE;
cmd->offset = nextOffset;
cmd->count = writeSize;
cmd->offset = 0;
cmd->count = 9;
cmd->flen = 10;
memcpy(&cmd->filename, filename, cmd->flen);
data = (uint8_t *) &cmd->filename[cmd->flen];
memcpy(data, &writeData, writeSize);
memcpy(data, "123456789", cmd->count);
hdr->type = SM_MSG_START;
hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count;
@@ -235,13 +278,14 @@ bool writetask()
assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4);
assert(resp->header.flags == 0);
assert(resp->returnCode == writeSize);
nextOffset += (writeSize);
}
// This leaves behind object journal and metadata files currently
assert(resp->returnCode == 9);
MetadataFile mdf("./writetest1");
mdf.printObjects();
//check file contents
err = ::read(fd, buf, 1024);
assert(err == 9);
buf[9] = 0;
assert(!strcmp("123456789", (const char *) buf));
::unlink(filename);
cout << "write task OK" << endl;
return true;
}
@@ -814,14 +858,43 @@ bool syncTest1()
return true;
}
void metadataUpdateTest()
{
MetadataFile mdfTest("metadataUpdateTest");
mdfTest.addMetadataObject("metadataUpdateTest",100);
mdfTest.printObjects();
mdfTest.updateEntryLength(0,200);
mdfTest.printObjects();
//mdfTest.updateEntryLength(0,100);
//mdfTest.printObjects();
::unlink("metadataUpdateTest.meta");
}
int main()
{
std::size_t sizeKB = 1024;
cout << "connecting" << endl;
makeConnection();
cout << "connected" << endl;
scoped_closer sc1(serverSock), sc2(sessionSock), sc3(clientSock);
opentask();
writetask();
metadataUpdateTest();
// requires 8K object size to test boundries
//Case 1 new write that spans full object
metadataJournalTest((10*sizeKB),0);
//Case 2 write data beyond end of data in object 2 that still ends in object 2
metadataJournalTest((4*sizeKB),(8*sizeKB));
//Case 3 write spans 2 journal objects
metadataJournalTest((8*sizeKB),(4*sizeKB));
//Case 4 write starts object1 ends object3
metadataJournalTest((10*sizeKB),(7*sizeKB));
//Case 5 write starts in new object at offset >0
//TODO add zero padding to writes in this scenario
//metadataJournalTest((8*sizeKB),4*sizeKB);
metadataJournalTestCleanup(17*sizeKB);
//writetask();
appendtask();
unlinktask();
stattask();
@@ -835,5 +908,6 @@ int main()
mergeJournalTest();
replicatorTest();
syncTest1();
return 0;
}