database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

Checkout unit_test fixes.

Browse Source
This commit is contained in:
benthompson15
2019-08-21 16:04:09 -05:00
parent 77f311c086
commit 5249066ada
1 changed files with 288 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

485
storage-manager/src/unit_tests.cpp
View File

@@ -86,16 +86,25 @@ void makeTestJournal(const char *dest)
scoped_closer s2(journalFD); scoped_closer s2(journalFD);
char header[] = "{ \"version\" : 1, \"max_offset\": 39 }"; char header[] = "{ \"version\" : 1, \"max_offset\": 39 }";
write(journalFD, header, strlen(header) + 1); size_t result = write(journalFD, header, strlen(header) + 1);
assert(result==(strlen(header) + 1));
uint64_t offlen[2] = { 20, 20 }; uint64_t offlen[2] = { 20, 20 };
write(journalFD, offlen, 16); result = write(journalFD, offlen, 16);
for (int i = 0; i < 5; i++) for (int i = 0; i < 5; i++)
assert(write(journalFD, &i, 4) == 4); assert(write(journalFD, &i, 4) == 4);
} }
const char *testObjKey = "12345_0_8192_test-file"; bf::path homepath = getenv("HOME");
const char *testFile = "test-file"; string prefix = "unittest";
string testObjKey = "12345_0_8192_" + prefix +"~test-file";
string copyfile1ObjKey = "12345_0_8192_" + prefix +"~source";
string copyfile3ObjKey = "12345_0_8192_" + prefix +"~source";
string metaTestFile = prefix + "/test-file";
bf::path testFilePath = homepath / metaTestFile;
const char* testFile = testFilePath.string().c_str();
const char *_metadata = const char *_metadata =
"{ \n\ "{ \n\
\"version\" : 1, \n\ \"version\" : 1, \n\
@@ -105,11 +114,38 @@ const char *_metadata =
{ \n\ { \n\
\"offset\" : 0, \n\ \"offset\" : 0, \n\
\"length\" : 8192, \n\ \"length\" : 8192, \n\
\"key\" : \"12345_0_8192_test-file\" \n\ \"key\" : \"12345_0_8192_unittest~test-file\" \n\
} \n\ } \n\
] \n\ ] \n\
}\n"; }\n";
const char *_metadataCopy1 =
"{ \n\
\"version\" : 1, \n\
\"revision\" : 1, \n\
\"objects\" : \n\
[ \n\
{ \n\
\"offset\" : 0, \n\
\"length\" : 8192, \n\
\"key\" : \"12345_0_8192_unittest~source\" \n\
} \n\
] \n\
}\n";
const char *_metadataCopy3 =
"{ \n\
\"version\" : 1, \n\
\"revision\" : 1, \n\
\"objects\" : \n\
[ \n\
{ \n\
\"offset\" : 0, \n\
\"length\" : 8192, \n\
\"key\" : \"12345_0_8192_unittest~source\" \n\
} \n\
] \n\
}\n";
void makeTestMetadata(const char *dest) void makeTestMetadata(const char *dest)
{ {
@@ -118,9 +154,31 @@ void makeTestMetadata(const char *dest)
scoped_closer sc(metaFD); scoped_closer sc(metaFD);
// need to parameterize the object name in the objects list // need to parameterize the object name in the objects list
write(metaFD, _metadata, strlen(_metadata)); size_t result = write(metaFD, _metadata, strlen(_metadata));
assert(result==strlen(_metadata));
} }
void makeTestMetadataCopy1(const char *dest)
{
int metaFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600);
assert(metaFD >= 0);
scoped_closer sc(metaFD);
// need to parameterize the object name in the objects list
size_t result = write(metaFD, _metadataCopy1, strlen(_metadataCopy1));
assert(result==strlen(_metadataCopy1));
}
void makeTestMetadataCopy3(const char *dest)
{
int metaFD = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0600);
assert(metaFD >= 0);
scoped_closer sc(metaFD);
// need to parameterize the object name in the objects list
size_t result = write(metaFD, _metadataCopy3, strlen(_metadataCopy3));
assert(result==strlen(_metadataCopy3));
}
int getSocket() int getSocket()
{ {
int sock = ::socket(AF_UNIX, SOCK_STREAM, 0); int sock = ::socket(AF_UNIX, SOCK_STREAM, 0);
@@ -176,19 +234,21 @@ bool opentask()
uint8_t buf[1024]; uint8_t buf[1024];
sm_msg_header *hdr = (sm_msg_header *) buf; sm_msg_header *hdr = (sm_msg_header *) buf;
open_cmd *cmd = (open_cmd *) &hdr[1]; open_cmd *cmd = (open_cmd *) &hdr[1];
string testFile = "metadataJournalTest";
// open/create a file named 'opentest1' // open/create a file named 'opentest1'
const char *filename = "opentest1"; const char *filename = string(homepath.string()+"/"+prefix+"/"+testFile).c_str();
hdr->type = SM_MSG_START; hdr->type = SM_MSG_START;
hdr->flags = 0; hdr->flags = 0;
hdr->payloadLen = sizeof(*cmd) + 9; hdr->payloadLen = sizeof(*cmd) + 40;
cmd->opcode = OPEN; cmd->opcode = OPEN;
cmd->openmode = O_WRONLY | O_CREAT; cmd->openmode = O_WRONLY | O_CREAT;
cmd->flen = 9; cmd->flen = 19;
strcpy((char *) cmd->filename, filename); strcpy((char *) cmd->filename, filename);
::unlink(filename); ::unlink(filename);
::write(sessionSock, cmd, hdr->payloadLen); ssize_t result = ::write(sessionSock, cmd, hdr->payloadLen);
assert(result==(hdr->payloadLen));
OpenTask o(clientSock, hdr->payloadLen); OpenTask o(clientSock, hdr->payloadLen);
o.run(); o.run();
@@ -197,7 +257,7 @@ bool opentask()
sm_response *resp = (sm_response *) buf; sm_response *resp = (sm_response *) buf;
assert(err == sizeof(struct stat) + sizeof(sm_response)); assert(err == sizeof(struct stat) + sizeof(sm_response));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == sizeof(struct stat) + 4); assert(resp->header.payloadLen == sizeof(struct stat) + sizeof(ssize_t));
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->returnCode == 0); assert(resp->returnCode == 0);
struct stat *_stat = (struct stat *) resp->payload; struct stat *_stat = (struct stat *) resp->payload;
@@ -210,10 +270,10 @@ bool opentask()
/* verify the file is there */ /* verify the file is there */
string metaPath = Config::get()->getValue("ObjectStorage", "metadata_path"); string metaPath = Config::get()->getValue("ObjectStorage", "metadata_path");
assert(!metaPath.empty()); assert(!metaPath.empty());
metaPath += string("/") + filename + ".meta"; metaPath += string("/" + prefix + "/" + testFile + ".meta");
cout << "metaPath " << metaPath.c_str() << endl;
assert(boost::filesystem::exists(metaPath)); assert(boost::filesystem::exists(metaPath));
::unlink(metaPath.c_str());
cout << "opentask OK" << endl; cout << "opentask OK" << endl;
return true; return true;
} }
@@ -227,9 +287,9 @@ bool replicatorTest()
Replicator *repli = Replicator::get(); Replicator *repli = Replicator::get();
int err,fd; int err,fd;
const char *newobject = "newobjectTest"; string newobject = prefix + "/newobjectTest";
string newObjectJournalFullPath = journalPath + "/" + "newobjectTest.journal"; string newObjectJournalFullPath = journalPath + "/" + prefix + "/newobjectTest.journal";
string newObjectCacheFullPath = cacehPath + "/" + "newobjectTest"; string newObjectCacheFullPath = cacehPath + "/" + prefix + "/newobjectTest";
uint8_t buf[1024]; uint8_t buf[1024];
uint8_t data[1024]; uint8_t data[1024];
int version = 1; int version = 1;
@@ -270,7 +330,8 @@ bool replicatorTest()
void metadataJournalTest(std::size_t size, off_t offset) void metadataJournalTest(std::size_t size, off_t offset)
{ {
// make an empty file to write to // make an empty file to write to
const char *filename = "metadataJournalTest"; bf::path fullPath = homepath / prefix / "metadataJournalTest";
const char *filename = fullPath.string().c_str();
uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)]; uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)];
uint64_t *data; uint64_t *data;
@@ -300,7 +361,7 @@ void metadataJournalTest(std::size_t size, off_t offset)
sm_response *resp = (sm_response *) buf; sm_response *resp = (sm_response *) buf;
assert(err == sizeof(*resp)); assert(err == sizeof(*resp));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4); assert(resp->header.payloadLen == sizeof(ssize_t));
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->returnCode == (int) size); assert(resp->returnCode == (int) size);
} }
@@ -308,7 +369,9 @@ void metadataJournalTest(std::size_t size, off_t offset)
void metadataJournalTest_append(std::size_t size) void metadataJournalTest_append(std::size_t size)
{ {
// make an empty file to write to // make an empty file to write to
const char *filename = "metadataJournalTest"; bf::path fullPath = homepath / prefix / "metadataJournalTest";
const char *filename = fullPath.string().c_str();
uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)]; uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)];
uint64_t *data; uint64_t *data;
@@ -337,39 +400,23 @@ void metadataJournalTest_append(std::size_t size)
sm_response *resp = (sm_response *) buf; sm_response *resp = (sm_response *) buf;
assert(err == sizeof(*resp)); assert(err == sizeof(*resp));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4); assert(resp->header.payloadLen == sizeof(ssize_t));
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->returnCode == (int) size); assert(resp->returnCode == (int) size);
} }
void metadataJournalTestCleanup(std::size_t size) void metadataJournalTestCleanup(std::size_t size)
{ {
Config* config = Config::get(); IOCoordinator *ioc = IOCoordinator::get();
string metaPath = config->getValue("ObjectStorage", "metadata_path"); bf::path fullPath = homepath / prefix / "metadataJournalTest";
string journalPath = config->getValue("ObjectStorage", "journal_path"); ioc->unlink(fullPath.string().c_str());
string cachePath = config->getValue("Cache", "path");
const char *filename = "metadataJournalTest";
MetadataFile mdfTest(filename);
mdfTest.printObjects();
std::vector<metadataObject> objects = mdfTest.metadataRead(0,size);
for (std::vector<metadataObject>::const_iterator i = objects.begin(); i != objects.end(); ++i)
{
string casheObject = cachePath + "/" + i->key;
string keyJournal = journalPath + "/" + i->key + ".journal";
if(boost::filesystem::exists(i->key.c_str()))
::unlink(casheObject.c_str());
if(boost::filesystem::exists(keyJournal.c_str()))
::unlink(keyJournal.c_str());
}
string mdfFile = metaPath + "/" + "metadataJournalTest.meta";
::unlink(mdfFile.c_str());
} }
bool writetask() bool writetask()
{ {
// make an empty file to write to // make an empty file to write to
const char *filename = "writetest1"; bf::path fullPath = homepath / prefix / "writetest1";
const char *filename = fullPath.string().c_str();
::unlink(filename); ::unlink(filename);
int fd = ::open(filename, O_CREAT | O_RDWR, 0666); int fd = ::open(filename, O_CREAT | O_RDWR, 0666);
assert(fd > 0); assert(fd > 0);
@@ -392,7 +439,9 @@ bool writetask()
hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; hdr->payloadLen = sizeof(*cmd) + cmd->flen + cmd->count;
WriteTask w(clientSock, hdr->payloadLen); WriteTask w(clientSock, hdr->payloadLen);
::write(sessionSock, cmd, hdr->payloadLen); ssize_t result = ::write(sessionSock, cmd, hdr->payloadLen);
assert(result==(hdr->payloadLen));
w.run(); w.run();
// verify response // verify response
@@ -400,7 +449,7 @@ bool writetask()
sm_response *resp = (sm_response *) buf; sm_response *resp = (sm_response *) buf;
assert(err == sizeof(*resp)); assert(err == sizeof(*resp));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4); assert(resp->header.payloadLen == sizeof(ssize_t));
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->returnCode == 9); assert(resp->returnCode == 9);
@@ -417,7 +466,8 @@ bool writetask()
bool appendtask() bool appendtask()
{ {
// make a file and put some stuff in it // make a file and put some stuff in it
const char *filename = "appendtest1"; bf::path fullPath = homepath / prefix / "appendtest1";
const char *filename = fullPath.string().c_str();
::unlink(filename); ::unlink(filename);
int fd = ::open(filename, O_CREAT | O_RDWR, 0666); int fd = ::open(filename, O_CREAT | O_RDWR, 0666);
assert(fd > 0); assert(fd > 0);
@@ -439,7 +489,9 @@ bool appendtask()
int payloadLen = sizeof(*cmd) + cmd->flen + cmd->count; int payloadLen = sizeof(*cmd) + cmd->flen + cmd->count;
AppendTask a(clientSock, payloadLen); AppendTask a(clientSock, payloadLen);
::write(sessionSock, cmd, payloadLen); ssize_t result = ::write(sessionSock, cmd, payloadLen);
assert(result==(payloadLen));
a.run(); a.run();
// verify response // verify response
@@ -447,7 +499,7 @@ bool appendtask()
sm_response *resp = (sm_response *) buf; sm_response *resp = (sm_response *) buf;
assert(err == sizeof(*resp)); assert(err == sizeof(*resp));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4); assert(resp->header.payloadLen == sizeof(ssize_t));
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->returnCode == 9); assert(resp->returnCode == 9);
@@ -465,13 +517,17 @@ bool appendtask()
void unlinktask() void unlinktask()
{ {
// make a meta file and delete it // make a meta file and delete it
const char *filename = "unlinktest1"; bf::path fullPath = homepath / prefix / "unlinktest1";
string pathMeta = prefix + "/unlinktest1";
const char *Metafilename = pathMeta.c_str();
const char *filename = fullPath.string().c_str();
IOCoordinator *ioc = IOCoordinator::get(); IOCoordinator *ioc = IOCoordinator::get();
bf::path fullPath = ioc->getMetadataPath()/(string(filename) + ".meta"); bf::path fullPathMeta = ioc->getMetadataPath()/(string(Metafilename) + ".meta");
bf::remove(fullPath); bf::remove(fullPathMeta);
MetadataFile meta(filename); MetadataFile meta(Metafilename);
assert(bf::exists(fullPath)); assert(bf::exists(fullPathMeta));
uint8_t buf[1024]; uint8_t buf[1024];
unlink_cmd *cmd = (unlink_cmd *) buf; unlink_cmd *cmd = (unlink_cmd *) buf;
@@ -481,7 +537,8 @@ void unlinktask()
memcpy(&cmd->filename, filename, cmd->flen); memcpy(&cmd->filename, filename, cmd->flen);
UnlinkTask u(clientSock, sizeof(unlink_cmd) + cmd->flen); UnlinkTask u(clientSock, sizeof(unlink_cmd) + cmd->flen);
::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen); size_t result = ::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen);
assert(result==(sizeof(unlink_cmd) + cmd->flen));
u.run(); u.run();
// verify response // verify response
@@ -489,12 +546,12 @@ void unlinktask()
sm_response *resp = (sm_response *) buf; sm_response *resp = (sm_response *) buf;
assert(err == sizeof(*resp)); assert(err == sizeof(*resp));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4); assert(resp->header.payloadLen == sizeof(ssize_t));
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->returnCode == 0); assert(resp->returnCode == 0);
// confirm it no longer exists // confirm it no longer exists
assert(!bf::exists(fullPath)); assert(!bf::exists(fullPathMeta));
// delete it again, make sure we get an error message & reasonable error code // delete it again, make sure we get an error message & reasonable error code
// Interesting. boost::filesystem::remove() doesn't consider it an error if the file doesn't // Interesting. boost::filesystem::remove() doesn't consider it an error if the file doesn't
@@ -508,7 +565,9 @@ void unlinktask()
memcpy(&cmd->filename, filename, cmd->flen); memcpy(&cmd->filename, filename, cmd->flen);
UnlinkTask u2(clientSock, sizeof(unlink_cmd) + cmd->flen); UnlinkTask u2(clientSock, sizeof(unlink_cmd) + cmd->flen);
::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen); ssize_t result = ::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen);
assert(result==(sizeof(unlink_cmd) + cmd->flen));
u2.run(); u2.run();
// verify response // verify response
@@ -528,8 +587,11 @@ void unlinktask()
bool stattask() bool stattask()
{ {
string filename = "stattest1"; bf::path fullPath = homepath / prefix / "stattest1";
string fullFilename = Config::get()->getValue("ObjectStorage", "metadata_path") + "/" + filename + ".meta"; string filename = fullPath.string();
string Metafilename = prefix + "/stattest1";
string fullFilename = Config::get()->getValue("ObjectStorage", "metadata_path") + "/" + Metafilename + ".meta";
::unlink(fullFilename.c_str()); ::unlink(fullFilename.c_str());
makeTestMetadata(fullFilename.c_str()); makeTestMetadata(fullFilename.c_str());
@@ -541,7 +603,9 @@ bool stattask()
cmd->flen = filename.length(); cmd->flen = filename.length();
strcpy((char *) cmd->filename, filename.c_str()); strcpy((char *) cmd->filename, filename.c_str());
::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen);
assert(result==(sizeof(*cmd) + cmd->flen));
StatTask s(clientSock, sizeof(*cmd) + cmd->flen); StatTask s(clientSock, sizeof(*cmd) + cmd->flen);
s.run(); s.run();
@@ -551,7 +615,7 @@ bool stattask()
assert(err == sizeof(struct stat) + sizeof(sm_response)); assert(err == sizeof(struct stat) + sizeof(sm_response));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->header.payloadLen == sizeof(struct stat) + 4); assert(resp->header.payloadLen == sizeof(struct stat) + sizeof(ssize_t));
assert(resp->returnCode == 0); assert(resp->returnCode == 0);
struct stat *_stat = (struct stat *) resp->payload; struct stat *_stat = (struct stat *) resp->payload;
@@ -605,9 +669,9 @@ bool IOCTruncate()
truncate @ 0 bytes, verify no files are left truncate @ 0 bytes, verify no files are left
*/ */
bf::path metadataFile = metaPath/"test-file.meta"; bf::path metadataFile = metaPath/prefix/"test-file.meta";
bf::path objectPath = cloudPath/testObjKey; bf::path objectPath = cloudPath/testObjKey;
bf::path cachedObjectPath = cachePath/testObjKey; bf::path cachedObjectPath = cachePath/prefix/testObjKey;
makeTestMetadata(metadataFile.string().c_str()); makeTestMetadata(metadataFile.string().c_str());
makeTestObject(objectPath.string().c_str()); makeTestObject(objectPath.string().c_str());
@@ -630,7 +694,7 @@ bool IOCTruncate()
err = ioc->truncate(testFile, 4000); err = ioc->truncate(testFile, 4000);
assert(!err); assert(!err);
MetadataFile meta(testFile); MetadataFile meta(metaTestFile);
assert(meta.getLength() == 4000); assert(meta.getLength() == 4000);
// read the data, make sure there are only 4000 bytes & the object still exists // read the data, make sure there are only 4000 bytes & the object still exists
@@ -645,7 +709,7 @@ bool IOCTruncate()
// truncate to 0 bytes, make sure everything is consistent with that, and the object no longer exists // truncate to 0 bytes, make sure everything is consistent with that, and the object no longer exists
err = ioc->truncate(testFile, 0); err = ioc->truncate(testFile, 0);
assert(!err); assert(!err);
meta = MetadataFile(testFile); meta = MetadataFile(metaTestFile);
assert(meta.getLength() == 0); assert(meta.getLength() == 0);
err = ioc->read(testFile, buf, 0, 1); err = ioc->read(testFile, buf, 0, 1);
assert(err == 0); assert(err == 0);
@@ -656,13 +720,13 @@ bool IOCTruncate()
assert(!bf::exists(objectPath)); assert(!bf::exists(objectPath));
// recreate the meta file, make a 2-object version // recreate the meta file, make a 2-object version
::unlink(metadataFile.string().c_str()); ioc->unlink(testFile);
makeTestMetadata(metadataFile.string().c_str()); makeTestMetadata(metadataFile.string().c_str());
makeTestObject(objectPath.string().c_str()); makeTestObject(objectPath.string().c_str());
meta = MetadataFile(testFile); meta = MetadataFile(metaTestFile);
bf::path secondObjectPath = cloudPath / meta.addMetadataObject(testFile, 8192).key; bf::path secondObjectPath = cloudPath / meta.addMetadataObject(testFile, 8192).key;
bf::path cachedSecondObject = cachePath/secondObjectPath.filename(); bf::path cachedSecondObject = cachePath/prefix/secondObjectPath.filename();
makeTestObject(secondObjectPath.string().c_str()); makeTestObject(secondObjectPath.string().c_str());
meta.writeMetadata(); meta.writeMetadata();
@@ -678,7 +742,7 @@ bool IOCTruncate()
// truncate to 10k, make sure everything looks right // truncate to 10k, make sure everything looks right
err = ioc->truncate(testFile, 10240); err = ioc->truncate(testFile, 10240);
assert(!err); assert(!err);
meta = MetadataFile(testFile); meta = MetadataFile(metaTestFile);
assert(meta.getLength() == 10240); assert(meta.getLength() == 10240);
memset(buf, 0, 16384); memset(buf, 0, 16384);
err = ioc->read(testFile, buf, 0, 10240); err = ioc->read(testFile, buf, 0, 10240);
@@ -689,7 +753,7 @@ bool IOCTruncate()
// truncate to 6000 bytes, make sure second object got deleted // truncate to 6000 bytes, make sure second object got deleted
err = ioc->truncate(testFile, 6000); err = ioc->truncate(testFile, 6000);
meta = MetadataFile(testFile); meta = MetadataFile(metaTestFile);
assert(meta.getLength() == 6000); assert(meta.getLength() == 6000);
err = ioc->read(testFile, buf, 0, 8192); err = ioc->read(testFile, buf, 0, 8192);
assert(err == 6000); assert(err == 6000);
@@ -698,9 +762,9 @@ bool IOCTruncate()
assert(!bf::exists(secondObjectPath)); assert(!bf::exists(secondObjectPath));
assert(!bf::exists(cachedSecondObject)); assert(!bf::exists(cachedSecondObject));
bf::remove(metadataFile);
bf::remove(objectPath);
cache->reset(); cache->reset();
ioc->unlink(testFile);
cout << "IOCTruncate OK" << endl; cout << "IOCTruncate OK" << endl;
return true; return true;
} }
@@ -712,12 +776,15 @@ bool truncatetask()
Cache *cache = Cache::get(); Cache *cache = Cache::get();
bf::path metaPath = ioc->getMetadataPath(); bf::path metaPath = ioc->getMetadataPath();
bf::path fullPath = homepath / prefix / "trunctest1";
const char *filename = "trunctest1"; string metaStr = prefix + "/trunctest1";
const char *filename = fullPath.string().c_str();
const char *Metafilename = metaStr.c_str();
// get the metafile created // get the metafile created
string metaFullName = (metaPath/filename).string() + ".meta"; string metaFullName = (metaPath/Metafilename).string() + ".meta";
::unlink(metaFullName.c_str()); ::unlink(metaFullName.c_str());
MetadataFile meta(filename); MetadataFile meta(Metafilename);
uint8_t buf[1024]; uint8_t buf[1024];
truncate_cmd *cmd = (truncate_cmd *) buf; truncate_cmd *cmd = (truncate_cmd *) buf;
@@ -727,7 +794,9 @@ bool truncatetask()
cmd->flen = strlen(filename); cmd->flen = strlen(filename);
strcpy((char *) cmd->filename, filename); strcpy((char *) cmd->filename, filename);
::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen);
assert(result==(sizeof(*cmd) + cmd->flen));
TruncateTask t(clientSock, sizeof(*cmd) + cmd->flen); TruncateTask t(clientSock, sizeof(*cmd) + cmd->flen);
t.run(); t.run();
@@ -737,11 +806,11 @@ bool truncatetask()
assert(err == sizeof(sm_response)); assert(err == sizeof(sm_response));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->header.payloadLen == 4); assert(resp->header.payloadLen == sizeof(ssize_t));
assert(resp->returnCode == 0); assert(resp->returnCode == 0);
// reload the metadata, check that it is 1000 bytes // reload the metadata, check that it is 1000 bytes
meta = MetadataFile(filename); meta = MetadataFile(Metafilename);
assert(meta.getLength() == 1000); assert(meta.getLength() == 1000);
cache->reset(); cache->reset();
@@ -754,8 +823,12 @@ bool listdirtask()
{ {
IOCoordinator *ioc = IOCoordinator::get(); IOCoordinator *ioc = IOCoordinator::get();
const bf::path metaPath = ioc->getMetadataPath(); const bf::path metaPath = ioc->getMetadataPath();
const char *relPath = "listdirtask"; bf::path fullPath = homepath / prefix / "listdirtask";
bf::path tmpPath = metaPath/relPath; string metaStr = prefix + "/listdirtask";
const char *relPath = fullPath.string().c_str();
const char *MetarelPath = metaStr.c_str();
bf::path tmpPath = metaPath/MetarelPath;
// make some dummy files, make sure they are in the list returned. // make some dummy files, make sure they are in the list returned.
set<string> files; set<string> files;
@@ -780,7 +853,9 @@ bool listdirtask()
cmd->plen = strlen(relPath); cmd->plen = strlen(relPath);
memcpy(cmd->path, relPath, cmd->plen); memcpy(cmd->path, relPath, cmd->plen);
::write(sessionSock, cmd, sizeof(*cmd) + cmd->plen); size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->plen);
assert(result==(sizeof(*cmd) + cmd->plen));
ListDirectoryTask l(clientSock, sizeof(*cmd) + cmd->plen); ListDirectoryTask l(clientSock, sizeof(*cmd) + cmd->plen);
l.run(); l.run();
@@ -818,7 +893,9 @@ void pingtask()
ping_cmd *cmd = (ping_cmd *) buf; ping_cmd *cmd = (ping_cmd *) buf;
cmd->opcode = PING; cmd->opcode = PING;
::write(sessionSock, cmd, sizeof(*cmd)); ssize_t result = ::write(sessionSock, cmd, sizeof(*cmd));
assert(result==(sizeof(*cmd)));
PingTask p(clientSock, sizeof(*cmd)); PingTask p(clientSock, sizeof(*cmd));
p.run(); p.run();
@@ -827,7 +904,7 @@ void pingtask()
sm_response *resp = (sm_response *) buf; sm_response *resp = (sm_response *) buf;
assert(err == sizeof(sm_response)); assert(err == sizeof(sm_response));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4); assert(resp->header.payloadLen == sizeof(ssize_t));
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->returnCode == 0); assert(resp->returnCode == 0);
cout << "pingtask OK" << endl; cout << "pingtask OK" << endl;
@@ -840,9 +917,18 @@ bool copytask()
copy it copy it
verify it exists verify it exists
*/ */
const char *source = "dummy1"; bf::path fullPathSrc = homepath / prefix / "dummy1";
const char *dest = "dummy2"; string metaStrSrc = prefix + "/dummy1";
MetadataFile meta1(source); const char *source = fullPathSrc.string().c_str();
const char *Metasource = metaStrSrc.c_str();
bf::path fullPathDest = homepath / prefix / "dummy2";
string metaStrDest = prefix + "/dummy2";
const char *dest = fullPathDest.string().c_str();
const char *Metadest = metaStrDest.c_str();
MetadataFile meta1(Metasource);
uint8_t buf[1024]; uint8_t buf[1024];
copy_cmd *cmd = (copy_cmd *) buf; copy_cmd *cmd = (copy_cmd *) buf;
@@ -854,7 +940,9 @@ bool copytask()
strncpy(file2->filename, dest, file2->flen); strncpy(file2->filename, dest, file2->flen);
uint len = (uint64_t) &file2->filename[file2->flen] - (uint64_t) buf; uint len = (uint64_t) &file2->filename[file2->flen] - (uint64_t) buf;
::write(sessionSock, buf, len); ssize_t result = ::write(sessionSock, buf, len);
assert(result==len);
CopyTask c(clientSock, len); CopyTask c(clientSock, len);
c.run(); c.run();
@@ -863,12 +951,12 @@ bool copytask()
sm_response *resp = (sm_response *) buf; sm_response *resp = (sm_response *) buf;
assert(err == sizeof(sm_response)); assert(err == sizeof(sm_response));
assert(resp->header.type == SM_MSG_START); assert(resp->header.type == SM_MSG_START);
assert(resp->header.payloadLen == 4); assert(resp->header.payloadLen == sizeof(ssize_t));
assert(resp->header.flags == 0); assert(resp->header.flags == 0);
assert(resp->returnCode == 0); assert(resp->returnCode == 0);
// verify copytest2 is there // verify copytest2 is there
MetadataFile meta2(dest, MetadataFile::no_create_t(),false); MetadataFile meta2(Metadest, MetadataFile::no_create_t(),true);
assert(meta2.exists()); assert(meta2.exists());
bf::path metaPath = IOCoordinator::get()->getMetadataPath(); bf::path metaPath = IOCoordinator::get()->getMetadataPath();
@@ -902,43 +990,44 @@ bool cacheTest1()
assert(cache->getCurrentCacheSize() == 0); assert(cache->getCurrentCacheSize() == 0);
bf::path storagePath = ls->getPrefix(); bf::path storagePath = ls->getPrefix();
bf::path cachePath = cache->getCachePath(); bf::path cachePath = cache->getCachePath() / prefix;
vector<string> v_bogus; vector<string> v_bogus;
vector<bool> exists; vector<bool> exists;
// make sure nothing shows up in the cache path for files that don't exist // make sure nothing shows up in the cache path for files that don't exist
v_bogus.push_back("does-not-exist"); v_bogus.push_back("does-not-exist");
cache->read("", v_bogus); cache->read(prefix, v_bogus);
assert(!bf::exists(cachePath / "does-not-exist")); assert(!bf::exists(cachePath / "does-not-exist"));
cache->exists("", v_bogus, &exists); cache->exists(prefix, v_bogus, &exists);
assert(exists.size() == 1); assert(exists.size() == 1);
assert(!exists[0]); assert(!exists[0]);
// make sure a file that does exist does show up in the cache path // make sure a file that does exist does show up in the cache path
string realFile("storagemanager.cnf");
bf::copy_file(realFile, storagePath / realFile, bf::copy_option::overwrite_if_exists); makeTestObject((storagePath/testObjKey).string().c_str());
v_bogus[0] = realFile;
cache->read("", v_bogus); v_bogus[0] = testObjKey.c_str();
assert(bf::exists(cachePath / realFile)); cache->read(prefix, v_bogus);
assert(bf::exists(cachePath / testObjKey));
exists.clear(); exists.clear();
cache->exists("", v_bogus, &exists); cache->exists(prefix, v_bogus, &exists);
assert(exists.size() == 1); assert(exists.size() == 1);
assert(exists[0]); assert(exists[0]);
size_t currentSize = cache->getCurrentCacheSize(); size_t currentSize = cache->getCurrentCacheSize();
assert(currentSize == bf::file_size(cachePath / realFile)); assert(currentSize == bf::file_size(cachePath / testObjKey));
// lie about the file being deleted and then replaced // lie about the file being deleted and then replaced
cache->deletedObject("", realFile, currentSize); cache->deletedObject(prefix, testObjKey, currentSize);
assert(cache->getCurrentCacheSize() == 0); assert(cache->getCurrentCacheSize() == 0);
cache->newObject("", realFile, currentSize); cache->newObject(prefix, testObjKey, currentSize);
assert(cache->getCurrentCacheSize() == currentSize); assert(cache->getCurrentCacheSize() == currentSize);
cache->exists("", v_bogus, &exists); cache->exists(prefix, v_bogus, &exists);
assert(exists.size() == 1); assert(exists.size() == 1);
assert(exists[0]); assert(exists[0]);
// cleanup // cleanup
bf::remove(cachePath / realFile); bf::remove(cachePath / testObjKey);
bf::remove(storagePath / realFile); bf::remove(storagePath / testObjKey);
cout << "cache test 1 OK" << endl; cout << "cache test 1 OK" << endl;
return true; return true;
} }
@@ -1007,6 +1096,7 @@ bool mergeJournalTest()
bool syncTest1() bool syncTest1()
{ {
IOCoordinator *ioc = IOCoordinator::get();
Config *config = Config::get(); Config *config = Config::get();
Synchronizer *sync = Synchronizer::get(); Synchronizer *sync = Synchronizer::get();
Cache *cache = Cache::get(); Cache *cache = Cache::get();
@@ -1022,6 +1112,7 @@ bool syncTest1()
// delete everything in the fake cloud to make it easier to list later // delete everything in the fake cloud to make it easier to list later
bf::path fakeCloudPath = ls->getPrefix(); bf::path fakeCloudPath = ls->getPrefix();
cout << "fakeCLoudPath = " << fakeCloudPath << endl;
for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir) for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir)
bf::remove(dir->path()); bf::remove(dir->path());
@@ -1035,39 +1126,38 @@ bool syncTest1()
bf::create_directories(metaPath); bf::create_directories(metaPath);
// make the test obj, journal, and metadata // make the test obj, journal, and metadata
string key = "12345_0_8192_test-file"; string journalName = prefix + "/" + testObjKey + ".journal";
string journalName = key + ".journal";
makeTestObject((cachePath/key).string().c_str()); makeTestObject((cachePath/prefix/testObjKey).string().c_str());
makeTestJournal((journalPath/journalName).string().c_str()); makeTestJournal((journalPath/journalName).string().c_str());
makeTestMetadata((metaPath/"test-file.meta").string().c_str()); makeTestMetadata((metaPath/string(metaTestFile + ".meta")).string().c_str());
cache->newObject("", key, bf::file_size(cachePath/key)); cache->newObject(prefix, testObjKey, bf::file_size(cachePath/prefix/testObjKey));
cache->newJournalEntry("", bf::file_size(journalPath/journalName)); cache->newJournalEntry(prefix, bf::file_size(journalPath/journalName));
vector<string> vObj; vector<string> vObj;
vObj.push_back(key); vObj.push_back(testObjKey);
sync->newObjects("", vObj); sync->newObjects(prefix, vObj);
sync->forceFlush(); sync->forceFlush();
sleep(1); // wait for the job to run sleep(2); // wait for the job to run
// make sure that it made it to the cloud // make sure that it made it to the cloud
bool exists = false; bool exists = false;
int err = cs->exists(key, &exists); int err = cs->exists(testObjKey, &exists);
assert(!err); assert(!err);
assert(exists); assert(exists);
sync->newJournalEntry("", key, 0); sync->newJournalEntry(prefix, testObjKey, 0);
sync->forceFlush(); sync->forceFlush();
sleep(1); // let it do what it does sleep(1); // let it do what it does
// check that the original objects no longer exist // check that the original objects no longer exist
assert(!cache->exists("", key)); assert(!cache->exists(prefix, testObjKey));
assert(!bf::exists(journalPath/journalName)); assert(!bf::exists(journalPath/journalName));
// Replicator doesn't implement all of its functionality yet, need to delete key from the cache manually for now // Replicator doesn't implement all of its functionality yet, need to delete key from the cache manually for now
bf::remove(cachePath/key); bf::remove(cachePath/testObjKey);
// check that a new version of object exists in cloud storage // check that a new version of object exists in cloud storage
// D'oh, this would have to list the objects to find it, not going to implement // D'oh, this would have to list the objects to find it, not going to implement
@@ -1077,27 +1167,27 @@ bool syncTest1()
for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator() && !foundIt; ++dir) for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator() && !foundIt; ++dir)
{ {
newKey = dir->path().filename().string(); newKey = dir->path().filename().string();
foundIt = (MetadataFile::getSourceFromKey(newKey) == "test-file"); foundIt = (MetadataFile::getSourceFromKey(newKey) == metaTestFile);
if (foundIt) if (foundIt)
{ {
assert(cache->exists("", newKey)); assert(cache->exists(prefix, newKey));
cs->deleteObject(newKey); cs->deleteObject(newKey);
break; break;
} }
} }
assert(foundIt); assert(foundIt);
cache->makeSpace("", cache->getMaxCacheSize()); // clear the cache & make it call sync->flushObject() cache->makeSpace(prefix, cache->getMaxCacheSize()); // clear the cache & make it call sync->flushObject()
// the key should now be back in cloud storage and deleted from the cache // the key should now be back in cloud storage and deleted from the cache
assert(!cache->exists("", newKey)); assert(!cache->exists(prefix, newKey));
err = cs->exists(newKey, &exists); err = cs->exists(newKey, &exists);
assert(!err && exists); assert(!err && exists);
// make the journal again, call sync->newJournalObject() // make the journal again, call sync->newJournalObject()
makeTestJournal((journalPath / (newKey + ".journal")).string().c_str()); makeTestJournal((journalPath / prefix / (newKey + ".journal")).string().c_str());
cache->newJournalEntry("", bf::file_size(journalPath / (newKey + ".journal"))); cache->newJournalEntry(prefix, bf::file_size(journalPath / prefix / (newKey + ".journal")));
sync->newJournalEntry("", newKey, 0); sync->newJournalEntry(prefix, newKey, 0);
sync->forceFlush(); sync->forceFlush();
sleep(1); sleep(1);
@@ -1107,8 +1197,8 @@ bool syncTest1()
foundIt = false; foundIt = false;
for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator() && !foundIt; ++dir) for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator() && !foundIt; ++dir)
{ {
key = dir->path().filename().string(); testObjKey = dir->path().filename().string();
foundIt = (MetadataFile::getSourceFromKey(key) == "test-file"); foundIt = (MetadataFile::getSourceFromKey(testObjKey) == metaTestFile);
} }
assert(foundIt); assert(foundIt);
@@ -1119,10 +1209,10 @@ bool syncTest1()
vector<string> keys; vector<string> keys;
for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir) for (bf::directory_iterator dir(fakeCloudPath); dir != bf::directory_iterator(); ++dir)
keys.push_back(dir->path().filename().string()); keys.push_back(dir->path().filename().string());
sync->deletedObjects("", keys); sync->deletedObjects(prefix, keys);
sync->forceFlush(); sync->forceFlush();
sleep(1); sleep(1);
::unlink((metaPath/"test-file.meta").string().c_str()); ioc->unlink(testFile);
cout << "Sync test 1 OK" << endl; cout << "Sync test 1 OK" << endl;
return true; return true;
@@ -1247,30 +1337,27 @@ void IOCReadTest1()
cout << "IOC read test 1 requires LocalStorage for now." << endl; cout << "IOC read test 1 requires LocalStorage for now." << endl;
return; return;
} }
testObjKey = "12345_0_8192_" + prefix +"~test-file";
cache->reset();
bf::path storagePath = ls->getPrefix(); bf::path storagePath = ls->getPrefix();
bf::path cachePath = cache->getCachePath(); bf::path cachePath = cache->getCachePath();
bf::path journalPath = cache->getJournalPath(); bf::path journalPath = cache->getJournalPath();
bf::path metaPath = config->getValue("ObjectStorage", "metadata_path"); bf::path metaPath = config->getValue("ObjectStorage", "metadata_path");
assert(!metaPath.empty()); assert(!metaPath.empty());
bf::create_directories(metaPath); bf::create_directories(metaPath/prefix);
string objFilename = (storagePath/testObjKey).string(); string objFilename = (storagePath/testObjKey).string();
string journalFilename = (journalPath/testObjKey).string() + ".journal"; string journalFilename = (journalPath/prefix/testObjKey).string() + ".journal";
string metaFilename = (metaPath/testFile).string() + ".meta"; string metaFilename = (metaPath/metaTestFile).string() + ".meta";
cache->reset();
bf::remove(objFilename);
bf::remove(journalFilename);
bf::remove(metaFilename);
int err;
boost::scoped_array<uint8_t> data(new uint8_t[1<<20]); boost::scoped_array<uint8_t> data(new uint8_t[1<<20]);
memset(data.get(), 0, 1<<20); memset(data.get(), 0, 1<<20);
int err;
err = ioc->read(testFile, data.get(), 0, 1<<20); err = ioc->read(testFile, data.get(), 0, 1<<20);
assert(err < 0); assert(err < 0);
assert(errno == ENOENT); assert(errno == ENOENT);
makeTestObject(objFilename.c_str()); makeTestObject(objFilename.c_str());
makeTestMetadata(metaFilename.c_str()); makeTestMetadata(metaFilename.c_str());
size_t objSize = bf::file_size(objFilename); size_t objSize = bf::file_size(objFilename);
@@ -1299,9 +1386,8 @@ void IOCReadTest1()
assert(data32[i] == 0); assert(data32[i] == 0);
cache->reset(); cache->reset();
bf::remove(objFilename); err = ioc->unlink(testFile);
bf::remove(journalFilename); assert(err>= 0);
bf::remove(metaFilename);
cout << "IOC read test 1 OK" << endl; cout << "IOC read test 1 OK" << endl;
} }
@@ -1326,37 +1412,35 @@ void IOCUnlink()
bf::path metaPath = ioc->getMetadataPath(); bf::path metaPath = ioc->getMetadataPath();
bf::path cachePath = ioc->getCachePath(); bf::path cachePath = ioc->getCachePath();
bf::path journalPath = ioc->getJournalPath(); bf::path journalPath = ioc->getJournalPath();
bf::path cachedObjPath = cachePath/testObjKey; bf::path cachedObjPath = cachePath/prefix/testObjKey;
bf::path cachedJournalPath = journalPath/(string(testObjKey) + ".journal"); bf::path cachedJournalPath = journalPath/prefix/(string(testObjKey) + ".journal");
bf::path basedir = "unlinktest"; bf::path metadataFile = metaPath/(string(metaTestFile) + ".meta");
bf::path metadataFile = metaPath/basedir/(string(testFile) + ".meta");
bf::create_directories(metaPath/basedir);
makeTestMetadata(metadataFile.string().c_str()); makeTestMetadata(metadataFile.string().c_str());
makeTestObject(cachedObjPath.string().c_str()); makeTestObject(cachedObjPath.string().c_str());
makeTestJournal(cachedJournalPath.string().c_str()); makeTestJournal(cachedJournalPath.string().c_str());
cache->newObject("", cachedObjPath.filename().string(), bf::file_size(cachedObjPath)); cache->newObject(prefix, cachedObjPath.filename().string(), bf::file_size(cachedObjPath));
cache->newJournalEntry("", bf::file_size(cachedJournalPath)); cache->newJournalEntry(prefix, bf::file_size(cachedJournalPath));
vector<string> keys; vector<string> keys;
keys.push_back(cachedObjPath.filename().string()); keys.push_back(cachedObjPath.filename().string());
sync->newObjects("", keys); sync->newObjects(prefix, keys);
//sync->newJournalEntry(keys[0]); don't want to end up renaming it //sync->newJournalEntry(keys[0]); don't want to end up renaming it
sync->forceFlush(); sync->forceFlush();
sleep(1); sleep(1);
// ok, they should be fully 'in the system' now. // ok, they should be fully 'in the system' now.
// verify that they are // verify that they are
assert(bf::exists(metaPath/basedir)); assert(bf::exists(metadataFile));
assert(bf::exists(cachedObjPath)); assert(bf::exists(cachedObjPath));
assert(bf::exists(cachedJournalPath)); assert(bf::exists(cachedJournalPath));
bool exists; bool exists;
cs->exists(cachedObjPath.filename().string(), &exists); cs->exists(cachedObjPath.filename().string(), &exists);
assert(exists); assert(exists);
int err = ioc->unlink(basedir.string().c_str()); int err = ioc->unlink(testFile);
assert(err == 0); assert(err == 0);
assert(!bf::exists(metaPath/basedir)); assert(!bf::exists(metadataFile));
assert(!bf::exists(cachedObjPath)); assert(!bf::exists(cachedObjPath));
assert(!bf::exists(cachedJournalPath)); assert(!bf::exists(cachedJournalPath));
sync->forceFlush(); sync->forceFlush();
@@ -1368,6 +1452,7 @@ void IOCUnlink()
cout << "IOC unlink test OK" << endl; cout << "IOC unlink test OK" << endl;
} }
void IOCCopyFile1() void IOCCopyFile1()
{ {
/* /*
@@ -1387,35 +1472,37 @@ void IOCCopyFile1()
cout << "IOCCopyFile1 requires local storage at the moment" << endl; cout << "IOCCopyFile1 requires local storage at the moment" << endl;
return; return;
} }
bf::path metaPath = ioc->getMetadataPath(); bf::path metaPath = ioc->getMetadataPath();
bf::path cachePath = ioc->getCachePath();
bf::path csPath = ls->getPrefix(); bf::path csPath = ls->getPrefix();
bf::path journalPath = ioc->getJournalPath(); bf::path journalPath = ioc->getJournalPath();
bf::path sourcePath = metaPath/"copyfile1"/"source.meta"; bf::path cachedObjPath = cachePath/prefix/copyfile1ObjKey;
bf::path destPath = metaPath/"copyfile2"/"dest.meta"; bf::path sourcePath = metaPath/prefix/"source.meta";
const char *l_sourceFile = "copyfile1/source"; bf::path destPath = metaPath/prefix/"dest.meta";
const char *l_destFile = "copyfile2/dest"; bf::path l_sourceFile = homepath / prefix / string("source");
bf::path l_destFile = homepath / prefix / string("dest");
cache->reset(); cache->reset();
bf::create_directories(sourcePath.parent_path()); bf::create_directories(sourcePath.parent_path());
makeTestMetadata(sourcePath.string().c_str()); makeTestMetadataCopy1(sourcePath.string().c_str());
makeTestObject((csPath/testObjKey).string().c_str()); makeTestObject((csPath/copyfile1ObjKey).string().c_str());
makeTestJournal((journalPath/(string(testObjKey) + ".journal")).string().c_str()); makeTestJournal((journalPath/(string(copyfile1ObjKey) + ".journal")).string().c_str());
cache->newJournalEntry("", bf::file_size(journalPath/(string(testObjKey) + ".journal"))); cache->newJournalEntry(prefix, bf::file_size(journalPath/(string(copyfile1ObjKey) + ".journal")));
int err = ioc->copyFile("copyfile1/source", "copyfile2/dest"); int err = ioc->copyFile(l_sourceFile.string().c_str(), l_destFile.string().c_str());
assert(!err); assert(!err);
uint8_t buf1[8192], buf2[8192]; uint8_t buf1[8192], buf2[8192];
err = ioc->read(l_sourceFile, buf1, 0, 8192); err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, 8192);
assert(err == 8192); assert(err == 8192);
err = ioc->read(l_destFile, buf2, 0, 8192); err = ioc->read(l_destFile.string().c_str(), buf2, 0, 8192);
assert(err == 8192); assert(err == 8192);
assert(memcmp(buf1, buf2, 8192) == 0); assert(memcmp(buf1, buf2, 8192) == 0);
assert(ioc->unlink("copyfile1") == 0); assert(ioc->unlink(l_sourceFile.string().c_str()) == 0);
assert(ioc->unlink("copyfile2") == 0); assert(ioc->unlink(l_destFile.string().c_str()) == 0);
assert(cache->getCurrentCacheSize() == 0);
cout << "IOC copy file 1 OK" << endl; cout << "IOC copy file 1 OK" << endl;
} }
@@ -1424,11 +1511,16 @@ void IOCCopyFile2()
// call IOC::copyFile() with non-existant file // call IOC::copyFile() with non-existant file
IOCoordinator *ioc = IOCoordinator::get(); IOCoordinator *ioc = IOCoordinator::get();
bf::path fullPath = homepath / prefix / "not-there";
const char *source = fullPath.string().c_str();
fullPath = homepath / prefix / "not-there2";
const char *dest = fullPath.string().c_str();
bf::path metaPath = ioc->getMetadataPath(); bf::path metaPath = ioc->getMetadataPath();
bf::remove(metaPath/"not-there.meta"); bf::remove(metaPath/prefix/"not-there.meta");
bf::remove(metaPath/"not-there2.meta"); bf::remove(metaPath/prefix/"not-there2.meta");
int err = ioc->copyFile("not-there", "not-there2"); int err = ioc->copyFile(source, dest);
assert(err); assert(err);
assert(errno == ENOENT); assert(errno == ENOENT);
assert(!bf::exists(metaPath/"not-there.meta")); assert(!bf::exists(metaPath/"not-there.meta"));
@@ -1447,36 +1539,36 @@ void IOCCopyFile3()
*/ */
IOCoordinator *ioc = IOCoordinator::get(); IOCoordinator *ioc = IOCoordinator::get();
Cache *cache = Cache::get(); Cache *cache = Cache::get();
bf::path metaPath = ioc->getMetadataPath(); bf::path metaPath = ioc->getMetadataPath();
bf::path journalPath = ioc->getJournalPath(); bf::path journalPath = ioc->getJournalPath();
bf::path cachePath = ioc->getCachePath(); bf::path cachePath = ioc->getCachePath();
bf::path sourcePath = metaPath/"copyfile3"/"source.meta"; bf::path sourcePath = metaPath/prefix/"source.meta";
bf::path destPath = metaPath/"copyfile4"/"dest.meta"; bf::path destPath = metaPath/prefix/"dest.meta";
const char *l_sourceFile = "copyfile3/source"; bf::path l_sourceFile = homepath / prefix / string("source");
const char *l_destFile = "copyfile4/dest"; bf::path l_destFile = homepath / prefix / string("dest");
cache->reset(); cache->reset();
bf::create_directories(sourcePath.parent_path()); makeTestObject((cachePath/prefix/copyfile3ObjKey).string().c_str());
makeTestMetadata(sourcePath.string().c_str()); makeTestJournal((journalPath/prefix/(string(copyfile3ObjKey) + ".journal")).string().c_str());
makeTestObject((cachePath/testObjKey).string().c_str()); makeTestMetadataCopy3(sourcePath.string().c_str());
makeTestJournal((journalPath/(string(testObjKey) + ".journal")).string().c_str());
cache->newObject("", testObjKey, bf::file_size(cachePath/testObjKey)); cache->newObject(prefix, copyfile3ObjKey, bf::file_size(cachePath/prefix/copyfile3ObjKey));
cache->newJournalEntry("", bf::file_size(journalPath/(string(testObjKey) + ".journal"))); cache->newJournalEntry(prefix, bf::file_size(journalPath/prefix/(string(copyfile3ObjKey) + ".journal")));
int err = ioc->copyFile("copyfile3/source", "copyfile4/dest"); int err = ioc->copyFile(l_sourceFile.string().c_str(), l_destFile.string().c_str());
assert(!err); assert(!err);
uint8_t buf1[8192], buf2[8192]; uint8_t buf1[8192], buf2[8192];
err = ioc->read(l_sourceFile, buf1, 0, 8192); err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, 8192);
assert(err == 8192); assert(err == 8192);
err = ioc->read(l_destFile, buf2, 0, 8192); err = ioc->read(l_destFile.string().c_str(), buf2, 0, 8192);
assert(err == 8192); assert(err == 8192);
assert(memcmp(buf1, buf2, 8192) == 0); assert(memcmp(buf1, buf2, 8192) == 0);
assert(ioc->unlink("copyfile3") == 0); assert(ioc->unlink(l_sourceFile.string().c_str()) == 0);
assert(ioc->unlink("copyfile4") == 0); assert(ioc->unlink(l_destFile.string().c_str()) == 0);
assert(cache->getCurrentCacheSize() == 0);
cout << "IOC copy file 3 OK" << endl; cout << "IOC copy file 3 OK" << endl;
} }
@@ -1524,7 +1616,6 @@ int main()
opentask(); opentask();
//metadataUpdateTest(); //metadataUpdateTest();
// create the metadatafile to use // create the metadatafile to use
MetadataFile tmpfile("metadataJournalTest");
// requires 8K object size to test boundries // requires 8K object size to test boundries
//Case 1 new write that spans full object //Case 1 new write that spans full object
metadataJournalTest((10*sizeKB),0); metadataJournalTest((10*sizeKB),0);