diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e7b0b724..472de7e3b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,8 +62,13 @@ add_dependencies(StorageManager ms3) target_link_libraries(StorageManager boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) set_property(TARGET StorageManager PROPERTY CXX_STANDARD 11) +# There's probably a better way to do this, but this works, so moving on +# for now. +add_custom_target(test_files + mkdir -p ${CMAKE_BINARY_DIR}/test_data && cp -R ${CMAKE_SOURCE_DIR}/test_data/* ${CMAKE_BINARY_DIR}/test_data +) add_executable(unit_tests src/unit_tests.cpp ${storagemanager_SRCS}) -add_dependencies(unit_tests ms3) +add_dependencies(unit_tests ms3 test_files) target_link_libraries(unit_tests boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) set_property(TARGET unit_tests PROPERTY CXX_STANDARD 11) diff --git a/src/IOCoordinator.cpp b/src/IOCoordinator.cpp index 1372ed08c..5a15745a9 100755 --- a/src/IOCoordinator.cpp +++ b/src/IOCoordinator.cpp @@ -946,6 +946,7 @@ boost::shared_array IOCoordinator::mergeJournal(const char *object, con if (err != 16) // got EOF break; + //cout << "MJ: got offset " << offlen[0] << " length " << offlen[1] << endl; // if this entry overlaps, read the overlapping section uint64_t lastJournalOffset = offlen[0] + offlen[1]; uint64_t lastBufOffset = offset + len; @@ -954,6 +955,9 @@ boost::shared_array IOCoordinator::mergeJournal(const char *object, con uint64_t startReadingAt = max(offlen[0], (uint64_t) offset); uint64_t lengthOfRead = min(lastBufOffset, lastJournalOffset) - startReadingAt; + //cout << "MJ: startReadingAt = " << startReadingAt << " lengthOfRead = " << lengthOfRead << endl; + + // seek to the portion of the entry to start reading at if (startReadingAt != offlen[0]) ::lseek(journalFD, startReadingAt - offlen[0], SEEK_CUR); @@ -970,15 +974,17 @@ boost::shared_array IOCoordinator::mergeJournal(const char *object, con } else if (err == 0) { - logger->log(LOG_ERR, "mergeJournal: got early EOF"); + logger->log(LOG_ERR, "mergeJournal: got early EOF. offset=%ld, len=%ld, jOffset=%ld, jLen=%ld," + " startReadingAt=%ld, lengthOfRead=%ld", offset, len, offlen[0], offlen[1], startReadingAt, lengthOfRead); ret.reset(); return ret; } count += err; } - if (lengthOfRead != offlen[1]) - ::lseek(journalFD, offlen[1] - lengthOfRead, SEEK_CUR); + // advance the file pos if we didn't read to the end of the entry + if (startReadingAt - offlen[0] + lengthOfRead != offlen[1]) + ::lseek(journalFD, offlen[1] - (lengthOfRead + startReadingAt - offlen[0]), SEEK_CUR); } else // skip over this journal entry diff --git a/src/unit_tests.cpp b/src/unit_tests.cpp index d0809d394..846e3e79f 100755 --- a/src/unit_tests.cpp +++ b/src/unit_tests.cpp @@ -1452,9 +1452,33 @@ void IOCCopyFile() IOCCopyFile3(); } +/* Correctness was tested by inspecting debugging outputs in mergeJournal(). With a little more work + we could capture the merged output and use that to confirm the expected result. Later. +*/ +void bigMergeJournal1() +{ + const char *jName = "test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~" + "mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em.journal"; + const char *fName = "test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~" + "mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em"; + + IOCoordinator *ioc = IOCoordinator::get(); + boost::shared_array buf; + buf = ioc->mergeJournal(fName, jName, 0, 68332); + assert(buf); + buf = ioc->mergeJournal(fName, jName, 100, 68232); + assert(buf); + buf = ioc->mergeJournal(fName, jName, 0, 68232); + assert(buf); + buf = ioc->mergeJournal(fName, jName, 100, 68132); + assert(buf); + buf = ioc->mergeJournal(fName, jName, 100, 10); + assert(buf); +} + + int main() { - std::size_t sizeKB = 1024; cout << "connecting" << endl; makeConnection(); @@ -1505,7 +1529,12 @@ int main() IOCTruncate(); IOCUnlink(); IOCCopyFile(); - + + // For the moment, this next one just verifies no error happens as reported by the fcns called. + // It doesn't verify the result yet. + bigMergeJournal1(); + + sleep(5); // sometimes this deletes them before syncwithjournal is called metadataJournalTestCleanup(17*sizeKB); diff --git a/test_data/BRM_saves_em.meta b/test_data/BRM_saves_em.meta new file mode 100644 index 000000000..61f21ebc2 --- /dev/null +++ b/test_data/BRM_saves_em.meta @@ -0,0 +1,11 @@ +{ + "version": "1", + "revision": "1", + "objects": [ + { + "offset": "0", + "length": "68332", + "key": "e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em" + } + ] +} diff --git a/test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em b/test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em new file mode 100644 index 000000000..57515c220 Binary files /dev/null and b/test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em differ diff --git a/test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em.journal b/test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em.journal new file mode 100644 index 000000000..eed7b28e6 Binary files /dev/null and b/test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em.journal differ