Debugged mergeJournal() using some data that failed to merge in a

'real' run.  Added the files that failed to merge, & added to the unit test.

CMakeLists.txt

@@ -62,8 +62,13 @@ add_dependencies(StorageManager ms3)
 target_link_libraries(StorageManager boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS})
 set_property(TARGET StorageManager PROPERTY CXX_STANDARD 11)
 
+# There's probably a better way to do this, but this works, so moving on
+# for now.
+add_custom_target(test_files
+    mkdir -p ${CMAKE_BINARY_DIR}/test_data && cp -R ${CMAKE_SOURCE_DIR}/test_data/* ${CMAKE_BINARY_DIR}/test_data
+)
 add_executable(unit_tests src/unit_tests.cpp ${storagemanager_SRCS})
-add_dependencies(unit_tests ms3)
+add_dependencies(unit_tests ms3 test_files)
 target_link_libraries(unit_tests boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS})
 set_property(TARGET unit_tests PROPERTY CXX_STANDARD 11)
 

src/IOCoordinator.cpp

@@ -946,6 +946,7 @@ boost::shared_array<uint8_t> IOCoordinator::mergeJournal(const char *object, con
         if (err != 16)   // got EOF
             break;
         
+        //cout << "MJ: got offset " << offlen[0] << " length " << offlen[1] << endl;
         // if this entry overlaps, read the overlapping section
         uint64_t lastJournalOffset = offlen[0] + offlen[1];
         uint64_t lastBufOffset = offset + len;
@@ -954,6 +955,9 @@ boost::shared_array<uint8_t> IOCoordinator::mergeJournal(const char *object, con
             uint64_t startReadingAt = max(offlen[0], (uint64_t) offset);
             uint64_t lengthOfRead = min(lastBufOffset, lastJournalOffset) - startReadingAt;
             
+            //cout << "MJ: startReadingAt = " << startReadingAt << " lengthOfRead = " << lengthOfRead << endl;
+            
+            // seek to the portion of the entry to start reading at
             if (startReadingAt != offlen[0])
                 ::lseek(journalFD, startReadingAt - offlen[0], SEEK_CUR);
                 
@@ -970,15 +974,17 @@ boost::shared_array<uint8_t> IOCoordinator::mergeJournal(const char *object, con
                 }
                 else if (err == 0)
                 {
-                    logger->log(LOG_ERR, "mergeJournal: got early EOF");
+                    logger->log(LOG_ERR, "mergeJournal: got early EOF. offset=%ld, len=%ld, jOffset=%ld, jLen=%ld,"
+                        " startReadingAt=%ld, lengthOfRead=%ld", offset, len, offlen[0], offlen[1], startReadingAt, lengthOfRead);
                     ret.reset();
                     return ret;
                 }
                 count += err;
             }
             
-            if (lengthOfRead != offlen[1])
+            // advance the file pos if we didn't read to the end of the entry
-                ::lseek(journalFD, offlen[1] - lengthOfRead, SEEK_CUR);
+            if (startReadingAt - offlen[0] + lengthOfRead != offlen[1])
+                ::lseek(journalFD, offlen[1] - (lengthOfRead + startReadingAt - offlen[0]), SEEK_CUR);
         }
         else
             // skip over this journal entry

src/unit_tests.cpp

@@ -1452,9 +1452,33 @@ void IOCCopyFile()
     IOCCopyFile3();
 }
 
+/* Correctness was tested by inspecting debugging outputs in mergeJournal().  With a little more work
+   we could capture the merged output and use that to confirm the expected result.  Later.
+*/
+void bigMergeJournal1()
+{
+    const char *jName = "test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~"
+        "mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em.journal";
+    const char *fName = "test_data/e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~"
+        "mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em";
+        
+    IOCoordinator *ioc = IOCoordinator::get();
+    boost::shared_array<uint8_t> buf;
+    buf = ioc->mergeJournal(fName, jName, 0, 68332);
+    assert(buf);
+    buf = ioc->mergeJournal(fName, jName, 100, 68232);
+    assert(buf);
+    buf = ioc->mergeJournal(fName, jName, 0, 68232);
+    assert(buf);
+    buf = ioc->mergeJournal(fName, jName, 100, 68132);
+    assert(buf);
+    buf = ioc->mergeJournal(fName, jName, 100, 10);
+    assert(buf);
+}
+    
+
 int main()
 {
-
     std::size_t sizeKB = 1024;
     cout << "connecting" << endl;
     makeConnection();
@@ -1505,7 +1529,12 @@ int main()
     IOCTruncate();
     IOCUnlink();
     IOCCopyFile();
-
+    
+    // For the moment, this next one just verifies no error happens as reported by the fcns called.
+    // It doesn't verify the result yet.
+    bigMergeJournal1();
+    
+    
     sleep(5); // sometimes this deletes them before syncwithjournal is called
     metadataJournalTestCleanup(17*sizeKB);
 

test_data/BRM_saves_em.meta

@@ -0,0 +1,11 @@
+{
+    "version": "1",
+    "revision": "1",
+    "objects": [
+        {
+            "offset": "0",
+            "length": "68332",
+            "key": "e7a81ca3-0af8-48cc-b224-0f59c187e0c1_0_3436_~home~patrick~mariadb~columnstore~data1~systemFiles~dbrm~BRM_saves_em"
+        }
+    ]
+}