Add more tests for LSM log file recovery. Fix a problem in recovering log

files that contain range deletes.

FossilOrigin-Name: e34eafd4c5b2bbf2735e136ad69b67bb4288ad4d01a0128d8e107ac46209a182

ext/lsm1/lsm-test/lsmtest1.c

@@ -81,12 +81,12 @@ struct Datatest2 {
 ** Generate a unique name for the test case pTest with database system
 ** zSystem.
 */
-static char *getName(const char *zSystem, Datatest1 *pTest){
+static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){
   char *zRet;
   char *zData;
   zData = testDatasourceName(&pTest->defn);
-  zRet = testMallocPrintf("data.%s.%s.%d.%d", 
+  zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", 
-      zSystem, zData, pTest->nRow, pTest->nVerify
+      zSystem, zData, bRecover, pTest->nRow, pTest->nVerify
   );
   testFree(zData);
   return zRet;
@@ -251,8 +251,21 @@ static void printScanCb(
 }
 #endif
 
+void testReopenRecover(TestDb **ppDb, int *pRc){
+  if( *pRc==0 ){
+    const char *zLib = tdb_library_name(*ppDb);
+    const char *zDflt = tdb_default_db(zLib);
+    testCopyLsmdb(zDflt, "bak.db");
+    testClose(ppDb);
+    testCopyLsmdb("bak.db", zDflt);
+    *pRc = tdb_open(zLib, 0, 0, ppDb);
+  }
+}
+
+
 static void doDataTest1(
   const char *zSystem,            /* Database system to test */
+  int bRecover,
   Datatest1 *p,                   /* Structure containing test parameters */
   int *pRc                        /* OUT: Error code */
 ){
@@ -277,8 +290,11 @@ static void doDataTest1(
     /* Check that the db content is correct. */
     testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
 
-    /* Close and reopen the database. */
+    if( bRecover ){
-    testReopen(&pDb, &rc);
+      testReopenRecover(&pDb, &rc);
+    }else{
+      testReopen(&pDb, &rc);
+    }
 
     /* Check that the db content is still correct. */
     testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
@@ -299,7 +315,11 @@ static void doDataTest1(
     testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
 
     /* Close and reopen the database. */
-    testReopen(&pDb, &rc);
+    if( bRecover ){
+      testReopenRecover(&pDb, &rc);
+    }else{
+      testReopen(&pDb, &rc);
+    }
 
     /* Check that the db content is still correct. */
     testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
@@ -339,13 +359,17 @@ void test_data_1(
   };
 
   int i;
+  int bRecover;
 
-  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
+  for(bRecover=0; bRecover<2; bRecover++){
-    char *zName = getName(zSystem, &aTest[i]);
+    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
-    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
+    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
-      doDataTest1(zSystem, &aTest[i], pRc);
+      char *zName = getName(zSystem, bRecover, &aTest[i]);
+      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
+        doDataTest1(zSystem, bRecover, &aTest[i], pRc);
+      }
+      testFree(zName);
     }
-    testFree(zName);
   }
 }
 
@@ -396,6 +420,7 @@ void testCompareDb(
 
 static void doDataTest2(
   const char *zSystem,            /* Database system to test */
+  int bRecover,
   Datatest2 *p,                   /* Structure containing test parameters */
   int *pRc                        /* OUT: Error code */
 ){
@@ -437,7 +462,11 @@ static void doDataTest2(
     testFree(pKey1);
 
     testCompareDb(pData, nRange, i, pControl, pDb, &rc);
-    testReopen(&pDb, &rc);
+    if( bRecover ){
+      testReopenRecover(&pDb, &rc);
+    }else{
+      testReopen(&pDb, &rc);
+    }
     testCompareDb(pData, nRange, i, pControl, pDb, &rc);
 
     /* Update the progress dots... */
@@ -451,12 +480,12 @@ static void doDataTest2(
   *pRc = rc;
 }
 
-static char *getName2(const char *zSystem, Datatest2 *pTest){
+static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){
   char *zRet;
   char *zData;
   zData = testDatasourceName(&pTest->defn);
-  zRet = testMallocPrintf("data2.%s.%s.%d.%d.%d", 
+  zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", 
-      zSystem, zData, pTest->nRange, pTest->nWrite, pTest->nIter
+      zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter
   );
   testFree(zData);
   return zRet;
@@ -476,13 +505,17 @@ void test_data_2(
   };
 
   int i;
+  int bRecover;
 
-  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
+  for(bRecover=0; bRecover<2; bRecover++){
-    char *zName = getName2(zSystem, &aTest[i]);
+    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
-    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
+    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
-      doDataTest2(zSystem, &aTest[i], pRc);
+      char *zName = getName2(zSystem, bRecover, &aTest[i]);
+      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
+        doDataTest2(zSystem, bRecover, &aTest[i], pRc);
+      }
+      testFree(zName);
     }
-    testFree(zName);
   }
 }
 

ext/lsm1/lsm-test/lsmtest_main.c

@@ -26,7 +26,7 @@ void test_failed(){
 static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){
   if( rc ){
     *pRc = rc;
-    printf("FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
+    fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
     test_failed();
   }
 }

ext/lsm1/lsm-test/lsmtest_tdb.c

@@ -745,6 +745,14 @@ const char *tdb_system_name(int i){
   return aLib[i].zName;
 }
 
+const char *tdb_default_db(const char *zSys){
+  int i;
+  for(i=0; i<ArraySize(aLib); i++){
+    if( strcmp(aLib[i].zName, zSys)==0 ) return aLib[i].zDefaultDb;
+  }
+  return 0;
+}
+
 int tdb_open(const char *zLib, const char *zDb, int bClear, TestDb **ppDb){
   int i;
   int rc = 1;

ext/lsm1/lsm-test/lsmtest_tdb.h

@@ -125,6 +125,7 @@ int tdb_scan(
 );
 
 const char *tdb_system_name(int i);
+const char *tdb_default_db(const char *zSys);
 
 int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb);
 

ext/lsm1/lsmInt.h

@@ -864,7 +864,7 @@ int lsmInfoFreelist(lsm_db *pDb, char **pzOut);
 ** Functions from file "lsm_log.c".
 */
 int lsmLogBegin(lsm_db *pDb);
-int lsmLogWrite(lsm_db *, void *, int, void *, int);
+int lsmLogWrite(lsm_db *, int, void *, int, void *, int);
 int lsmLogCommit(lsm_db *);
 void lsmLogEnd(lsm_db *pDb, int bCommit);
 void lsmLogTell(lsm_db *, LogMark *);
@@ -874,6 +874,10 @@ void lsmLogClose(lsm_db *);
 int lsmLogRecover(lsm_db *);
 int lsmInfoLogStructure(lsm_db *pDb, char **pzVal);
 
+/* Valid values for the second argument to lsmLogWrite(). */
+#define LSM_WRITE        0x06
+#define LSM_DELETE       0x08
+#define LSM_DRANGE       0x0A
 
 /**************************************************************************
 ** Functions from file "lsm_shared.c".

ext/lsm1/lsm_log.c

@@ -199,9 +199,13 @@
 
 #define LSM_LOG_WRITE        0x06
 #define LSM_LOG_WRITE_CKSUM  0x07
+
 #define LSM_LOG_DELETE       0x08
 #define LSM_LOG_DELETE_CKSUM 0x09
 
+#define LSM_LOG_DRANGE       0x0A
+#define LSM_LOG_DRANGE_CKSUM 0x0B
+
 /* Require a checksum every 32KB. */
 #define LSM_CKSUM_MAXDATA (32*1024)
 
@@ -653,6 +657,7 @@ static int logFlush(lsm_db *pDb, int eType){
 */
 int lsmLogWrite(
   lsm_db *pDb,                    /* Database handle */
+  int eType,
   void *pKey, int nKey,           /* Database key to write to log */
   void *pVal, int nVal            /* Database value (or nVal<0) to write */
 ){
@@ -661,13 +666,19 @@ int lsmLogWrite(
   int nReq;                       /* Bytes of space required in log */
   int bCksum = 0;                 /* True to embed a checksum in this record */
 
+  assert( eType==LSM_WRITE || eType==LSM_DELETE || eType==LSM_DRANGE );
+  assert( LSM_LOG_WRITE==LSM_WRITE );
+  assert( LSM_LOG_DELETE==LSM_DELETE );
+  assert( LSM_LOG_DRANGE==LSM_DRANGE );
+  assert( (eType==LSM_LOG_DELETE)==(nVal<0) );
+
   if( pDb->bUseLog==0 ) return LSM_OK;
   pLog = pDb->pLogWriter;
 
   /* Determine how many bytes of space are required, assuming that a checksum
   ** will be embedded in this record (even though it may not be).  */
   nReq = 1 + lsmVarintLen32(nKey) + 8 + nKey;
-  if( nVal>=0 ) nReq += lsmVarintLen32(nVal) + nVal;
+  if( eType!=LSM_LOG_DELETE ) nReq += lsmVarintLen32(nVal) + nVal;
 
   /* Jump over the jump region if required. Set bCksum to true to tell the
   ** code below to include a checksum in the record if either (a) writing
@@ -687,9 +698,10 @@ int lsmLogWrite(
     ** DELETE) or 2 (for WRITE) varints.  */
     assert( LSM_LOG_WRITE_CKSUM == (LSM_LOG_WRITE | 0x0001) );
     assert( LSM_LOG_DELETE_CKSUM == (LSM_LOG_DELETE | 0x0001) );
-    *(a++) = (nVal>=0 ? LSM_LOG_WRITE : LSM_LOG_DELETE) | (u8)bCksum;
+    assert( LSM_LOG_DRANGE_CKSUM == (LSM_LOG_DRANGE | 0x0001) );
+    *(a++) = (u8)eType | (u8)bCksum;
     a += lsmVarintPut32(a, nKey);
-    if( nVal>=0 ) a += lsmVarintPut32(a, nVal);
+    if( eType!=LSM_LOG_DELETE ) a += lsmVarintPut32(a, nVal);
 
     if( bCksum ){
       pLog->buf.n = (a - (u8 *)pLog->buf.z);
@@ -699,7 +711,7 @@ int lsmLogWrite(
 
     memcpy(a, pKey, nKey);
     a += nKey;
-    if( nVal>=0 ){
+    if( eType!=LSM_LOG_DELETE ){
       memcpy(a, pVal, nVal);
       a += nVal;
     }
@@ -1005,6 +1017,8 @@ int lsmLogRecover(lsm_db *pDb){
             break;
           }
 
+          case LSM_LOG_DRANGE:
+          case LSM_LOG_DRANGE_CKSUM:
           case LSM_LOG_WRITE:
           case LSM_LOG_WRITE_CKSUM: {
             int nKey;
@@ -1013,7 +1027,7 @@ int lsmLogRecover(lsm_db *pDb){
             logReaderVarint(&reader, &buf1, &nKey, &rc);
             logReaderVarint(&reader, &buf2, &nVal, &rc);
 
-            if( eType==LSM_LOG_WRITE_CKSUM ){
+            if( eType==LSM_LOG_WRITE_CKSUM || eType==LSM_LOG_DRANGE_CKSUM ){
               logReaderCksum(&reader, &buf1, &bEof, &rc);
             }else{
               bEof = logRequireCksum(&reader, nKey+nVal);
@@ -1023,7 +1037,11 @@ int lsmLogRecover(lsm_db *pDb){
             logReaderBlob(&reader, &buf1, nKey, 0, &rc);
             logReaderBlob(&reader, &buf2, nVal, &aVal, &rc);
             if( iPass==1 && rc==LSM_OK ){ 
-              rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
+              if( eType==LSM_LOG_WRITE || eType==LSM_LOG_WRITE_CKSUM ){
+                rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
+              }else{
+                rc = lsmTreeDelete(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
+              }
             }
             break;
           }

ext/lsm1/lsm_main.c

@@ -669,11 +669,8 @@ static int doWriteOp(
   }
 
   if( rc==LSM_OK ){
-    if( bDeleteRange==0 ){
+    int eType = (bDeleteRange ? LSM_DRANGE : (nVal>=0?LSM_WRITE:LSM_DELETE));
-      rc = lsmLogWrite(pDb, (void *)pKey, nKey, (void *)pVal, nVal);
+    rc = lsmLogWrite(pDb, eType, (void *)pKey, nKey, (void *)pVal, nVal);
-    }else{
-      /* TODO */
-    }
   }
 
   lsmSortedSaveTreeCursors(pDb);

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\smemory\smanagement\sproblem\sin\slsm\slog\srecovery\scode.
+C Add\smore\stests\sfor\sLSM\slog\sfile\srecovery.\sFix\sa\sproblem\sin\srecovering\slog\nfiles\sthat\scontain\srange\sdeletes.
-D 2017-07-01T20:59:06.315
+D 2017-07-03T09:00:18.115
 F Makefile.in 081e48dfe7f995d57ce1a88ddf4d2917b4349158648a6cd45b42beae30de3a12
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 4ebb1d257cac7fb1bcb4ba59278416d410ff1c4bf59447a9c37a415f3516056a
@@ -213,7 +213,7 @@ F ext/lsm1/Makefile a2ea4975162be8932b5efa727080f4982715d34c32035d9eb7a015ae7840
 F ext/lsm1/Makefile.msc c8140509750fb207f0e984991bc426e91a11adb2f6434f218c52103b9d33a8e1
 F ext/lsm1/lsm-test/README 87ea529d2abe615e856d4714bfe8bb185e6c2771b8612aa6298588b7b43e6f86
 F ext/lsm1/lsm-test/lsmtest.h ba9c3cdd4dee660f14de8fe0a5b8c36d72bb5524730f0ce7641aa08f0ef56da5
-F ext/lsm1/lsm-test/lsmtest1.c 27c3cf6512514b25a145154ae4e54d053d883b2f7f52ed214747b5ebaceedd3e
+F ext/lsm1/lsm-test/lsmtest1.c 2570b0e21e854d3d079dc04a535ce58abe50ed512da23117ab95b01fecdb90a0
 F ext/lsm1/lsm-test/lsmtest2.c 385efd49f3411bb2c8e0f9ffbd4f87450d5ac368aeee62d509d2c0ce10e4215b
 F ext/lsm1/lsm-test/lsmtest3.c 9ab87528a36dbf4a61d7c8ad954f5ee368c0878c127b84b942b2e2abe522de26
 F ext/lsm1/lsm-test/lsmtest4.c d258d6a245db5d8eaede096e2368d23f859c5e92c80ab9122463f708514fe10c
@@ -226,21 +226,21 @@ F ext/lsm1/lsm-test/lsmtest_bt.c d70d9a9be5eef9360af1251dd083948d74fd30137a08f61
 F ext/lsm1/lsm-test/lsmtest_datasource.c 5d770be191d0ca51315926723009b2c25c0b4b8136840494ef710ac324aa916c
 F ext/lsm1/lsm-test/lsmtest_func.c 159aa401bc8032bfa3d8cf2977bd687abebab880255895a5eb45770d626fa38d
 F ext/lsm1/lsm-test/lsmtest_io.c ba678e052f497b4c2ff41d47672a1f2a02f495d14083845ee66e0bda678f5b80
-F ext/lsm1/lsm-test/lsmtest_main.c dd062fe16d778aaaade3147199749ba9d8743e1408d249823ee9ecf3c24b5466
+F ext/lsm1/lsm-test/lsmtest_main.c ea24e68f0295753f42723ce0fa1606993b2109f2b21feb24a0696b42160708f5
 F ext/lsm1/lsm-test/lsmtest_mem.c 996b1e76cc876e8d765182a2f14159b1acbf56cbf86d286173e13e970b79a945
-F ext/lsm1/lsm-test/lsmtest_tdb.c d85694a9e5cd3908cb3e10bb5cb5d6a47ad101adb9fb477ca61f1e3725253b06
+F ext/lsm1/lsm-test/lsmtest_tdb.c e46c3f44880d45fd444daf2b2a53bca7a64f9df3f6480cbb1b47a039cc95626a
-F ext/lsm1/lsm-test/lsmtest_tdb.h de1ee8c71a7ef61d964e40e057cffea387d7b58a51d95905ab909937d24e4a91
+F ext/lsm1/lsm-test/lsmtest_tdb.h 85d6a5f30f6264e1d0b2062f23d03adcf2d04460ddbfe40d35e7270be9b50a30
 F ext/lsm1/lsm-test/lsmtest_tdb2.cc 99ea7f2dd9c7536c8fb9bdd329e4cfeb76899f3ddf6f48bdd3926e016922b715
 F ext/lsm1/lsm-test/lsmtest_tdb3.c 105ef644ad03c8d7f21f8d1f6514adaf30a7c26f4e21cccb3874b70a14985041
 F ext/lsm1/lsm-test/lsmtest_tdb4.c 47e8bb5eba266472d690fb8264f1855ebdba0ae5a0e541e35fcda61ebf1d277f
 F ext/lsm1/lsm-test/lsmtest_util.c 241622db5a332a09c8e6e7606b617d288a37b557f7d3bce0bb97809f67cc2806
 F ext/lsm1/lsm-test/lsmtest_win32.c 5605aac3bf3852dcc2509fb1d097f5f11556418c1cc9cf0fdd09f9af53c97fb4
 F ext/lsm1/lsm.h 0f6f64ff071471cb87bf98beb8386566f30ea001
-F ext/lsm1/lsmInt.h b5d6d073aa5d233614cf4d97d81ba313e9f2c50cfaf12952f7efd9cd945bcd04
+F ext/lsm1/lsmInt.h 68945f00c4fc97a5c82bd285a15d0baacd0019cf2e0b7d535759f000459462e1
 F ext/lsm1/lsm_ckpt.c 239a8442693f1de6a06c9136a235757ab63f5a772ff44e1e81f00d71cc96151e
 F ext/lsm1/lsm_file.c fc95548f8cf8e47fda4120215d2c1fc22dfa85e09294e5656a99a846dc80a62b
-F ext/lsm1/lsm_log.c 82fd21043868b8982c58c3229dc420cc4ad67a77e7134f427805132c666a33e3
+F ext/lsm1/lsm_log.c c5b69f99a80d352357b1102e5c4c1c855fd5f403e30427b0f62b887e050d225a
-F ext/lsm1/lsm_main.c f52eada2910f8a57bd4cafcee39c6c375f6b7ed8
+F ext/lsm1/lsm_main.c 8f1f68a87126dcecaed463ed38e4b1c0dcc184614de1d7b01542d88c06551d9e
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
 F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea
 F ext/lsm1/lsm_shared.c 54cc3a5157c6abd77f7d3ae60708b9f7bf022b3c
@@ -1628,7 +1628,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 66de6f4a9504ec2670b7273de8fb6955c80e03f7f73414ea6b80fd8a99f75976
+P dd55af30b4440bba1b97c2155b9f499d791cd7d8bae8341972c41439fdc41edc
-R 08eea8839166fcca2c06eb8d028d9968
+R 28a4b3f47ac9a6ed16a81e9fcc9e6dc7
 U dan
-Z d5e677a9a0d306f6127c3d95383e56f5
+Z 748ed88e39ac7ec5284cc27ded18a498

manifest.uuid

@@ -1 +1 @@
-dd55af30b4440bba1b97c2155b9f499d791cd7d8bae8341972c41439fdc41edc
+e34eafd4c5b2bbf2735e136ad69b67bb4288ad4d01a0128d8e107ac46209a182