Add sqlite3recover() support to fuzzcheck.

FossilOrigin-Name: e65c5bdc5bd6e1989c3de59798ff0a761125fc490fdec8c7671ca70ed688c4f8

Makefile.in

@@ -626,7 +626,10 @@ SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
 FUZZERSHELL_OPT = 
-FUZZCHECK_OPT = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
+FUZZCHECK_OPT += -I$(TOP)/test
+FUZZCHECK_OPT += -I$(TOP)/ext/recover
+FUZZCHECK_OPT += -DSQLITE_OMIT_LOAD_EXTENSION
+FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4
@@ -636,7 +639,12 @@ FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE
 FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY
 FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
-FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c $(TOP)/test/fuzzinvariants.c
+FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c
+FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c
+FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c
+FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c
+FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c
+FUZZCHECK_SRC += $(TOP)/test/vt02.c
 DBFUZZ_OPT =
 ST_OPT = -DSQLITE_OS_KV_OPTIONAL
 
@@ -687,7 +695,7 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
 	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
 
-fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h
+fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
 
 ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h

Makefile.msc

@@ -1701,15 +1701,25 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
 #
 MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_COMPILE_OPTS =
-FUZZCHECK_OPTS = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
+FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover
+FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5
+FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ
+FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
+FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
+FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzcheck.c
+FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\ossfuzz.c
+FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c
+FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c
+FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c
+FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c
 
-FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c $(TOP)\test\fuzzinvariants.c
 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
 DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
 KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ

main.mk

@@ -544,7 +544,9 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
-FUZZCHECK_OPT = -DSQLITE_ENABLE_MEMSYS5
+FUZZCHECK_OPT += -I$(TOP)/test
+FUZZCHECK_OPT += -I$(TOP)/ext/recover
+FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4
@@ -554,7 +556,10 @@ FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 FUZZSRC += $(TOP)/test/fuzzcheck.c
 FUZZSRC += $(TOP)/test/ossfuzz.c
+FUZZSRC += $(TOP)/test/vt02.c
 FUZZSRC += $(TOP)/test/fuzzinvariants.c
+FUZZSRC += $(TOP)/ext/recover/dbdata.c
+FUZZSRC += $(TOP)/ext/recover/sqlite3recover.c
 DBFUZZ_OPT =
 KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
 ST_OPT = -DSQLITE_THREADSAFE=0
@@ -613,7 +618,7 @@ dbfuzz2$(EXE):	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
 	$(TCCX) -I. -g -O0 -DSTANDALONE -o dbfuzz2$(EXE) \
 	  $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c  $(TLIBS) $(THREADLIB)
 
-fuzzcheck$(EXE):	$(FUZZSRC) sqlite3.c sqlite3.h
+fuzzcheck$(EXE):	$(FUZZSRC) sqlite3.c sqlite3.h $(FUZZDEP)
 	$(TCCX) -o fuzzcheck$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_MEMSYS5 $(FUZZCHECK_OPT) -DSQLITE_OSS_FUZZ \
 		$(FUZZSRC) sqlite3.c $(TLIBS) $(THREADLIB)

manifest

@@ -1,11 +1,11 @@
-C Remove\ssqlite3_interrupt()\sfrom\sthe\sWASM\sbuild,\sas\sit\sis\sessentially\simpossible\sto\semploy\sin\sJS's\sthreading\smodel.
+C Add\ssqlite3recover()\ssupport\sto\sfuzzcheck.
-D 2022-10-28T14:33:50.731
+D 2022-10-28T18:35:06.668
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
-F Makefile.in 579150683deaff2d8e70559a3a117a82a740b2491d4ecb56debb6a6f7c1705cd
+F Makefile.in 78e4c4916f2c3993a8a454018745ff02094a8029d449d0c22db98f1cf8260420
 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241
-F Makefile.msc ea790f1db3c02b62cb13416a680fa909b54bc1e0533cca222db0dae3f7af3db9
+F Makefile.msc e7a564ceec71f0d9666031d5638cf0d4f88b050b44e8df5d32125137cd259ac0
 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e
 F VERSION 8868ddfa6e1eee218286021a94b3e22d13e550c76c72d878857547ca001de24a
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -555,7 +555,7 @@ F ext/wasm/wasmfs.make ee0004813e16c283ff633e08b482008d56adf9b7d42f6c5612f7ab002
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0
-F main.mk d5fdf519235d657a3b87d70324f8ee1f057207626e5ab4daf23a0199d20a25ad
+F main.mk 73410c1f180660fd95c8203f35e1d4c1003e033d6bd0dbcb2c41610e4166500d
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1153,7 +1153,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c
 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634
 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830
 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2
-F test/fuzzcheck.c 7b501d55631c2d759e0bed02ed329904a35690fc6563d7b6cc69b7788a024f26
+F test/fuzzcheck.c 277da15bc937e9c71ec0000223365e2e0d55366605ff12cb9c004575648a5029
 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517
 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -1809,6 +1809,7 @@ F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
 F test/view.test d16e49e89ada6137d1447777ef2a74574526a3b024e6733bf53ae2960da8c17c
 F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679
 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456
+F test/vt02.c b94cbc59aba9b5a47f645fc8fc8aa7bdd96e686b98f3766e4080659cc07bf631
 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07
 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082
 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e
@@ -2053,8 +2054,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 24f12e681e06e3b71a6ac9c82255fe0270953a74c711405841f7e385eeafe874
+P e85387590061edbb6cdc04de792ab86f43afaa5c8d1b0792206cb5b481c7d76a
-R 71ac4be467afcb9551a419bbdef91c5e
+R dbf1d43633dc57fc37ff5c25dc929e02
-U stephan
+U drh
-Z 6815d8fe49b90127f256439e44a2ed16
+Z 6124d4e303fc5fa0fccc696b151a8f8d
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-e85387590061edbb6cdc04de792ab86f43afaa5c8d1b0792206cb5b481c7d76a
+e65c5bdc5bd6e1989c3de59798ff0a761125fc490fdec8c7671ca70ed688c4f8

test/fuzzcheck.c

@@ -85,6 +85,7 @@
 #include <ctype.h>
 #include <assert.h>
 #include "sqlite3.h"
+#include "sqlite3recover.h"
 #define ISSPACE(X) isspace((unsigned char)(X))
 #define ISDIGIT(X) isdigit((unsigned char)(X))
 
@@ -158,12 +159,10 @@ static struct GlobalVars {
 } g;
 
 /*
-** Include the external vt02.c module, if requested by compile-time
+** Include the external vt02.c module.
-** options.
 */
-#ifdef VT02_SOURCES
+extern int sqlite3_vt02_init(sqlite3*,char***,void*);
-# include "vt02.c"
+
-#endif
 
 /*
 ** Print an error message and quit.
@@ -629,6 +628,9 @@ static unsigned int oomCounter = 0;    /* Simulate OOM when equals 1 */
 static unsigned int oomRepeat = 0;     /* Number of OOMs in a row */
 static void*(*defaultMalloc)(int) = 0; /* The low-level malloc routine */
 
+/* Enable recovery */
+static int bNoRecover = 0;
+
 /* This routine is called when a simulated OOM occurs.  It is broken
 ** out as a separate routine to make it easy to set a breakpoint on
 ** the OOM
@@ -969,7 +971,7 @@ static int block_troublesome_sql(
 }
 
 /* Implementation found in fuzzinvariant.c */
-int fuzz_invariant(
+extern int fuzz_invariant(
   sqlite3 *db,            /* The database connection */
   sqlite3_stmt *pStmt,    /* Test statement stopped on an SQLITE_ROW */
   int iCnt,               /* Invariant sequence number, starting at 0 */
@@ -979,6 +981,52 @@ int fuzz_invariant(
   int eVerbosity          /* How much debugging output */
 );
 
+/* Implementation of sqlite_dbdata and sqlite_dbptr */
+extern int sqlite3_dbdata_init(sqlite3*,const char**,void*);
+
+
+/*
+** This function is used as a callback by the recover extension. Simply
+** print the supplied SQL statement to stdout.
+*/
+static int recoverSqlCb(void *pCtx, const char *zSql){
+  if( eVerbosity>=2 ){
+    printf("%s\n", zSql);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This function is called to recover data from the database.
+*/
+static int recoverDatabase(sqlite3 *db){
+  int rc = SQLITE_OK;
+  const char *zRecoveryDb = "";   /* Name of "recovery" database */
+  const char *zLAF = "lost_and_found";
+  int bFreelist = 1;
+  int bRowids = 1;
+  sqlite3_recover *p = 0;
+
+  p = sqlite3_recover_init_sql(db, "main", recoverSqlCb, 0);
+  sqlite3_recover_config(p, 789, (void*)zRecoveryDb);
+  sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF);
+  sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids);
+  sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist);
+  sqlite3_recover_run(p);
+  if( sqlite3_recover_errcode(p)!=SQLITE_OK ){
+    const char *zErr = sqlite3_recover_errmsg(p);
+    int errCode = sqlite3_recover_errcode(p);
+    if( eVerbosity>0 ){
+      printf("recovery error: %s (%d)\n", zErr, errCode);
+    }
+  }
+  rc = sqlite3_recover_finish(p);
+  if( eVerbosity>0 && rc ){
+     printf("recovery returns error code %d\n", rc);
+  }
+  return rc;
+}
+
 /*
 ** Run the SQL text
 */
@@ -1189,9 +1237,12 @@ int runCombinedDbSqlInput(
   ** deserialize to do this because deserialize depends on ATTACH */
   sqlite3_set_authorizer(cx.db, block_troublesome_sql, &btsFlags);
 
-#ifdef VT02_SOURCES
+  /* Add the vt02 virtual table */
   sqlite3_vt02_init(cx.db, 0, 0);
-#endif
+
+  /* Add support for sqlite_dbdata and sqlite_dbptr virtual tables used
+  ** by the recovery API */
+  sqlite3_dbdata_init(cx.db, 0, 0);
 
   /* Consistent PRNG seed */
 #ifdef SQLITE_TESTCTRL_PRNG_SEED
@@ -1201,6 +1252,12 @@ int runCombinedDbSqlInput(
   sqlite3_randomness(0,0);
 #endif
 
+  /* Run recovery on the initial database, just to make sure recovery
+  ** works. */
+  if( !bNoRecover ){
+    recoverDatabase(cx.db);
+  }
+
   zSql = sqlite3_malloc( nSql + 1 );
   if( zSql==0 ){
     fprintf(stderr, "Out of memory!\n");
@@ -1700,6 +1757,7 @@ static void showHelp(void){
 "  -m TEXT              Add a description to the database\n"
 "  --native-vfs         Use the native VFS for initially empty database files\n"
 "  --native-malloc      Turn off MEMSYS3/5 and Lookaside\n"
+"  --no-recover         Do not run recovery on dbsqlfuzz databases\n"
 "  --oss-fuzz           Enable OSS-FUZZ testing\n"
 "  --prng-seed N        Seed value for the PRGN inside of SQLite\n"
 "  -q|--quiet           Reduced output\n"
@@ -1851,6 +1909,9 @@ int main(int argc, char **argv){
       if( strcmp(z,"native-vfs")==0 ){
         nativeFlag = 1;
       }else
+      if( strcmp(z,"no-recover")==0 ){
+        bNoRecover = 1;
+      }else
       if( strcmp(z,"oss-fuzz")==0 ){
         ossFuzz = 1;
       }else