Improve handling of corruption in the sqlite_schema table b-tree structure.

FossilOrigin-Name: 2be0dba12fb9b98cd9632f1ff3f30aeb29661a2e121b68c2f3335617b027797b

ext/recover/recovercorrupt.test

@@ -18,6 +18,8 @@ source $testdir/tester.tcl
 
 set testprefix recovercorrupt
 
+database_may_be_corrupt
+
 do_execsql_test 1.0 {
   PRAGMA page_size = 512;
   CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
@@ -46,19 +48,21 @@ proc toggle_bit {blob bit} {
 
 
 db_save_and_close
-for {set ii 200} {$ii < 10000} {incr ii} {
+for {set ii 0} {$ii < 10000} {incr ii} {
   db_restore_and_reopen
   db func toggle_bit toggle_bit
-  set pg [expr {($ii / 512)+1}]
-  set byte [expr {$ii % 512}]
+  set bitsperpage [expr 512*8]
+
+  set pg [expr {($ii / $bitsperpage)+1}]
+  set byte [expr {$ii % $bitsperpage}]
   db eval {
     UPDATE sqlite_dbpage SET data = toggle_bit(data, $byte) WHERE pgno=$pg
   }
 
-  do_test 1.2.$ii {
     set R [sqlite3_recover_init db main test.db2]
     $R config lostandfound lost_and_found
     $R run
+  do_test 1.2.$ii {
     $R finish
   } {}
 }

ext/recover/sqlite3recover.c

@@ -192,6 +192,10 @@ struct sqlite3_recover {
   int bRecoverRowid;              /* SQLITE_RECOVER_ROWIDS setting */
   int bSlowIndexes;               /* SQLITE_RECOVER_SLOWINDEXES setting */
 
+  int pgsz;
+  u8 *pPage1Disk;
+  u8 *pPage1Cache;
+
   /* Error code and error message */
   int errCode;                    /* For sqlite3_recover_errcode() */
   char *zErrMsg;                  /* For sqlite3_recover_errmsg() */
@@ -248,6 +252,7 @@ struct sqlite3_recover {
 typedef struct RecoverGlobal RecoverGlobal;
 struct RecoverGlobal {
   const sqlite3_io_methods *pMethods;
+  sqlite3_recover *p;
 };
 static RecoverGlobal recover_g;
 
@@ -661,8 +666,20 @@ static void recoverGetPage(
     if( pStmt ){
       sqlite3_bind_int64(pStmt, 1, pgno);
       if( SQLITE_ROW==sqlite3_step(pStmt) ){
+        int bDone = 0;
         assert( p->errCode==SQLITE_OK );
-        sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0));
+        if( pgno==1 ){
+          const u8 *aPg = sqlite3_column_blob(pStmt, 0);
+          int nPg = sqlite3_column_bytes(pStmt, 0);
+          if( nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){
+            sqlite3_result_blob(pCtx, p->pPage1Disk, nPg, SQLITE_STATIC);
+            bDone = 1;
+          }
+        }
+        
+        if( !bDone ){
+          sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0));
+        }
       }
       recoverReset(p, pStmt);
     }
@@ -1914,7 +1931,7 @@ static void recoverPutU16(u8 *a, u32 v){
   a[0] = (v>>8) & 0x00FF;
   a[1] = (v>>0) & 0x00FF;
 }
-static u32 recoverPutU32(u8 *a, u32 v){
+static void recoverPutU32(u8 *a, u32 v){
   a[0] = (v>>24) & 0x00FF;
   a[1] = (v>>16) & 0x00FF;
   a[2] = (v>>8) & 0x00FF;
@@ -1926,7 +1943,10 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){
   if( pFd->pMethods==&recover_methods ){
     pFd->pMethods = recover_g.pMethods;
     rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff);
-    if( rc==SQLITE_OK && iOff==0 && nByte>=100 ){
+    if( nByte==16 ){
+      sqlite3_randomness(16, aBuf);
+    }else
+    if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){
       /* Ensure that the database has a valid header file. The only fields
       ** that really matter to recovery are:
       **
@@ -1942,9 +1962,12 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){
       ** We also try to preserve the auto-vacuum, incr-value, user-version
       ** and application-id fields - all 32 bit quantities at offsets 
       ** 52, 60, 64 and 68. All other fields are set to known good values.
+      **
+      ** Byte offset 105 should also contain the page-size as a 16-bit 
+      ** integer.
       */
       const int aPreserve[] = {32, 36, 52, 60, 64, 68};
-      u8 aHdr[100] = {
+      u8 aHdr[108] = {
         0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 
         0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00,
         0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20,
@@ -1957,7 +1980,9 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){
         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x2e, 0x5b, 0x30
+        0x00, 0x2e, 0x5b, 0x30,
+
+        0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00
       };
       u8 *a = (u8*)aBuf;
 
@@ -1966,23 +1991,44 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){
       u32 dbsz = 0;
       i64 dbFileSize = 0;
       int ii;
+      sqlite3_recover *p = recover_g.p;
 
       if( pgsz==0x01 ) pgsz = 65536;
       rc = pFd->pMethods->xFileSize(pFd, &dbFileSize);
 
-      dbsz = dbFileSize / pgsz;
+      if( pgsz ){
+        dbsz = dbFileSize / pgsz;
+      }
       if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){
         enc = SQLITE_UTF8;
       }
 
-      recoverPutU32(&aHdr[28], dbsz);
-      recoverPutU32(&aHdr[56], enc);
-      if( pgsz==65536 ) pgsz = 1;
-      recoverPutU16(&aHdr[16], pgsz);
-      for(ii=0; ii<sizeof(aPreserve)/sizeof(aPreserve[0]); ii++){
-        memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4);
+      sqlite3_free(p->pPage1Cache);
+      p->pPage1Cache = 0;
+      p->pPage1Disk = 0;
+
+      p->pgsz = nByte;
+      p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2);
+      if( p->pPage1Cache ){
+        p->pPage1Disk = &p->pPage1Cache[nByte];
+        memcpy(p->pPage1Disk, aBuf, nByte);
+
+        recoverPutU32(&aHdr[28], dbsz);
+        recoverPutU32(&aHdr[56], enc);
+        recoverPutU16(&aHdr[105], pgsz);
+        if( pgsz==65536 ) pgsz = 1;
+        recoverPutU16(&aHdr[16], pgsz);
+        for(ii=0; ii<sizeof(aPreserve)/sizeof(aPreserve[0]); ii++){
+          memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4);
+        }
+        memcpy(aBuf, aHdr, sizeof(aHdr));
+        memset(&((u8*)aBuf)[sizeof(aHdr)], 0, nByte-sizeof(aHdr));
+
+        memcpy(p->pPage1Cache, aBuf, nByte);
+      }else{
+        rc = p->errCode;
       }
-      memcpy(aBuf, aHdr, sizeof(aHdr));
+
     }
     pFd->pMethods = &recover_methods;
   }else{
@@ -2088,6 +2134,7 @@ static void recoverInstallWrapper(sqlite3_recover *p){
   sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd);
   if( pFd ){
     recover_g.pMethods = pFd->pMethods;
+    recover_g.p = p;
     pFd->pMethods = &recover_methods;
   }
 }
@@ -2098,6 +2145,7 @@ static void recoverUninstallWrapper(sqlite3_recover *p){
     assert( pFd );
     pFd->pMethods = recover_g.pMethods;
     recover_g.pMethods = 0;
+    recover_g.p = 0;
   }
 }
 
@@ -2384,6 +2432,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){
     sqlite3_free(p->zErrMsg);
     sqlite3_free(p->zStateDb);
     sqlite3_free(p->zLostAndFound);
+    sqlite3_free(p->pPage1Cache);
     sqlite3_free(p);
   }
   return rc;

manifest

@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges.
-D 2022-10-04T19:47:17.725
+C Improve\shandling\sof\scorruption\sin\sthe\ssqlite_schema\stable\sb-tree\sstructure.
+D 2022-10-05T16:58:28.898
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -389,7 +389,7 @@ F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f5974282
 F ext/recover/recover1.test 2b0f048ba572c3ca4e4235b632bf75702975a9656146eab766f6c75861f946a0
 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c
 F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60
-F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54
+F ext/recover/recovercorrupt.test 69af3d68aedc2cf1c3c41dbd6afb45b3cebadb57796d513550b5fd1e2a8b3fba
 F ext/recover/recovercorrupt2.test a131d8005337c092e2dfa3b84909ed67ae82d22399a8cfb8c984b2939969ca42
 F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7a59144e47d7cb568
 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b
@@ -397,7 +397,7 @@ F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e9
 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8
 F ext/recover/recoverslowidx.test 7e3889e50758c614b9b3e75187eeeea425c0ca8a2387441fc19ae749a96a7949
 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f
-F ext/recover/sqlite3recover.c 9c4d742223c519ef7322c51cb8997ca1046ca1971c06b617d455ec0bf0a9b390
+F ext/recover/sqlite3recover.c c40381677e18f841be9e878591bf68cab043e99b408374561179829d0b041095
 F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36
 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
@@ -534,7 +534,7 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca
 F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
-F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 w config.h.in
+F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2
 F src/alter.c 0390ca1d69ec3626cfa9f153114b7ab233e6b2bada6a9eb91361ed385fe90deb
 F src/analyze.c aabdf3769c7fd9954a8ec508eb7041ae174b66f88d12c47199fabbea9a646467
 F src/attach.c 4431f82f0247bf3aaf91589acafdff77d1882235c95407b36da1585c765fbbc8
@@ -2014,8 +2014,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 17f68d803685405d880025134c71bac01077962d677f2902a608952d0b9cdb5a 3efa811251d4510a074231a1e0b667783cc5f90777466b9dcae675cd9892b423
-R a7e16b1f572172e278e4e3a2a9c939da
+P b0feecaa84b88f534edb49056cf7e92256f4a687c8b4c44e1d1186709a304378
+R c8e986f8000e60f0f5539ba1d6775390
 U dan
-Z 733ee477925392618f80fcf6c31db3f8
+Z 4a7b8684cf08c1b2ec449e50db6581b5
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-b0feecaa84b88f534edb49056cf7e92256f4a687c8b4c44e1d1186709a304378
+2be0dba12fb9b98cd9632f1ff3f30aeb29661a2e121b68c2f3335617b027797b