Fix recovery of utf-16 databases.

FossilOrigin-Name: 5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49

ext/misc/dbdata.c

@@ -75,6 +75,7 @@
 #include "sqlite3ext.h"
 
 typedef unsigned char u8;
+typedef unsigned int u32;
 
 #endif
 SQLITE_EXTENSION_INIT1
@@ -107,6 +108,7 @@ struct DbdataCursor {
   int iField;                     /* Current field number */
   u8 *pHdrPtr;
   u8 *pPtr;
+  u32 enc;                        /* Text encoding */
   
   sqlite3_int64 iIntkey;          /* Integer key value */
 };
@@ -299,14 +301,14 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){
 /* 
 ** Utility methods to decode 16 and 32-bit big-endian unsigned integers. 
 */
-static unsigned int get_uint16(unsigned char *a){
+static u32 get_uint16(unsigned char *a){
   return (a[0]<<8)|a[1];
 }
-static unsigned int get_uint32(unsigned char *a){
+static u32 get_uint32(unsigned char *a){
-  return ((unsigned int)a[0]<<24)
+  return ((u32)a[0]<<24)
-       | ((unsigned int)a[1]<<16)
+       | ((u32)a[1]<<16)
-       | ((unsigned int)a[2]<<8)
+       | ((u32)a[2]<<8)
-       | ((unsigned int)a[3]);
+       | ((u32)a[3]);
 }
 
 /*
@@ -321,7 +323,7 @@ static unsigned int get_uint32(unsigned char *a){
 */
 static int dbdataLoadPage(
   DbdataCursor *pCsr,             /* Cursor object */
-  unsigned int pgno,              /* Page number of page to load */
+  u32 pgno,                       /* Page number of page to load */
   u8 **ppPage,                    /* OUT: pointer to page buffer */
   int *pnPage                     /* OUT: Size of (*ppPage) in bytes */
 ){
@@ -405,6 +407,7 @@ static int dbdataValueBytes(int eType){
 */
 static void dbdataValue(
   sqlite3_context *pCtx, 
+  u32 enc,
   int eType, 
   u8 *pData,
   int nData
@@ -449,7 +452,17 @@ static void dbdataValue(
       default: {
         int n = ((eType-12) / 2);
         if( eType % 2 ){
-          sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT);
+          switch( enc ){
+            case SQLITE_UTF16BE:
+              sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
+              break;
+            case SQLITE_UTF16LE:
+              sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
+              break;
+            default:
+              sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT);
+              break;
+          }
         }else{
           sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
         }
@@ -588,7 +601,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
             /* Load content from overflow pages */
             if( nPayload>nLocal ){
               sqlite3_int64 nRem = nPayload - nLocal;
-              unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
+              u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
               while( nRem>0 ){
                 u8 *aOvfl = 0;
                 int nOvfl = 0;
@@ -703,6 +716,25 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
   return rc;
 }
 
+/*
+** Attempt to figure out the encoding of the database by retrieving page 1
+** and inspecting the header field. If successful, set the pCsr->enc variable
+** and return SQLITE_OK. Otherwise, return an SQLite error code.
+*/
+static int dbdataGetEncoding(DbdataCursor *pCsr){
+  int rc = SQLITE_OK;
+  int nPg1 = 0;
+  u8 *aPg1 = 0;
+  rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1);
+  assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 );
+  if( rc==SQLITE_OK && nPg1>0 ){
+    pCsr->enc = get_uint32(&aPg1[56]);
+  }
+  sqlite3_free(aPg1);
+  return rc;
+}
+
+
 /* 
 ** xFilter method for sqlite_dbdata and sqlite_dbptr.
 */
@@ -725,7 +757,6 @@ static int dbdataFilter(
     pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
     pCsr->bOnePage = 1;
   }else{
-    pCsr->nPage = dbdataDbsize(pCsr, zSchema);
     rc = dbdataDbsize(pCsr, zSchema);
   }
 
@@ -754,6 +785,13 @@ static int dbdataFilter(
   }else{
     pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
   }
+
+  /* Try to determine the encoding of the db by inspecting the header
+  ** field on page 1. */
+  if( rc==SQLITE_OK ){
+    rc = dbdataGetEncoding(pCsr);
+  }
+
   if( rc==SQLITE_OK ){
     rc = dbdataNext(pCursor);
   }
@@ -808,7 +846,8 @@ static int dbdataColumn(
           sqlite3_int64 iType;
           dbdataGetVarint(pCsr->pHdrPtr, &iType);
           dbdataValue(
-              ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
+              ctx, pCsr->enc, iType, pCsr->pPtr, 
+              &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
           );
         }
         break;

ext/recover/recover1.test

@@ -192,6 +192,7 @@ do_execsql_test 11.1 {
   INSERT INTO u1 VALUES('edvin marton', 'bond');
   INSERT INTO u1 VALUES(1, 4.0);
 }
+
 do_recover_test 11 
 
 

ext/recover/recoverfault2.test

@@ -78,7 +78,6 @@ ifcapable utf16 {
   faultsim_save_and_close
   
   proc my_sql_hook {sql} {
-  puts "HOOK $sql"
     lappend ::lSql $sql
     return 0
   }

ext/recover/test_recover.c

@@ -229,6 +229,40 @@ static int test_sqlite3_recover_init(
   return TCL_OK;
 }
 
+/*
+** Declaration for public API function in file dbdata.c. This may be called
+** with NULL as the final two arguments to register the sqlite_dbptr and
+** sqlite_dbdata virtual tables with a database handle.
+*/
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*);
+
+/*
+** sqlite3_recover_init DB DBNAME URI
+*/
+static int test_sqlite3_dbdata_init(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db = 0;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR;
+  sqlite3_dbdata_init(db, 0, 0);
+
+  Tcl_ResetResult(interp);
+  return TCL_OK;
+}
+
+
+
 int TestRecover_Init(Tcl_Interp *interp){
   struct Cmd {
     const char *zCmd;
@@ -237,6 +271,7 @@ int TestRecover_Init(Tcl_Interp *interp){
   } aCmd[] = {
     { "sqlite3_recover_init", test_sqlite3_recover_init, 0 },
     { "sqlite3_recover_init_sql", test_sqlite3_recover_init, (void*)1 },
+    { "sqlite3_dbdata_init", test_sqlite3_dbdata_init, (void*)1 },
   };
   int i;
 

manifest

@@ -1,5 +1,5 @@
-C Add\sOOM\stests\sfor\sthe\srecovery\sextension.
+C Fix\srecovery\sof\sutf-16\sdatabases.
-D 2022-09-13T20:40:57.096
+D 2022-09-14T16:37:59.285
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -299,7 +299,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8
 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9
 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9
 F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73
-F ext/misc/dbdata.c 9bb3666519bd8a54cce4934076a557fe6441c5bafce7e9c24d8b5ced148e8154
+F ext/misc/dbdata.c ca7b235fa2396e8fc2e950826872f820f31268ac2cb51368b0d655bb71568f07
 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01
 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12
 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1
@@ -387,18 +387,18 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2
 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291
 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812
 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a
-F ext/recover/recover1.test 623afa77b91996bb1319b069ced5245c243caa995f66793d6879fbc2c190f0be
+F ext/recover/recover1.test 167ad4b267f6db5c06963a72196527cb42369c7cd77de2b4273d9fdc8bd7a254
 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c
 F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60
 F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54
 F ext/recover/recoverfault.test f3587c218c448545a082b99d59294dff5ec0b7daa15b0556cf926f6c350f221e
-F ext/recover/recoverfault2.test 699b3ec07ba6982291e65e1811807f18d7f115234b407c819eaf3529878867f5
+F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b
 F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0
 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8
 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f
 F ext/recover/sqlite3recover.c 1afcac2bbfcf5ef67a3391f59678dc866348ac88745067e683052ede02c425fb
 F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640
-F ext/recover/test_recover.c 5941ecf484b6158be26e34c6f7b6c7f03967c72a63db0c08e7fd0fa43023eafa
+F ext/recover/test_recover.c 6a6f86ea61d728c67382047d574c62df83e6a28db23c329e93a177093689cd20
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996
 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890
@@ -1536,7 +1536,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl d759ac44a501fb832f2ea966429ca18acfba0f9a8d34ad5c499332b079b37023
+F test/tester.tcl 65c29b6f1dbf71b0e59a7b221d7e849dfa5a55fa7d0a2902811e8abdecdb1d44
 F test/testrunner.tcl 86b57135754ab2160aeb04b4829d321fb285a5cfa7a505fe61d69aed605854cc
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502
@@ -2011,8 +2011,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 60089547e1fc77ecc02f207ebf75ee3160e5ff25f41d12e02e170fd7fde66602
+P 9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502
-R 4e62121f5a47cd1728b6e036c3f3c8c0
+R 7d7577824671621531c65dae0de3846f
 U dan
-Z 5fc633ab904798afb52a452a7e949f53
+Z b3ef1682460d7df284b8cd64863e6715
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502
+5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49

test/tester.tcl

@@ -1548,6 +1548,47 @@ proc explain_i {sql {db db}} {
   output2 "----  ------------  ------  ------  ------  ----------------  --  -"
 }
 
+proc execsql_pp {sql {db db}} {
+  set nCol 0
+  $db eval $sql A {
+    if {$nCol==0} {
+      set nCol [llength $A(*)]
+      foreach c $A(*) { 
+        set aWidth($c) [string length $c] 
+        lappend data $c
+      }
+    }
+    foreach c $A(*) { 
+      set n [string length $A($c)]
+      if {$n > $aWidth($c)} {
+        set aWidth($c) $n
+      }
+      lappend data $A($c)
+    }
+  }
+  if {$nCol>0} {
+    set nTotal 0
+    foreach e [array names aWidth] { incr nTotal $aWidth($e) }
+    incr nTotal [expr ($nCol-1) * 3]
+    incr nTotal 4
+
+    set fmt ""
+    foreach c $A(*) { 
+      lappend fmt "% -$aWidth($c)s"
+    }
+    set fmt "| [join $fmt { | }] |"
+    
+    puts [string repeat - $nTotal]
+    for {set i 0} {$i < [llength $data]} {incr i $nCol} {
+      set vals [lrange $data $i [expr $i+$nCol-1]]
+      puts [format $fmt {*}$vals]
+      if {$i==0} { puts [string repeat - $nTotal] }
+    }
+    puts [string repeat - $nTotal]
+  }
+}
+
+
 # Show the VDBE program for an SQL statement but omit the Trace
 # opcode at the beginning.  This procedure can be used to prove
 # that different SQL statements generate exactly the same VDBE code.