From b8eaf9a10dbbcb217ffa5c85f2f2c99f9c584dc6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Sep 2022 16:37:59 +0000 Subject: [PATCH] Fix recovery of utf-16 databases. FossilOrigin-Name: 5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49 --- ext/misc/dbdata.c | 61 ++++++++++++++++++++++++++++------ ext/recover/recover1.test | 1 + ext/recover/recoverfault2.test | 1 - ext/recover/test_recover.c | 35 +++++++++++++++++++ manifest | 20 +++++------ manifest.uuid | 2 +- test/tester.tcl | 41 +++++++++++++++++++++++ 7 files changed, 138 insertions(+), 23 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index a18304b96b..29a6497bef 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -75,6 +75,7 @@ #include "sqlite3ext.h" typedef unsigned char u8; +typedef unsigned int u32; #endif SQLITE_EXTENSION_INIT1 @@ -107,6 +108,7 @@ struct DbdataCursor { int iField; /* Current field number */ u8 *pHdrPtr; u8 *pPtr; + u32 enc; /* Text encoding */ sqlite3_int64 iIntkey; /* Integer key value */ }; @@ -299,14 +301,14 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ /* ** Utility methods to decode 16 and 32-bit big-endian unsigned integers. */ -static unsigned int get_uint16(unsigned char *a){ +static u32 get_uint16(unsigned char *a){ return (a[0]<<8)|a[1]; } -static unsigned int get_uint32(unsigned char *a){ - return ((unsigned int)a[0]<<24) - | ((unsigned int)a[1]<<16) - | ((unsigned int)a[2]<<8) - | ((unsigned int)a[3]); +static u32 get_uint32(unsigned char *a){ + return ((u32)a[0]<<24) + | ((u32)a[1]<<16) + | ((u32)a[2]<<8) + | ((u32)a[3]); } /* @@ -321,7 +323,7 @@ static unsigned int get_uint32(unsigned char *a){ */ static int dbdataLoadPage( DbdataCursor *pCsr, /* Cursor object */ - unsigned int pgno, /* Page number of page to load */ + u32 pgno, /* Page number of page to load */ u8 **ppPage, /* OUT: pointer to page buffer */ int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ @@ -405,6 +407,7 @@ static int dbdataValueBytes(int eType){ */ static void dbdataValue( sqlite3_context *pCtx, + u32 enc, int eType, u8 *pData, int nData @@ -449,7 +452,17 @@ static void dbdataValue( default: { int n = ((eType-12) / 2); if( eType % 2 ){ - sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); + switch( enc ){ + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } }else{ sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); } @@ -588,7 +601,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Load content from overflow pages */ if( nPayload>nLocal ){ sqlite3_int64 nRem = nPayload - nLocal; - unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); while( nRem>0 ){ u8 *aOvfl = 0; int nOvfl = 0; @@ -703,6 +716,25 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ return rc; } +/* +** Attempt to figure out the encoding of the database by retrieving page 1 +** and inspecting the header field. If successful, set the pCsr->enc variable +** and return SQLITE_OK. Otherwise, return an SQLite error code. +*/ +static int dbdataGetEncoding(DbdataCursor *pCsr){ + int rc = SQLITE_OK; + int nPg1 = 0; + u8 *aPg1 = 0; + rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); + assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 ); + if( rc==SQLITE_OK && nPg1>0 ){ + pCsr->enc = get_uint32(&aPg1[56]); + } + sqlite3_free(aPg1); + return rc; +} + + /* ** xFilter method for sqlite_dbdata and sqlite_dbptr. */ @@ -725,7 +757,6 @@ static int dbdataFilter( pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); pCsr->bOnePage = 1; }else{ - pCsr->nPage = dbdataDbsize(pCsr, zSchema); rc = dbdataDbsize(pCsr, zSchema); } @@ -754,6 +785,13 @@ static int dbdataFilter( }else{ pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } + + /* Try to determine the encoding of the db by inspecting the header + ** field on page 1. */ + if( rc==SQLITE_OK ){ + rc = dbdataGetEncoding(pCsr); + } + if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); } @@ -808,7 +846,8 @@ static int dbdataColumn( sqlite3_int64 iType; dbdataGetVarint(pCsr->pHdrPtr, &iType); dbdataValue( - ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr + ctx, pCsr->enc, iType, pCsr->pPtr, + &pCsr->pRec[pCsr->nRec] - pCsr->pPtr ); } break; diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index e7a45e8e90..358015a591 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -192,6 +192,7 @@ do_execsql_test 11.1 { INSERT INTO u1 VALUES('edvin marton', 'bond'); INSERT INTO u1 VALUES(1, 4.0); } + do_recover_test 11 diff --git a/ext/recover/recoverfault2.test b/ext/recover/recoverfault2.test index 7489206050..4f7131ecfb 100644 --- a/ext/recover/recoverfault2.test +++ b/ext/recover/recoverfault2.test @@ -78,7 +78,6 @@ ifcapable utf16 { faultsim_save_and_close proc my_sql_hook {sql} { - puts "HOOK $sql" lappend ::lSql $sql return 0 } diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index b273fbcd1d..7ab440fa37 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -229,6 +229,40 @@ static int test_sqlite3_recover_init( return TCL_OK; } +/* +** Declaration for public API function in file dbdata.c. This may be called +** with NULL as the final two arguments to register the sqlite_dbptr and +** sqlite_dbdata virtual tables with a database handle. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +/* +** sqlite3_recover_init DB DBNAME URI +*/ +static int test_sqlite3_dbdata_init( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; + sqlite3_dbdata_init(db, 0, 0); + + Tcl_ResetResult(interp); + return TCL_OK; +} + + + int TestRecover_Init(Tcl_Interp *interp){ struct Cmd { const char *zCmd; @@ -237,6 +271,7 @@ int TestRecover_Init(Tcl_Interp *interp){ } aCmd[] = { { "sqlite3_recover_init", test_sqlite3_recover_init, 0 }, { "sqlite3_recover_init_sql", test_sqlite3_recover_init, (void*)1 }, + { "sqlite3_dbdata_init", test_sqlite3_dbdata_init, (void*)1 }, }; int i; diff --git a/manifest b/manifest index 18c614dd00..b17390fa0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sOOM\stests\sfor\sthe\srecovery\sextension. -D 2022-09-13T20:40:57.096 +C Fix\srecovery\sof\sutf-16\sdatabases. +D 2022-09-14T16:37:59.285 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -299,7 +299,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 -F ext/misc/dbdata.c 9bb3666519bd8a54cce4934076a557fe6441c5bafce7e9c24d8b5ced148e8154 +F ext/misc/dbdata.c ca7b235fa2396e8fc2e950826872f820f31268ac2cb51368b0d655bb71568f07 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 @@ -387,18 +387,18 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test 623afa77b91996bb1319b069ced5245c243caa995f66793d6879fbc2c190f0be +F ext/recover/recover1.test 167ad4b267f6db5c06963a72196527cb42369c7cd77de2b4273d9fdc8bd7a254 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 F ext/recover/recoverfault.test f3587c218c448545a082b99d59294dff5ec0b7daa15b0556cf926f6c350f221e -F ext/recover/recoverfault2.test 699b3ec07ba6982291e65e1811807f18d7f115234b407c819eaf3529878867f5 +F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f F ext/recover/sqlite3recover.c 1afcac2bbfcf5ef67a3391f59678dc866348ac88745067e683052ede02c425fb F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 -F ext/recover/test_recover.c 5941ecf484b6158be26e34c6f7b6c7f03967c72a63db0c08e7fd0fa43023eafa +F ext/recover/test_recover.c 6a6f86ea61d728c67382047d574c62df83e6a28db23c329e93a177093689cd20 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -1536,7 +1536,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl d759ac44a501fb832f2ea966429ca18acfba0f9a8d34ad5c499332b079b37023 +F test/tester.tcl 65c29b6f1dbf71b0e59a7b221d7e849dfa5a55fa7d0a2902811e8abdecdb1d44 F test/testrunner.tcl 86b57135754ab2160aeb04b4829d321fb285a5cfa7a505fe61d69aed605854cc F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2011,8 +2011,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 60089547e1fc77ecc02f207ebf75ee3160e5ff25f41d12e02e170fd7fde66602 -R 4e62121f5a47cd1728b6e036c3f3c8c0 +P 9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502 +R 7d7577824671621531c65dae0de3846f U dan -Z 5fc633ab904798afb52a452a7e949f53 +Z b3ef1682460d7df284b8cd64863e6715 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c3276310d..05a5c951ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502 \ No newline at end of file +5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49 \ No newline at end of file diff --git a/test/tester.tcl b/test/tester.tcl index 5612311dab..9353b91244 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1548,6 +1548,47 @@ proc explain_i {sql {db db}} { output2 "---- ------------ ------ ------ ------ ---------------- -- -" } +proc execsql_pp {sql {db db}} { + set nCol 0 + $db eval $sql A { + if {$nCol==0} { + set nCol [llength $A(*)] + foreach c $A(*) { + set aWidth($c) [string length $c] + lappend data $c + } + } + foreach c $A(*) { + set n [string length $A($c)] + if {$n > $aWidth($c)} { + set aWidth($c) $n + } + lappend data $A($c) + } + } + if {$nCol>0} { + set nTotal 0 + foreach e [array names aWidth] { incr nTotal $aWidth($e) } + incr nTotal [expr ($nCol-1) * 3] + incr nTotal 4 + + set fmt "" + foreach c $A(*) { + lappend fmt "% -$aWidth($c)s" + } + set fmt "| [join $fmt { | }] |" + + puts [string repeat - $nTotal] + for {set i 0} {$i < [llength $data]} {incr i $nCol} { + set vals [lrange $data $i [expr $i+$nCol-1]] + puts [format $fmt {*}$vals] + if {$i==0} { puts [string repeat - $nTotal] } + } + puts [string repeat - $nTotal] + } +} + + # Show the VDBE program for an SQL statement but omit the Trace # opcode at the beginning. This procedure can be used to prove # that different SQL statements generate exactly the same VDBE code.