From be2e212cf93f6bae68dc75ed0aa9f2b07988cd3f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Sep 2022 20:31:36 +0000 Subject: [PATCH] Take the freelist into account when recovering data that is not linked in to any tree associated with a schema entry. FossilOrigin-Name: dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a --- ext/recover/recoverold.test | 17 ++++++++++ ext/recover/sqlite3recover.c | 64 ++++++++++++++++++++++++++++++++++-- manifest | 14 ++++---- manifest.uuid | 2 +- 4 files changed, 86 insertions(+), 11 deletions(-) diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test index 61f09397b8..001426b88d 100644 --- a/ext/recover/recoverold.test +++ b/ext/recover/recoverold.test @@ -131,6 +131,23 @@ do_recover_test 2.4.1 { 2 2 3 {} 8 9 7 } +do_execsql_test 2.5 { + CREATE TABLE x1(a, b, c); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO x1 SELECT i, i, hex(randomblob(500)) FROM s; + DROP TABLE x1; +} +do_recover_test 2.5.1 { + SELECT name FROM sqlite_master; + SELECT * FROM lost_and_found_1; +} {lost_and_found lost_and_found_0 lost_and_found_1 + 2 2 3 {} 2 3 1 + 2 2 3 {} 5 6 4 + 2 2 3 {} 8 9 7 +} + #------------------------------------------------------------------------- breakpoint reset_db diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index ec333f0d05..34cc0cdd50 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -107,7 +107,7 @@ static int recoverStrlen(const char *zStr){ return nRet; } -static void *recoverMalloc(sqlite3_recover *p, sqlite3_int64 nByte){ +static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ void *pRet = 0; assert( nByte>0 ); if( p->errCode==SQLITE_OK ){ @@ -276,6 +276,36 @@ static i64 recoverPageCount(sqlite3_recover *p){ return nPg; } +/* +** Scalar function "read_i32". The first argument to this function +** must be a blob. The second a non-negative integer. This function +** reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +*/ +static void recoverReadI32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + assert( argc==2 ); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]); + + if( iInt>=0 && (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + i64 iVal = ((i64)a[0]<<24) + + ((i64)a[1]<<16) + + ((i64)a[2]<< 8) + + ((i64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + /* ** SELECT page_is_used(pgno); */ @@ -285,7 +315,7 @@ static void recoverPageIsUsed( sqlite3_value **apArg ){ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); - sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]); + i64 pgno = sqlite3_value_int64(apArg[0]); sqlite3_stmt *pStmt = 0; int bRet; @@ -314,7 +344,7 @@ static void recoverGetPage( sqlite3_value **apArg ){ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); - sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]); + i64 pgno = sqlite3_value_int64(apArg[0]); sqlite3_stmt *pStmt = 0; assert( nArg==1 ); @@ -397,6 +427,11 @@ static int recoverOpenOutput(sqlite3_recover *p){ db, "page_is_used", 1, SQLITE_UTF8, (void*)p, recoverPageIsUsed, 0, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "read_i32", 2, SQLITE_UTF8, (void*)p, recoverReadI32, 0, 0 + ); + } if( rc!=SQLITE_OK ){ if( p->errCode==SQLITE_OK ) rc = recoverDbError(p, db); @@ -852,6 +887,29 @@ static int recoverLostAndFound(sqlite3_recover *p){ } recoverFinalize(p, pStmt); + /* Add all pages that appear to be part of the freelist to the bitmap. */ + pStmt = recoverPrepare(p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")" + "SELECT freepgno FROM freelist" + ); + while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iPg = sqlite3_column_int64(pStmt, 0); + recoverBitmapSet(pMap, iPg); + } + recoverFinalize(p, pStmt); + /* Add an entry for each page not already added to the bitmap to ** the recovery.map table. This loop leaves the "parent" column ** of each recovery.map row set to NULL - to be filled in below. */ diff --git a/manifest b/manifest index 10281dc64e..00fb244d2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\swork\son\smaking\sthe\srecover\sextension\scompatible\swith\sthe\s.recover\scommand. -D 2022-09-03T20:07:39.011 +C Take\sthe\sfreelist\sinto\saccount\swhen\srecovering\sdata\sthat\sis\snot\slinked\sin\sto\sany\stree\sassociated\swith\sa\sschema\sentry. +D 2022-09-03T20:31:36.832 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -389,8 +389,8 @@ F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9c F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/recoverold.test 33ccbe2393af0e82f292c135b725e3eca1e803960681cf6da41fc00d28bd8683 -F ext/recover/sqlite3recover.c 8d93b9aa056c3fae9a5e2736a4ffa71414bdb502863ef879e55bec7b37030266 +F ext/recover/recoverold.test 7578e9b938db15dc469a4af247e15866226f366bde0cbe09a40b0aef4a0506c8 +F ext/recover/sqlite3recover.c 6c9cbc993a970060f9fb881d78f6c7e182ec988a5e48acbf15bb4a5f05ce2902 F ext/recover/sqlite3recover.h b82974790b528480163d87dcd84afffe7568393194c9ec8241cfbc3ee6bbdd1b F ext/recover/test_recover.c b8dddd96ccd4a62bc14cb3a8d5696407892e184fe7d45ecbedde954577857de2 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2005,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759 -R 93788fe8081a2011cde8fac3142d518f +P f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2 +R 471b646b541e0fcab850e84cb036ac46 U dan -Z 21514150d144c868f0bbea27057fe308 +Z 9f579b130a06a2078244b88aebbd5365 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 093f196c7a..f01f3f35e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2 \ No newline at end of file +dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a \ No newline at end of file