From 73b09b87d5a7c9e35af045b5b59cbd49756cb4df Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Aug 2022 20:45:43 +0000 Subject: [PATCH] Add new files for an extension to recover data from corrupted databases. FossilOrigin-Name: f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 --- ext/misc/dbdata.c | 31 +- ext/recover/recover1.test | 116 ++++++ ext/recover/recover_common.tcl | 5 + ext/recover/sqlite3recover.c | 728 +++++++++++++++++++++++++++++++++ ext/recover/sqlite3recover.h | 66 +++ ext/recover/test_recover.c | 185 +++++++++ main.mk | 3 + manifest | 26 +- manifest.uuid | 2 +- src/test_tclsh.c | 2 + 10 files changed, 1152 insertions(+), 12 deletions(-) create mode 100644 ext/recover/recover1.test create mode 100644 ext/recover/recover_common.tcl create mode 100644 ext/recover/sqlite3recover.c create mode 100644 ext/recover/sqlite3recover.h create mode 100644 ext/recover/test_recover.c diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index 7405e7c890..b79eafce7c 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -660,6 +660,18 @@ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ return pCsr->aPage==0; } +/* +** Return true if nul-terminated string zSchema ends in "()". Or false +** otherwise. +*/ +static int dbdataIsFunction(const char *zSchema){ + int n = strlen(zSchema); + if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){ + return n-2; + } + return 0; +} + /* ** Determine the size in pages of database zSchema (where zSchema is ** "main", "temp" or the name of an attached database) and set @@ -670,10 +682,16 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; char *zSql = 0; int rc, rc2; + int nFunc = 0; sqlite3_stmt *pStmt = 0; - zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema); + }else{ + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + } if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -711,9 +729,18 @@ static int dbdataFilter( } if( rc==SQLITE_OK ){ + int nFunc = 0; if( pTab->pStmt ){ pCsr->pStmt = pTab->pStmt; pTab->pStmt = 0; + }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } }else{ rc = sqlite3_prepare_v2(pTab->db, "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, @@ -732,7 +759,7 @@ static int dbdataFilter( return rc; } -/* +/* ** Return a column for the sqlite_dbdata or sqlite_dbptr table. */ static int dbdataColumn( diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test new file mode 100644 index 0000000000..c4348baea0 --- /dev/null +++ b/ext/recover/recover1.test @@ -0,0 +1,116 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl + +set testprefix recover1 + + + +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} + +proc do_recover_test {tn} { + forcedelete test.db2 + + uplevel [list do_test $tn.1 { + set R [sqlite3_recover_init db main test.db2] + $R step + $R finish + } {}] + + sqlite3 db2 test.db2 + uplevel [list do_test $tn.2 [list compare_dbs db db2] {}] + db2 close +} + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b) WITHOUT ROWID; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT i*2, hex(randomblob(250)) FROM s; + INSERT INTO t2 SELECT * FROM t1; +} + +do_recover_test 1 + +do_execsql_test 2.0 { + ALTER TABLE t1 ADD COLUMN c DEFAULT 'xyz' +} +do_recover_test 2 + +do_execsql_test 3.0 { + CREATE INDEX i1 ON t1(c); +} +do_recover_test 3 + +do_execsql_test 4.0 { + CREATE VIEW v1 AS SELECT * FROM t2; +} +do_recover_test 4 + +do_execsql_test 5.0 { + CREATE UNIQUE INDEX i2 ON t1(c, b); +} +do_recover_test 5 + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1( + a INTEGER PRIMARY KEY, + b INT, + c TEXT, + d INT GENERATED ALWAYS AS (a*abs(b)) VIRTUAL, + e TEXT GENERATED ALWAYS AS (substr(c,b,b+1)) STORED, + f TEXT GENERATED ALWAYS AS (substr(c,b,b+1)) STORED + ); + + INSERT INTO t1 VALUES(1, 2, 'hello world'); +} +do_recover_test 6 + +do_execsql_test 7.0 { + CREATE TABLE t2(i, j GENERATED ALWAYS AS (i+1) STORED, k); + INSERT INTO t2 VALUES(10, 'ten'); +} +do_execsql_test 7.1 { + SELECT * FROM t2 +} {10 11 ten} + +do_recover_test 7.2 + +finish_test + diff --git a/ext/recover/recover_common.tcl b/ext/recover/recover_common.tcl new file mode 100644 index 0000000000..3f2ff2d6cc --- /dev/null +++ b/ext/recover/recover_common.tcl @@ -0,0 +1,5 @@ + + + + + diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c new file mode 100644 index 0000000000..6c7828c007 --- /dev/null +++ b/ext/recover/sqlite3recover.c @@ -0,0 +1,728 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + + +#include "sqlite3recover.h" +#include +#include + +typedef unsigned int u32; +typedef sqlite3_int64 i64; + +typedef struct RecoverColumn RecoverColumn; +struct RecoverColumn { + char *zCol; + int eHidden; +}; + +#define RECOVER_EHIDDEN_NONE 0 +#define RECOVER_EHIDDEN_HIDDEN 1 +#define RECOVER_EHIDDEN_VIRTUAL 2 +#define RECOVER_EHIDDEN_STORED 3 + +/* +** When running the ".recover" command, each output table, and the special +** orphaned row table if it is required, is represented by an instance +** of the following struct. +*/ +typedef struct RecoverTable RecoverTable; +struct RecoverTable { + u32 iRoot; /* Root page in original database */ + char *zTab; /* Name of table */ + int nCol; /* Number of columns in table */ + RecoverColumn *aCol; /* Array of columns */ + int bIntkey; /* True for intkey, false for without rowid */ + int iPk; /* Index of IPK column, if bIntkey */ + + RecoverTable *pNext; +}; + +/* +** +*/ +#define RECOVERY_SCHEMA \ +" CREATE TABLE recovery.freelist(" \ +" pgno INTEGER PRIMARY KEY" \ +" );" \ +" CREATE TABLE recovery.dbptr(" \ +" pgno, child, PRIMARY KEY(child, pgno)" \ +" ) WITHOUT ROWID;" \ +" CREATE TABLE recovery.map(" \ +" pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" \ +" );" \ +" CREATE TABLE recovery.schema(" \ +" type, name, tbl_name, rootpage, sql" \ +" );" + + +struct sqlite3_recover { + sqlite3 *dbIn; + sqlite3 *dbOut; + + sqlite3_stmt *pGetPage; + + char *zDb; + char *zUri; + RecoverTable *pTblList; + + int errCode; /* For sqlite3_recover_errcode() */ + char *zErrMsg; /* For sqlite3_recover_errmsg() */ + + char *zStateDb; +}; + +/* +** Like strlen(). But handles NULL pointer arguments. +*/ +static int recoverStrlen(const char *zStr){ + int nRet = 0; + if( zStr ){ + while( zStr[nRet] ) nRet++; + } + return nRet; +} + +static void *recoverMalloc(sqlite3_recover *p, sqlite3_int64 nByte){ + void *pRet = 0; + assert( nByte>0 ); + if( p->errCode==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + p->errCode = SQLITE_NOMEM; + } + } + return pRet; +} + +static int recoverError( + sqlite3_recover *p, + int errCode, + const char *zFmt, ... +){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + + sqlite3_free(p->zErrMsg); + p->zErrMsg = z; + p->errCode = errCode; + return errCode; +} + +static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ + return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); +} + +static sqlite3_stmt *recoverPrepare( + sqlite3_recover *p, + sqlite3 *db, + const char *zSql +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ + recoverDbError(p, db); + } + } + return pStmt; +} + +/* +** Create a prepared statement using printf-style arguments for the SQL. +*/ +static sqlite3_stmt *recoverPreparePrintf( + sqlite3_recover *p, + sqlite3 *db, + const char *zFmt, ... +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + p->errCode = SQLITE_NOMEM; + }else{ + pStmt = recoverPrepare(p, db, z); + sqlite3_free(z); + } + } + return pStmt; +} + + +static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, sqlite3_db_handle(pStmt)); + } + return pStmt; +} + +static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, db); + } +} + +static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc ){ + recoverDbError(p, p->dbOut); + } + } + return p->errCode; +} + +/* +** The implementation of a user-defined SQL function invoked by the +** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages +** of the database being recovered. +** +** This function always takes a single integer argument. If the arguement +** is zero, then the value returned is the number of pages in the db being +** recovered. If the argument is greater than zero, it is a page number. +** The value returned in this case is an SQL blob containing the data for +** the identified page of the db being recovered. e.g. +** +** SELECT getpage(0); -- return number of pages in db +** SELECT getpage(4); -- return page 4 of db as a blob of data +*/ +static void recoverGetPage( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]); + sqlite3_stmt *pStmt = 0; + + assert( nArg==1 ); + if( pgno==0 ){ + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + }else if( p->pGetPage==0 ){ + pStmt = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); + }else{ + pStmt = p->pGetPage; + } + + if( pStmt ){ + if( pgno ) sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); + } + if( pgno ){ + p->pGetPage = recoverReset(p, pStmt); + }else{ + recoverFinalize(p, pStmt); + } + } + + if( p->errCode ){ + if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); + sqlite3_result_error_code(pCtx, p->errCode); + } +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +static int recoverOpenOutput(sqlite3_recover *p){ + int rc = SQLITE_OK; + if( p->dbOut==0 ){ + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + sqlite3 *db = 0; + + assert( p->dbOut==0 ); + + rc = sqlite3_open_v2(p->zUri, &db, flags, 0); + if( rc==SQLITE_OK ){ + const char *zPath = p->zStateDb ? p->zStateDb : ":memory:"; + char *zSql = sqlite3_mprintf("ATTACH %Q AS recovery", zPath); + if( zSql==0 ){ + rc = p->errCode = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(db, zSql, 0, 0, 0); + } + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db, "recovery"); + if( pBackup ){ + while( sqlite3_backup_step(pBackup, 1000)==SQLITE_OK ); + rc = sqlite3_backup_finish(pBackup); + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, RECOVERY_SCHEMA, 0, 0, 0); + } + + if( rc==SQLITE_OK ){ + sqlite3_dbdata_init(db, 0, 0); + rc = sqlite3_create_function( + db, "getpage", 1, SQLITE_UTF8, (void*)p, recoverGetPage, 0, 0 + ); + } + + if( rc!=SQLITE_OK ){ + if( p->errCode==SQLITE_OK ) rc = recoverDbError(p, db); + sqlite3_close(db); + }else{ + p->dbOut = db; + } + } + return rc; +} + +static int recoverCacheDbptr(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "INSERT INTO recovery.dbptr " + "SELECT pgno, child FROM sqlite_dbptr('getpage()')" + ); +} + +static int recoverCacheSchema(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "WITH RECURSIVE pages(p) AS (" + " SELECT 1" + " UNION" + " SELECT child FROM recovery.dbptr, pages WHERE pgno=p" + ")" + "INSERT INTO recovery.schema SELECT" + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" + " SELECT p FROM pages" + ") GROUP BY pgno, cell" + ); +} + +static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ + sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, + "PRAGMA table_xinfo(%Q)", zName + ); + + if( pStmt ){ + RecoverTable *pNew = 0; + int nCol = 0; + int nName = recoverStrlen(zName); + int nByte = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol++; + nByte += (sqlite3_column_bytes(pStmt, 1)+1); + } + nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; + recoverReset(p, pStmt); + + pNew = recoverMalloc(p, nByte); + if( pNew ){ + int i = 0; + char *csr = 0; + pNew->aCol = (RecoverColumn*)&pNew[1]; + pNew->zTab = csr = (char*)&pNew->aCol[nCol]; + pNew->nCol = nCol; + pNew->iRoot = iRoot; + pNew->iPk = -1; + memcpy(csr, zName, nName); + csr += nName+1; + + for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int bPk = sqlite3_column_int(pStmt, 5); + int n = sqlite3_column_bytes(pStmt, 1); + const char *z = (const char*)sqlite3_column_text(pStmt, 1); + int eHidden = sqlite3_column_int(pStmt, 6); + + if( bPk ) pNew->iPk = i; + pNew->aCol[i].zCol = csr; + pNew->aCol[i].eHidden = eHidden; + memcpy(csr, z, n); + csr += (n+1); + } + + pNew->pNext = p->pTblList; + p->pTblList = pNew; + } + + recoverFinalize(p, pStmt); + + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_info(%Q)", zName); + if( pStmt && sqlite3_step(pStmt)!=SQLITE_ROW ){ + pNew->bIntkey = 1; + }else{ + pNew->iPk = -1; + } + recoverFinalize(p, pStmt); + } +} + +/* +** +*/ +static int recoverWriteSchema1(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + sqlite3_stmt *pTblname = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "SELECT rootpage, sql, type='table' FROM recovery.schema " + " WHERE type='table' OR (type='index' AND sql LIKE '%unique%')" + ); + + pTblname = recoverPrepare(p, p->dbOut, + "SELECT name FROM sqlite_schema " + "WHERE type='table' ORDER BY rowid DESC LIMIT 1" + ); + + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int bTable = sqlite3_column_int(pSelect, 2); + + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + if( bTable ){ + if( SQLITE_ROW==sqlite3_step(pTblname) ){ + const char *zName = sqlite3_column_text(pTblname, 0); + recoverAddTable(p, zName, iRoot); + } + recoverReset(p, pTblname); + } + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } + } + recoverFinalize(p, pSelect); + recoverFinalize(p, pTblname); + + return p->errCode; +} + +static int recoverWriteSchema2(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + ); + + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc!=SQLITE_OK && rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } + } + recoverFinalize(p, pSelect); + + return p->errCode; +} + + +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ + char *zRet = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( zRet==0 ){ + p->errCode = SQLITE_NOMEM; + } + } + return zRet; +} + +static sqlite3_stmt *recoverInsertStmt( + sqlite3_recover *p, + RecoverTable *pTab, + int nField +){ + const char *zSep = ""; + char *zSql = 0; + char *zBind = 0; + int ii; + sqlite3_stmt *pRet = 0; + + assert( nField<=pTab->nCol ); + + zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); + for(ii=0; iiaCol[ii].eHidden; + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); + zBind = recoverMPrintf(p, "%z%s?", zBind, zSep); + zSep = ", "; + } + } + zSql = recoverMPrintf(p, "%z) VALUES (%z)", zSql, zBind); + + pRet = recoverPrepare(p, p->dbOut, zSql); + sqlite3_free(zSql); + + return pRet; +} + + +static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ + RecoverTable *pRet = 0; + for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); + return pRet; +} + +static int recoverWriteData(sqlite3_recover *p){ + RecoverTable *pTbl; + int nMax = 0; + sqlite3_value **apVal = 0; + sqlite3_stmt *pSel = 0; + + /* Figure out the maximum number of columns for any table in the schema */ + for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ + if( pTbl->nCol>nMax ) nMax = pTbl->nCol; + } + + apVal = (sqlite3_value**)recoverMalloc(p, sizeof(sqlite3_value*) * nMax); + if( apVal==0 ) return p->errCode; + + pSel = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE pages(root, page) AS (" + " SELECT rootpage, rootpage FROM recovery.schema" + " UNION" + " SELECT root, child FROM recovery.dbptr, pages WHERE pgno=page" + ") " + "SELECT root, page, cell, field, value " + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " + "UNION ALL " + "SELECT 0, 0, 0, 0, 0" + ); + if( pSel ){ + RecoverTable *pTab = 0; + sqlite3_stmt *pInsert = 0; + int nInsert = -1; + i64 iPrevRoot = -1; + i64 iPrevPage = -1; + int iPrevCell = -1; + i64 iRowid = 0; + int nVal = -1; + + while( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSel, 0); + i64 iPage = sqlite3_column_int64(pSel, 1); + int iCell = sqlite3_column_int(pSel, 2); + int iField = sqlite3_column_int(pSel, 3); + sqlite3_value *pVal = sqlite3_column_value(pSel, 4); + + int bNewCell = (iPrevRoot!=iRoot || iPrevPage!=iPage || iPrevCell!=iCell); + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==nVal ); + + if( bNewCell ){ + if( nVal>=0 ){ + int ii; + + if( pTab ){ + int iVal = 0; + int iBind = 1; + + if( pInsert==0 || nVal!=nInsert ){ + recoverFinalize(p, pInsert); + pInsert = recoverInsertStmt(p, pTab, nVal); + nInsert = nVal; + } + + for(ii=0; iinCol && iValaCol[ii].eHidden; + switch( eHidden ){ + case RECOVER_EHIDDEN_NONE: + case RECOVER_EHIDDEN_HIDDEN: + if( ii==pTab->iPk ){ + sqlite3_bind_int64(pInsert, iBind, iRowid); + }else{ + sqlite3_bind_value(pInsert, iBind, apVal[iVal]); + } + iBind++; + iVal++; + break; + + case RECOVER_EHIDDEN_VIRTUAL: + break; + + case RECOVER_EHIDDEN_STORED: + iVal++; + break; + } + } + + sqlite3_step(pInsert); + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); + } + + for(ii=0; iierrCode; +} + +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +){ + sqlite3_recover *pRet = 0; + int nDb = 0; + int nUri = 0; + int nByte = 0; + + if( zDb==0 ){ zDb = "main"; } + if( zUri==0 ){ zUri = ""; } + + nDb = recoverStrlen(zDb); + nUri = recoverStrlen(zUri); + + nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; + pRet = (sqlite3_recover*)sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->dbIn = db; + pRet->zDb = (char*)&pRet[1]; + pRet->zUri = &pRet->zDb[nDb+1]; + memcpy(pRet->zDb, zDb, nDb); + memcpy(pRet->zUri, zUri, nUri); + } + + return pRet; +} + +const char *sqlite3_recover_errmsg(sqlite3_recover *p){ + return p ? p->zErrMsg : "not an error"; +} +int sqlite3_recover_errcode(sqlite3_recover *p){ + return p ? p->errCode : SQLITE_NOMEM; +} + +int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ + int rc = SQLITE_OK; + + switch( op ){ + case SQLITE_RECOVER_TESTDB: + sqlite3_free(p->zStateDb); + p->zStateDb = sqlite3_mprintf("%s", (char*)pArg); + break; + + default: + rc = SQLITE_NOTFOUND; + break; + } + + return rc; +} + +static void recoverStep(sqlite3_recover *p){ + + assert( p->errCode==SQLITE_OK ); + + if( p->dbOut==0 ){ + if( recoverOpenOutput(p) ) return; + if( recoverCacheDbptr(p) ) return; + if( recoverCacheSchema(p) ) return; + if( recoverWriteSchema1(p) ) return; + if( recoverWriteData(p) ) return; + if( recoverWriteSchema2(p) ) return; + } +} + +int sqlite3_recover_step(sqlite3_recover *p){ + if( p && p->errCode==SQLITE_OK ){ + recoverStep(p); + } + return p ? p->errCode : SQLITE_NOMEM; +} + +int sqlite3_recover_finish(sqlite3_recover *p){ + RecoverTable *pTab; + RecoverTable *pNext; + int rc; + + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + + sqlite3_finalize(p->pGetPage); + rc = sqlite3_close(p->dbOut); + assert( rc==SQLITE_OK ); + p->pGetPage = 0; + rc = p->errCode; + + sqlite3_free(p); + return rc; +} + diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h new file mode 100644 index 0000000000..401f83ea28 --- /dev/null +++ b/ext/recover/sqlite3recover.h @@ -0,0 +1,66 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + + +#ifndef _SQLITE_RECOVER_H +#define _SQLITE_RECOVER_H + +#include "sqlite3.h" /* Required for error code definitions */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sqlite3_recover sqlite3_recover; + +/* Create an object to recover data from database zDb (e.g. "main") +** opened by handle db. Data will be recovered into the database +** identified by parameter zUri. Database zUri is clobbered if it +** already exists. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +); + +/* Details TBD. */ +int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); + +#define SQLITE_RECOVER_TESTDB 789 + +/* Step the recovery object. Return SQLITE_DONE if recovery is complete, +** SQLITE_OK if recovery is not complete but no error has occurred, or +** an SQLite error code if an error has occurred. +*/ +int sqlite3_recover_step(sqlite3_recover*); + +const char *sqlite3_recover_errmsg(sqlite3_recover*); + +int sqlite3_recover_errcode(sqlite3_recover*); + +/* Clean up a recovery object created by a call to sqlite3_recover_init(). +** This function returns SQLITE_DONE if the new database was created, +** SQLITE_OK if it processing was abandoned before it as finished or +** an SQLite error code (e.g. SQLITE_IOERR, SQLITE_NOMEM etc.) if an +** error occurred. */ +int sqlite3_recover_finish(sqlite3_recover*); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_RECOVER_H */ + diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c new file mode 100644 index 0000000000..912b8dec5c --- /dev/null +++ b/ext/recover/test_recover.c @@ -0,0 +1,185 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + +#include "sqlite3recover.h" + +#include +#include + +typedef struct TestRecover TestRecover; +struct TestRecover { + sqlite3_recover *p; +}; + +static int getDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ + Tcl_CmdInfo info; + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + return TCL_ERROR; + } + *pDb = *(sqlite3 **)info.objClientData; + return TCL_OK; +} + +/* +** Implementation of the command created by [sqlite3_recover_init]: +** +** $cmd config OP ARG +** $cmd step +** $cmd errmsg +** $cmd errcode +** $cmd finalize +*/ +static int testRecoverCmd( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static struct RecoverSub { + const char *zSub; + int nArg; + const char *zMsg; + } aSub[] = { + { "config", 2, "REBASE-BLOB" }, /* 0 */ + { "step", 0, "" }, /* 1 */ + { "errmsg", 0, "" }, /* 2 */ + { "errcode", 0, "" }, /* 3 */ + { "finish", 0, "" }, /* 4 */ + { 0 } + }; + int rc = TCL_OK; + int iSub = 0; + TestRecover *pTest = (TestRecover*)clientData; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ..."); + return TCL_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( (objc-2)!=aSub[iSub].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + + switch( iSub ){ + case 0: assert( sqlite3_stricmp("config", aSub[iSub].zSub)==0 ); { + const char *aOp[] = { + "testdb", /* 0 */ + 0 + }; + int iOp = 0; + int res = 0; + if( Tcl_GetIndexFromObj(interp, objv[2], aOp, "option", 0, &iOp) ){ + return TCL_ERROR; + } + switch( iOp ){ + case 0: + res = sqlite3_recover_config( + pTest->p, SQLITE_RECOVER_TESTDB, (void*)Tcl_GetString(objv[3]) + ); + break; + } + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } + case 1: assert( sqlite3_stricmp("step", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_step(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } + case 2: assert( sqlite3_stricmp("errmsg", aSub[iSub].zSub)==0 ); { + const char *zErr = sqlite3_recover_errmsg(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + break; + } + case 3: assert( sqlite3_stricmp("errcode", aSub[iSub].zSub)==0 ); { + int errCode = sqlite3_recover_errcode(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(errCode)); + break; + } + case 4: assert( sqlite3_stricmp("finish", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_errcode(pTest->p); + int res2; + if( res!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(pTest->p); + char *zRes = sqlite3_mprintf("(%d) - %s", res, zErr); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zRes, -1)); + sqlite3_free(zRes); + } + res2 = sqlite3_recover_finish(pTest->p); + assert( res2==res ); + if( res ) return TCL_ERROR; + break; + } + } + + return TCL_OK; +} + +/* +** sqlite3_recover_init DB DBNAME URI +*/ +static int test_sqlite3_recover_init( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static int iTestRecoverCmd = 1; + + TestRecover *pNew = 0; + sqlite3 *db = 0; + const char *zDb = 0; + const char *zUri = 0; + char zCmd[128]; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME URI"); + return TCL_ERROR; + } + if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; + zDb = Tcl_GetString(objv[2]); + zUri = Tcl_GetString(objv[3]); + + pNew = ckalloc(sizeof(TestRecover)); + pNew->p = sqlite3_recover_init(db, zDb, zUri); + + sprintf(zCmd, "sqlite_recover%d", iTestRecoverCmd++); + Tcl_CreateObjCommand(interp, zCmd, testRecoverCmd, (void*)pNew, 0); + + Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1)); + return TCL_OK; +} + +int TestRecover_Init(Tcl_Interp *interp){ + struct Cmd { + const char *zCmd; + Tcl_ObjCmdProc *xProc; + } aCmd[] = { + { "sqlite3_recover_init", test_sqlite3_recover_init }, + }; + int i; + + for(i=0; izCmd, p->xProc, 0, 0); + } + + return TCL_OK; +} + diff --git a/main.mk b/main.mk index 3d8a07494d..5263546042 100644 --- a/main.mk +++ b/main.mk @@ -444,6 +444,9 @@ TESTSRC2 = \ $(TOP)/ext/misc/stmt.c \ $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/session/test_session.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/test_recover.c \ fts5.c # Header files used by all library source files. diff --git a/manifest b/manifest index 746bbe3c38..0fab53a46f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sb-tree\spage\ssorting\scode\sto\sensure\sthat\ssqlite3PagerRekey()\snever\noverloads\sa\spage\snumber\sand\suses\sonly\sthe\sPENDING_BYTE\spage\sfor\stemporary\nstorage. -D 2022-08-31T15:04:42.204 +C Add\snew\sfiles\sfor\san\sextension\sto\srecover\sdata\sfrom\scorrupted\sdatabases. +D 2022-08-31T20:45:43.730 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -299,7 +299,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 -F ext/misc/dbdata.c e316fba936571584e55abd5b974a32a191727a6b746053a0c9d439bd2cf93940 +F ext/misc/dbdata.c f317980cea788e67932828b94a16ee8a8b859e3c2d62859d09ba3d5ca85f87cb F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 @@ -387,6 +387,11 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a +F ext/recover/recover1.test 861ad5140566102a8c5a3d1f936a7d6da569f34c86597c274de695f597031bac +F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c +F ext/recover/sqlite3recover.c 594fb45777a14f0b88b944b9fb2ccb3e85a29ef5b17522b8dac3e3944c4c27ea +F ext/recover/sqlite3recover.h 3255f6491007e57be310aedb72a848c88f79fc14e7222bda4b8d4dab1a2450c3 +F ext/recover/test_recover.c 919f61df54776598b350250057fd2d3ea9cc2cef1aeac0dbb760958d26fe1afb F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -509,7 +514,7 @@ F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e4 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 20801eed419dc58936ff9449b04041edbbbc0488a9fc683e72471dded050e0bb +F main.mk 8c9965c408aaa8b93d0dd52e83445894835e1a42dc360c77435393f80f8d8d1d F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -641,7 +646,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 -F src/test_tclsh.c c4065ced25126e25c40122c5ff62dc89902ea617d72cdd27765151cdd7fcc477 +F src/test_tclsh.c 7dd98be675a1dc0d1fd302b8247bab992c909db384df054381a2279ad76f9b0e F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 269ea9e1fa5828dba550eb26f619aa18aedbc29fd92f8a5f6b93521fbb74a61c F src/test_vdbecov.c f60c6f135ec42c0de013a1d5136777aa328a776d33277f92abac648930453d43 @@ -1999,8 +2004,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dd017bb1b3e31c7692d29dc4865d6bda871e429978c8738a39160d0114e5bf9b -R 98122ff0aaf5ca87deda59c5c8a25251 -U drh -Z 8d73d18db9ab73a94a9689d17f937c1d +P 5007742886bd20de20be3973737cf46b010359911615eb3da69cd262bd9a2435 +R 563b8320bf923831e4768bc403655fc2 +T *branch * recover-extension +T *sym-recover-extension * +T -sym-trunk * +U dan +Z 1c7612740eb933f84d589533d182c6df # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 00cb077051..16703e1b2f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5007742886bd20de20be3973737cf46b010359911615eb3da69cd262bd9a2435 \ No newline at end of file +f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 \ No newline at end of file diff --git a/src/test_tclsh.c b/src/test_tclsh.c index 707c16812c..c133deca25 100644 --- a/src/test_tclsh.c +++ b/src/test_tclsh.c @@ -108,6 +108,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ extern int TestExpert_Init(Tcl_Interp*); extern int Sqlitetest_window_Init(Tcl_Interp *); extern int Sqlitetestvdbecov_Init(Tcl_Interp *); + extern int TestRecover_Init(Tcl_Interp*); Tcl_CmdInfo cmdInfo; @@ -175,6 +176,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ TestExpert_Init(interp); Sqlitetest_window_Init(interp); Sqlitetestvdbecov_Init(interp); + TestRecover_Init(interp); Tcl_CreateObjCommand( interp, "load_testfixture_extensions", load_testfixture_extensions,0,0