diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 8a29304922..7eca9b1321 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6278,6 +6278,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ if( 0==fts5MultiIterIsEmpty(p, pIter) ){ diff --git a/ext/rbu/rburename.test b/ext/rbu/rburename.test new file mode 100644 index 0000000000..2275396bca --- /dev/null +++ b/ext/rbu/rburename.test @@ -0,0 +1,54 @@ +# 2022 November 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rburename + + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); +} + +forcedelete test.db-vacuum + +proc my_rename {old new} { + lappend ::my_rename_calls [list [file tail $old] [file tail $new]] + file rename $old $new +} + +do_test 1.1 { + sqlite3rbu_vacuum rbu test.db + rbu rename_handler my_rename + while {[rbu step]=="SQLITE_OK"} {} + rbu close +} SQLITE_DONE + +do_test 1.2 { + set ::my_rename_calls +} {{test.db-oal test.db-wal}} + +proc my_rename {old new} { + error "something went wrong" +} + +do_test 1.3 { + sqlite3rbu_vacuum rbu test.db + rbu rename_handler my_rename + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 SQLITE_IOERR} + +finish_test diff --git a/ext/rbu/rbuvacuum2.test b/ext/rbu/rbuvacuum2.test index c8fba6a22d..d4f7c52328 100644 --- a/ext/rbu/rbuvacuum2.test +++ b/ext/rbu/rbuvacuum2.test @@ -227,10 +227,11 @@ do_test 6.1 { rbu close } {SQLITE_OK} -do_execsql_test 6.2 { - SELECT 1 FROM sqlite_master LIMIT 1; - PRAGMA wal_checkpoint; -} {1 0 4 4} +do_test 6.2 { + execsql { SELECT 1 FROM sqlite_master LIMIT 1 } + execsql { PRAGMA wal_checkpoint } + execsql { SELECT 1 FROM sqlite_master LIMIT 1 } +} {1} do_test 6.3 { sqlite3rbu_vacuum rbu test.db test.db2 diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 27a3720daa..482952006e 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -393,6 +393,8 @@ struct sqlite3rbu { int nPagePerSector; /* Pages per sector for pTargetFd */ i64 iOalSz; i64 nPhaseOneStep; + void *pRenameArg; + int (*xRename)(void*, const char*, const char*); /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -2781,7 +2783,7 @@ static void rbuOpenDatabase(sqlite3rbu *p, sqlite3 *dbMain, int *pbRetry){ sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); if( p->zState==0 ){ const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); - p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + p->zState = rbuMPrintf(p, "file:///%s-vacuum?modeof=%s", zFile, zFile); } } @@ -3241,32 +3243,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){ } if( p->rc==SQLITE_OK ){ -#if defined(_WIN32_WCE) - { - LPWSTR zWideOal; - LPWSTR zWideWal; - - zWideOal = rbuWinUtf8ToUnicode(zOal); - if( zWideOal ){ - zWideWal = rbuWinUtf8ToUnicode(zWal); - if( zWideWal ){ - if( MoveFileW(zWideOal, zWideWal) ){ - p->rc = SQLITE_OK; - }else{ - p->rc = SQLITE_IOERR; - } - sqlite3_free(zWideWal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - sqlite3_free(zWideOal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - } -#else - p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK; -#endif + p->rc = p->xRename(p->pRenameArg, zOal, zWal); } if( p->rc!=SQLITE_OK @@ -4005,6 +3982,7 @@ static sqlite3rbu *openRbuHandle( /* Create the custom VFS. */ memset(p, 0, sizeof(sqlite3rbu)); + sqlite3rbu_rename_handler(p, 0, 0); rbuCreateVfs(p); /* Open the target, RBU and state databases */ @@ -4396,6 +4374,54 @@ int sqlite3rbu_savestate(sqlite3rbu *p){ return rc; } +/* +** Default xRename callback for RBU. +*/ +static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){ + int rc = SQLITE_OK; +#if defined(_WIN32_WCE) + { + LPWSTR zWideOld; + LPWSTR zWideNew; + + zWideOld = rbuWinUtf8ToUnicode(zOld); + if( zWideOld ){ + zWideNew = rbuWinUtf8ToUnicode(zNew); + if( zWideNew ){ + if( MoveFileW(zWideOld, zWideNew) ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_IOERR; + } + sqlite3_free(zWideNew); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + sqlite3_free(zWideOld); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + } +#else + rc = rename(zOld, zNew) ? SQLITE_IOERR : SQLITE_OK; +#endif + return rc; +} + +void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +){ + if( xRename ){ + pRbu->xRename = xRename; + pRbu->pRenameArg = pArg; + }else{ + pRbu->xRename = xDefaultRename; + pRbu->pRenameArg = 0; + } +} + /************************************************************************** ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour ** of a standard VFS in the following ways: diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index 69d89500a0..c819cd3f2a 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -544,6 +544,34 @@ SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int*pnTwo); SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); +/* +** As part of applying an RBU update or performing an RBU vacuum operation, +** the system must at one point move the *-oal file to the equivalent *-wal +** path. Normally, it does this by invoking POSIX function rename(2) directly. +** Except on WINCE platforms, where it uses win32 API MoveFileW(). This +** function may be used to register a callback that the RBU module will invoke +** instead of one of these APIs. +** +** If a callback is registered with an RBU handle, it invokes it instead +** of rename(2) when it needs to move a file within the file-system. The +** first argument passed to the xRename() callback is a copy of the second +** argument (pArg) passed to this function. The second is the full path +** to the file to move and the third the full path to which it should be +** moved. The callback function should return SQLITE_OK to indicate +** success. If an error occurs, it should return an SQLite error code. +** In this case the RBU operation will be abandoned and the error returned +** to the RBU user. +** +** Passing a NULL pointer in place of the xRename argument to this function +** restores the default behaviour. +*/ +SQLITE_API void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +); + + /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 6d04bfe8cc..af794d80f8 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -26,6 +26,14 @@ # endif #endif #include +#include + +typedef struct TestRbu TestRbu; +struct TestRbu { + sqlite3rbu *pRbu; + Tcl_Interp *interp; + Tcl_Obj *xRename; +}; /* From main.c */ extern const char *sqlite3ErrName(int); @@ -55,6 +63,20 @@ void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){ Tcl_DecrRefCount(pScript); } +static int xRenameCallback(void *pArg, const char *zOld, const char *zNew){ + int rc = SQLITE_OK; + TestRbu *pTest = (TestRbu*)pArg; + Tcl_Obj *pEval = Tcl_DuplicateObj(pTest->xRename); + + Tcl_IncrRefCount(pEval); + Tcl_ListObjAppendElement(pTest->interp, pEval, Tcl_NewStringObj(zOld, -1)); + Tcl_ListObjAppendElement(pTest->interp, pEval, Tcl_NewStringObj(zNew, -1)); + + rc = Tcl_EvalObjEx(pTest->interp, pEval, TCL_GLOBAL_ONLY); + Tcl_DecrRefCount(pEval); + + return rc ? SQLITE_IOERR : SQLITE_OK; +} static int SQLITE_TCLAPI test_sqlite3rbu_cmd( ClientData clientData, @@ -63,7 +85,8 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd( Tcl_Obj *CONST objv[] ){ int ret = TCL_OK; - sqlite3rbu *pRbu = (sqlite3rbu*)clientData; + TestRbu *pTest = (TestRbu*)clientData; + sqlite3rbu *pRbu = pTest->pRbu; struct RbuCmd { const char *zName; int nArg; @@ -82,6 +105,7 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd( {"temp_size_limit", 3, "LIMIT"}, /* 10 */ {"temp_size", 2, ""}, /* 11 */ {"dbRbu_eval", 3, "SQL"}, /* 12 */ + {"rename_handler", 3, "SCRIPT"},/* 13 */ {0,0,0} }; int iCmd; @@ -127,6 +151,8 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd( } ret = TCL_ERROR; } + if( pTest->xRename ) Tcl_DecrRefCount(pTest->xRename); + ckfree(pTest); break; } @@ -214,6 +240,19 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd( break; } + case 13: /* rename_handler */ { + Tcl_Obj *pScript = objv[2]; + assert( !sqlite3_stricmp(aCmd[13].zName, "rename_handler") ); + if( Tcl_GetCharLength(pScript)==0 ){ + sqlite3rbu_rename_handler(pRbu, 0, 0); + }else{ + pTest->xRename = Tcl_DuplicateObj(pScript); + Tcl_IncrRefCount(pTest->xRename); + sqlite3rbu_rename_handler(pRbu, pTest, xRenameCallback); + } + break; + } + default: /* seems unlikely */ assert( !"cannot happen" ); break; @@ -222,6 +261,18 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd( return ret; } +static void createRbuWrapper( + Tcl_Interp *interp, + const char *zCmd, + sqlite3rbu *pRbu +){ + TestRbu *pTest = (TestRbu*)ckalloc(sizeof(TestRbu)); + memset(pTest, 0, sizeof(TestRbu)); + pTest->pRbu = pRbu; + pTest->interp = interp; + Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pTest, 0); +} + /* ** Tclcmd: sqlite3rbu CMD ?? */ @@ -247,7 +298,7 @@ static int SQLITE_TCLAPI test_sqlite3rbu( if( objc==5 ) zStateDb = Tcl_GetString(objv[4]); pRbu = sqlite3rbu_open(zTarget, zRbu, zStateDb); - Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); + createRbuWrapper(interp, zCmd, pRbu); Tcl_SetObjResult(interp, objv[1]); return TCL_OK; } @@ -276,7 +327,7 @@ static int SQLITE_TCLAPI test_sqlite3rbu_vacuum( if( zStateDb && zStateDb[0]=='\0' ) zStateDb = 0; pRbu = sqlite3rbu_vacuum(zTarget, zStateDb); - Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); + createRbuWrapper(interp, zCmd, pRbu); Tcl_SetObjResult(interp, objv[1]); return TCL_OK; } diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index dca468f7b8..9563ab502a 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -71,6 +71,7 @@ ** It contains one entry for each b-tree pointer between a parent and ** child page in the database. */ + #if !defined(SQLITEINT_H) #include "sqlite3ext.h" @@ -82,6 +83,8 @@ SQLITE_EXTENSION_INIT1 #include #include +#ifndef SQLITE_OMIT_VIRTUALTABLE + #define DBDATA_PADDING_BYTES 100 typedef struct DbdataTable DbdataTable; @@ -425,7 +428,7 @@ static void dbdataValue( u32 enc, int eType, u8 *pData, - int nData + sqlite3_int64 nData ){ if( eType>=0 && dbdataValueBytes(eType)<=nData ){ switch( eType ){ @@ -861,7 +864,7 @@ static int dbdataColumn( case DBDATA_COLUMN_VALUE: { if( pCsr->iField<0 ){ sqlite3_result_int64(ctx, pCsr->iIntkey); - }else{ + }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){ sqlite3_int64 iType; dbdataGetVarintU32(pCsr->pHdrPtr, &iType); dbdataValue( @@ -935,3 +938,5 @@ int sqlite3_dbdata_init( SQLITE_EXTENSION_INIT2(pApi); return sqlite3DbdataRegister(db); } + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 94bcd348a7..75f5dba1ff 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -10,16 +10,9 @@ #*********************************************************************** # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl - set testprefix recover1 - - proc compare_result {db1 db2 sql} { set r1 [$db1 eval $sql] set r2 [$db2 eval $sql] @@ -271,11 +264,13 @@ do_recover_test 14 #------------------------------------------------------------------------- reset_db -do_execsql_test 15.1 { +execsql { PRAGMA journal_mode=OFF; PRAGMA mmap_size=10; +} +do_execsql_test 15.1 { CREATE TABLE t1(x); -} {off 10} +} {} do_recover_test 15 finish_test diff --git a/ext/recover/recover_common.tcl b/ext/recover/recover_common.tcl index 3f2ff2d6cc..fdf735ee75 100644 --- a/ext/recover/recover_common.tcl +++ b/ext/recover/recover_common.tcl @@ -1,5 +1,14 @@ - - + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl + +if {[info commands sqlite3_recover_init]==""} { + finish_test + return -code return +} diff --git a/ext/recover/recoverclobber.test b/ext/recover/recoverclobber.test index 537af8e7c1..e096b2e216 100644 --- a/ext/recover/recoverclobber.test +++ b/ext/recover/recoverclobber.test @@ -12,17 +12,9 @@ # Tests for the SQLITE_RECOVER_ROWIDS option. # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl set testprefix recoverclobber -ifcapable !vtab { - finish_test; return -} - proc recover {db output} { set R [sqlite3_recover_init db main test.db2] $R run diff --git a/ext/recover/recovercorrupt.test b/ext/recover/recovercorrupt.test index 40859f3d1c..eb6fe53add 100644 --- a/ext/recover/recovercorrupt.test +++ b/ext/recover/recovercorrupt.test @@ -10,12 +10,7 @@ #*********************************************************************** # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl - set testprefix recovercorrupt database_may_be_corrupt diff --git a/ext/recover/recovercorrupt2.test b/ext/recover/recovercorrupt2.test index 20d4c39bcd..7147c67e93 100644 --- a/ext/recover/recovercorrupt2.test +++ b/ext/recover/recovercorrupt2.test @@ -10,12 +10,7 @@ #*********************************************************************** # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl - set testprefix recovercorrupt2 do_execsql_test 1.0 { diff --git a/ext/recover/recoverfault.test b/ext/recover/recoverfault.test index 2ea87860b3..30bb65527d 100644 --- a/ext/recover/recoverfault.test +++ b/ext/recover/recoverfault.test @@ -10,12 +10,7 @@ #*********************************************************************** # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl - set testprefix recoverfault diff --git a/ext/recover/recoverfault2.test b/ext/recover/recoverfault2.test index 4f7131ecfb..e80d480ce1 100644 --- a/ext/recover/recoverfault2.test +++ b/ext/recover/recoverfault2.test @@ -10,12 +10,7 @@ #*********************************************************************** # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl - set testprefix recoverfault2 diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test index 691737bdc5..c6acbb2f42 100644 --- a/ext/recover/recoverold.test +++ b/ext/recover/recoverold.test @@ -11,17 +11,9 @@ # # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl set testprefix recoverold -ifcapable !vtab { - finish_test; return -} - proc compare_result {db1 db2 sql} { set r1 [$db1 eval $sql] set r2 [$db2 eval $sql] @@ -71,7 +63,10 @@ proc do_recover_test {tn {tsql {}} {res {}}} { sqlite3 db2 test.db2 db2 eval [join $::sqlhook ";"] + + db cache flush if {$tsql==""} { + compare_dbs db db2 uplevel [list do_test $tn.sql [list compare_dbs db db2] {}] } else { uplevel [list do_execsql_test -db db2 $tn.sql $tsql $res] @@ -155,7 +150,6 @@ do_recover_test 2.4.1 { 2 2 3 {} 8 9 7 } -breakpoint do_execsql_test 2.5 { CREATE TABLE x1(a, b, c); WITH s(i) AS ( @@ -173,17 +167,19 @@ do_recover_test 2.5.1 { 2 2 3 {} 8 9 7 } -do_test 2.6 { - forcedelete test.db2 - set R [sqlite3_recover_init db main test.db2] - $R config lostandfound lost_and_found - $R config freelistcorrupt 1 - $R run - $R finish - sqlite3 db2 test.db2 - execsql { SELECT count(*) FROM lost_and_found_1; } db2 -} {103} -db2 close +ifcapable !secure_delete { + do_test 2.6 { + forcedelete test.db2 + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R config freelistcorrupt 1 + $R run + $R finish + sqlite3 db2 test.db2 + execsql { SELECT count(*) FROM lost_and_found_1; } db2 + } {103} + db2 close +} #------------------------------------------------------------------------- breakpoint diff --git a/ext/recover/recoverpgsz.test b/ext/recover/recoverpgsz.test index 71413b0ead..1a91f08459 100644 --- a/ext/recover/recoverpgsz.test +++ b/ext/recover/recoverpgsz.test @@ -10,11 +10,7 @@ #*********************************************************************** # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl db close sqlite3_test_control_pending_byte 0x1000000 @@ -27,6 +23,7 @@ foreach {pgsz bOverflow} { } { reset_db execsql "PRAGMA page_size = $pgsz" + execsql "PRAGMA auto_vacuum = 0" do_execsql_test 1.$pgsz.$bOverflow.1 { CREATE TABLE t1(a, b, c); CREATE INDEX i1 ON t1(b, a, c); diff --git a/ext/recover/recoverrowid.test b/ext/recover/recoverrowid.test index bd47422eaf..5855e84fa6 100644 --- a/ext/recover/recoverrowid.test +++ b/ext/recover/recoverrowid.test @@ -12,17 +12,9 @@ # Tests for the SQLITE_RECOVER_ROWIDS option. # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl set testprefix recoverrowid -ifcapable !vtab { - finish_test; return -} - proc recover {db bRowids output} { forcedelete $output diff --git a/ext/recover/recoverslowidx.test b/ext/recover/recoverslowidx.test index 4dfb7e5f53..269105113d 100644 --- a/ext/recover/recoverslowidx.test +++ b/ext/recover/recoverslowidx.test @@ -12,18 +12,11 @@ # Tests for the SQLITE_RECOVER_SLOWINDEXES option. # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl set testprefix recoverslowidx -ifcapable !vtab { - finish_test; return -} - do_execsql_test 1.0 { + PRAGMA auto_vacuum = 0; CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(a); INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3), (4, 4); diff --git a/ext/recover/recoversql.test b/ext/recover/recoversql.test index 19f7ec7a1c..0a6726727d 100644 --- a/ext/recover/recoversql.test +++ b/ext/recover/recoversql.test @@ -11,17 +11,9 @@ # # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} source [file join [file dirname [info script]] recover_common.tcl] -source $testdir/tester.tcl set testprefix recoversql -ifcapable !vtab { - finish_test; return -} - do_execsql_test 1.0 { CREATE TABLE "x.1" (x, y); INSERT INTO "x.1" VALUES(1, 1), (2, 2), (3, 3); diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 2952274ebb..30260f014e 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -17,6 +17,8 @@ #include #include +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* ** Declaration for public API function in file dbdata.c. This may be called ** with NULL as the final two arguments to register the sqlite_dbptr and @@ -280,10 +282,16 @@ static RecoverGlobal recover_g; */ #define RECOVER_ROWID_DEFAULT 1 +/* +** Mutex handling: +** +** recoverEnterMutex() - Enter the recovery mutex +** recoverLeaveMutex() - Leave the recovery mutex +** recoverAssertMutexHeld() - Assert that the recovery mutex is held +*/ #if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 # define recoverEnterMutex() # define recoverLeaveMutex() -# define recoverAssertMutexHeld() #else static void recoverEnterMutex(void){ sqlite3_mutex_enter(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); @@ -291,9 +299,13 @@ static void recoverEnterMutex(void){ static void recoverLeaveMutex(void){ sqlite3_mutex_leave(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); } +#endif +#if SQLITE_THREADSAFE+0>=1 && defined(SQLITE_DEBUG) static void recoverAssertMutexHeld(void){ assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) ); } +#else +# define recoverAssertMutexHeld() #endif @@ -301,11 +313,8 @@ static void recoverAssertMutexHeld(void){ ** Like strlen(). But handles NULL pointer arguments. */ static int recoverStrlen(const char *zStr){ - int nRet = 0; - if( zStr ){ - while( zStr[nRet] ) nRet++; - } - return nRet; + if( zStr==0 ) return 0; + return (int)(strlen(zStr)&0x7fffffff); } /* @@ -2844,3 +2853,5 @@ int sqlite3_recover_finish(sqlite3_recover *p){ } return rc; } + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index 99c7aeca34..1c333df8e0 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -18,6 +18,8 @@ #include #include +#ifndef SQLITE_OMIT_VIRTUALTABLE + typedef struct TestRecover TestRecover; struct TestRecover { sqlite3_recover *p; @@ -284,9 +286,10 @@ static int test_sqlite3_dbdata_init( return TCL_OK; } - +#endif /* SQLITE_OMIT_VIRTUALTABLE */ int TestRecover_Init(Tcl_Interp *interp){ +#ifndef SQLITE_OMIT_VIRTUALTABLE struct Cmd { const char *zCmd; Tcl_ObjCmdProc *xProc; @@ -302,7 +305,7 @@ int TestRecover_Init(Tcl_Interp *interp){ struct Cmd *p = &aCmd[i]; Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, p->pArg, 0); } - +#endif return TCL_OK; } diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 069e2a1da9..5fd0f073c1 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -110,7 +110,7 @@ emcc.WASM_BIGINT ?= 1 sqlite3.c := $(dir.top)/sqlite3.c sqlite3.h := $(dir.top)/sqlite3.h SQLITE_OPT = \ - -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ -DSQLITE_ENABLE_RTREE \ -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ @@ -465,7 +465,20 @@ emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) # debugging info, _huge_. ######################################################################## -sqlite3.js := $(dir.dout)/sqlite3.js +######################################################################## +# AN EXPERIMENT: undocumented Emscripten feature: if the target file +# extension is "mjs", it defaults to ES6 module builds: +# https://github.com/emscripten-core/emscripten/issues/14383 +ifeq (,$(filter esm,$(MAKECMDGOALS))) +sqlite3.js.ext := js +else +esm.deps := $(filter-out esm,$(MAKECMDGOALS)) +esm: $(if $(esm.deps),$(esm.deps),all) +sqlite3.js.ext := mjs +endif +# /esm +######################################################################## +sqlite3.js := $(dir.dout)/sqlite3.$(sqlite3.js.ext) sqlite3.wasm := $(dir.dout)/sqlite3.wasm sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c # sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index 84b99b53a6..d933a36265 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -15,7 +15,10 @@ impls which Emscripten installs at some point in the file above this. */ - const originalInit = self.sqlite3InitModule; + const originalInit = + /*Maintenance reminde: DO NOT use `self.` here. It's correct + for non-ES6 Module cases but wrong for ES6 modules because those + resolve this symbol differently! */ sqlite3InitModule; if(!originalInit){ throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); } diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 3cb5c8a3eb..18582400eb 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -472,9 +472,11 @@ const installOpfsVfs = function callee(options){ /** Returns an array of the deserialized state stored by the most recent serialize() operation (from from this thread or the - counterpart thread), or null if the serialization buffer is empty. + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. */ - state.s11n.deserialize = function(){ + state.s11n.deserialize = function(clear=false){ ++metrics.s11n.deserialize.count; const t = performance.now(); const argc = viewU8[0]; @@ -499,6 +501,7 @@ const installOpfsVfs = function callee(options){ rc.push(v); } } + if(clear) viewU8[0] = 0; //log("deserialize:",argc, rc); metrics.s11n.deserialize.time += performance.now() - t; return rc; diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index cf44f39707..fed1c56669 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -17,22 +17,29 @@ conventions, and build process are very much under construction and will be (re)documented once they've stopped fluctuating so much. - Specific goals of this project: + Project home page: https://sqlite.org + + Documentation home page: https://sqlite.org/wasm + + Specific goals of this subproject: - Except where noted in the non-goals, provide a more-or-less feature-complete wrapper to the sqlite3 C API, insofar as WASM - feature parity with C allows for. In fact, provide at least 3 + feature parity with C allows for. In fact, provide at least 4 APIs... - 1) Bind a low-level sqlite3 API which is as close to the native - one as feasible in terms of usage. + 1) 1-to-1 bindings as exported from WASM, with no automatic + type conversions between JS and C. + + 2) A binding of (1) which provides certain JS/C type conversions + to greatly simplify its use. - 2) A higher-level API, more akin to sql.js and node.js-style + 3) A higher-level API, more akin to sql.js and node.js-style implementations. This one speaks directly to the low-level API. This API must be used from the same thread as the low-level API. - 3) A second higher-level API which speaks to the previous APIs via + 4) A second higher-level API which speaks to the previous APIs via worker messages. This one is intended for use in the main thread, with the lower-level APIs installed in a Worker thread, and talking to them via Worker messages. Because Workers are @@ -90,11 +97,13 @@ config object is only honored the first time this is called. Subsequent calls ignore the argument and return the same (configured) object which gets initialized by the first call. + This function will throw if any of the required config options are + missing. The config object properties include: - `exports`[^1]: the "exports" object for the current WASM - environment. In an Emscripten build, this should be set to + environment. In an Emscripten-based build, this should be set to `Module['asm']`. - `memory`[^1]: optional WebAssembly.Memory object, defaulting to @@ -104,7 +113,7 @@ WASM-exported memory. - `bigIntEnabled`: true if BigInt support is enabled. Defaults to - true if self.BigInt64Array is available, else false. Some APIs + true if `self.BigInt64Array` is available, else false. Some APIs will throw exceptions if called without BigInt support, as BigInt is required for marshalling C-side int64 into and out of JS. @@ -116,10 +125,12 @@ the `free(3)`-compatible routine for the WASM environment. Defaults to `"free"`. - - `wasmfsOpfsDir`[^1]: if the environment supports persistent storage, this - directory names the "mount point" for that directory. It must be prefixed - by `/` and may currently contain only a single directory-name part. Using - the root directory name is not supported by any current persistent backend. + - `wasmfsOpfsDir`[^1]: if the environment supports persistent + storage, this directory names the "mount point" for that + directory. It must be prefixed by `/` and may contain only a + single directory-name part. Using the root directory name is not + supported by any current persistent backend. This setting is + only used in WASMFS-enabled builds. [^1] = This property may optionally be a function, in which case this @@ -388,8 +399,22 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( exceptions. */ class WasmAllocError extends Error { + /** + If called with 2 arguments and the 2nd one is an object, it + behaves like the Error constructor, else it concatenates all + arguments together with a single space between each to + construct an error message string. As a special case, if + called with no arguments then it uses a default error + message. + */ constructor(...args){ - super(...args); + if(2===args.length && 'object'===typeof args){ + super(...args); + }else if(args.length){ + super(args.join(' ')); + }else{ + super("Allocation failed."); + } this.name = 'WasmAllocError'; } }; @@ -699,21 +724,33 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( API NOT throw and must instead return SQLITE_NOMEM (or equivalent, depending on the context). - That said, very few cases in the API can result in + Very few cases in the sqlite3 JS APIs can result in client-defined functions propagating exceptions via the C-style - API. Most notably, this applies ot User-defined SQL Functions - (UDFs) registered via sqlite3_create_function_v2(). For that - specific case it is recommended that all UDF creation be - funneled through a utility function and that a wrapper function - be added around the UDF which catches any exception and sets - the error state to OOM. (The overall complexity of registering - UDFs essentially requires a helper for doing so!) + API. Most notably, this applies to WASM-bound JS functions + which are created directly by clients and passed on _as WASM + function pointers_ to functions such as + sqlite3_create_function_v2(). Such bindings created + transparently by this API will automatically use wrappers which + catch exceptions and convert them to appropriate error codes. + + For cases where non-throwing allocation is required, use + sqlite3.wasm.alloc.impl(), which is direct binding of the + underlying C-level allocator. + + Design note: this function is not named "malloc" primarily + because Emscripten uses that name and we wanted to avoid any + confusion early on in this code's development, when it still + had close ties to Emscripten's glue code. */ alloc: undefined/*installed later*/, + /** The API's one single point of access to the WASM-side memory deallocator. Works like free(3) (and is likely bound to free()). + + Design note: this function is not named "free" for the same + reason that this.alloc() is not called this.malloc(). */ dealloc: undefined/*installed later*/ @@ -741,7 +778,9 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( wasm.allocFromTypedArray = function(srcTypedArray){ affirmBindableTypedArray(srcTypedArray); const pRet = wasm.alloc(srcTypedArray.byteLength || 1); - wasm.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); + wasm.heapForSize(srcTypedArray.constructor).set( + srcTypedArray.byteLength ? srcTypedArray : [0], pRet + ); return pRet; }; @@ -752,13 +791,13 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); } - wasm.alloc = function(n){ - const m = wasm.exports[keyAlloc](n); - if(!m) throw new WasmAllocError("Failed to allocate "+n+" bytes."); + wasm.alloc = function f(n){ + const m = f.impl(n); + if(!m) throw new WasmAllocError("Failed to allocate",n," bytes."); return m; }; - - wasm.dealloc = (m)=>wasm.exports[keyDealloc](m); + wasm.alloc.impl = wasm.exports[keyAlloc]; + wasm.dealloc = wasm.exports[keyDealloc]; /** Reports info about compile-time options using diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 09c56ff1df..e4657484ef 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -44,6 +44,7 @@ if(self.window === self){ this API. */ const state = Object.create(null); + /** verbose: @@ -96,13 +97,27 @@ metrics.dump = ()=>{ }; /** - Map of sqlite3_file pointers (integers) to metadata related to a - given OPFS file handles. The pointers are, in this side of the - interface, opaque file handle IDs provided by the synchronous - part of this constellation. Each value is an object with a structure - demonstrated in the xOpen() impl. + __openFiles is a map of sqlite3_file pointers (integers) to + metadata related to a given OPFS file handles. The pointers are, in + this side of the interface, opaque file handle IDs provided by the + synchronous part of this constellation. Each value is an object + with a structure demonstrated in the xOpen() impl. */ const __openFiles = Object.create(null); +/** + __autoLocks is a Set of sqlite3_file pointers (integers) which were + "auto-locked". i.e. those for which we obtained a sync access + handle without an explicit xLock() call. Such locks will be + released during db connection idle time, whereas a sync access + handle obtained via xLock(), or subsequently xLock()'d after + auto-acquisition, will not be released until xUnlock() is called. + + Maintenance reminder: if we relinquish auto-locks at the end of the + operation which acquires them, we pay a massive performance + penalty: speedtest1 benchmarks take up to 4x as long. By delaying + the lock release until idle time, the hit is negligible. +*/ +const __autoLocks = new Set(); /** Expects an OPFS file path. It gets resolved, such that ".." @@ -191,6 +206,10 @@ const getSyncHandle = async (fh)=>{ } } log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + if(!fh.xLock){ + __autoLocks.add(fh.fid); + log("Auto-locked",fh.fid,fh.filenameAbs); + } } return fh.syncHandle; }; @@ -210,10 +229,30 @@ const closeSyncHandle = async (fh)=>{ log("Closing sync handle for",fh.filenameAbs); const h = fh.syncHandle; delete fh.syncHandle; + delete fh.xLock; + __autoLocks.delete(fh.fid); return h.close(); } }; +/** + A proxy for closeSyncHandle() which is guaranteed to not throw. + + This function is part of a lock/unlock step in functions which + require a sync access handle but may be called without xLock() + having been called first. Such calls need to release that + handle to avoid locking the file for all of time. This is an + _attempt_ at reducing cross-tab contention but it may prove + to be more of a problem than a solution and may need to be + removed. +*/ +const closeSyncHandleNoThrow = async (fh)=>{ + try{await closeSyncHandle(fh)} + catch(e){ + warn("closeSyncHandleNoThrow() ignoring:",e,fh); + } +}; + /** Stores the given value at state.sabOPView[state.opIds.rc] and then Atomics.notify()'s it. @@ -342,9 +381,10 @@ const vfsAsyncImpls = { xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; mTimeStart(opName); + __autoLocks.delete(fid); const fh = __openFiles[fid]; let rc = 0; - wTimeStart('xClose'); + wTimeStart(opName); if(fh){ delete __openFiles[fid]; await closeSyncHandle(fh); @@ -422,12 +462,17 @@ const vfsAsyncImpls = { mTimeStart('xLock'); const fh = __openFiles[fid]; let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; if( !fh.syncHandle ){ wTimeStart('xLock'); - try { await getSyncHandle(fh) } - catch(e){ + try { + await getSyncHandle(fh); + __autoLocks.delete(fid); + }catch(e){ state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR_LOCK; + fh.xLock = oldLockType; } wTimeEnd(); } @@ -461,6 +506,7 @@ const vfsAsyncImpls = { */ wTimeEnd(); __openFiles[fid] = Object.assign(Object.create(null),{ + fid: fid, filenameAbs: filename, filenamePart: filenamePart, dirHandle: hDir, @@ -610,7 +656,7 @@ const initS11n = ()=>{ default: toss("Invalid type ID:",tid); } }; - state.s11n.deserialize = function(){ + state.s11n.deserialize = function(clear=false){ ++metrics.s11n.deserialize.count; const t = performance.now(); const argc = viewU8[0]; @@ -635,6 +681,7 @@ const initS11n = ()=>{ rc.push(v); } } + if(clear) viewU8[0] = 0; //log("deserialize:",argc, rc); metrics.s11n.deserialize.time += performance.now() - t; return rc; @@ -701,21 +748,30 @@ const waitLoop = async function f(){ We need to wake up periodically to give the thread a chance to do other things. */ - const waitTime = 1000; + const waitTime = 500; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( state.sabOPView, state.opIds.whichOp, 0, waitTime )){ + if(__autoLocks.size){ + /* Release all auto-locks. */ + for(const fid of __autoLocks){ + const fh = __openFiles[fid]; + await closeSyncHandleNoThrow(fh); + log("Auto-unlocked",fid,fh.filenameAbs); + } + } continue; } const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); Atomics.store(state.sabOPView, state.opIds.whichOp, 0); const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); - const args = state.s11n.deserialize() || []; - state.s11n.serialize(/* clear s11n to keep the caller from - confusing this with an exception string - written by the upcoming operation */); + const args = state.s11n.deserialize( + true /* clear s11n to keep the caller from confusing this with + an exception string written by the upcoming + operation */ + ) || []; //warn("waitLoop() whichOp =",opId, hnd, args); if(hnd.f) await hnd.f(...args); else error("Missing callback for opId",opId); diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 203e70a3e6..af5ed6bf76 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -67,7 +67,7 @@ ** larger cache benefits the larger workloads. Speed differences ** between 2x and nearly 3x have been measured with ample page cache. */ -# define SQLITE_DEFAULT_CACHE_SIZE -16777216 +# define SQLITE_DEFAULT_CACHE_SIZE -16384 #endif #if 0 && !defined(SQLITE_DEFAULT_PAGE_SIZE) /* TODO: experiment with this. */ @@ -1108,9 +1108,6 @@ int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ /** It's not enough to instantiate the backend. We have to create a mountpoint in the VFS and attach the backend to it. */ if( pOpfs && 0!=access(zMountPoint, F_OK) ){ - /* mkdir() simply hangs when called from fiddle app. Cause is - not yet determined but the hypothesis is an init-order - issue. */ /* Note that this check and is not robust but it will hypothetically suffice for the transient wasm-based virtual filesystem we're currently running in. */ diff --git a/ext/wasm/module-symbols.html b/ext/wasm/module-symbols.html index bebefffac8..427d2dc8c3 100644 --- a/ext/wasm/module-symbols.html +++ b/ext/wasm/module-symbols.html @@ -189,7 +189,7 @@
-