diff --git a/Makefile.in b/Makefile.in index d91093835a..2c2ff7910d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -35,7 +35,7 @@ TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session TCC += -I${TOP}/ext/userauth # Define this for the autoconf-based build, so that the code knows it can -# include the generated config.h +# include the generated sqlite_cfg.h # TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite @@ -184,7 +184,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memdb.lo memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ - notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ + notify.lo opcodes.lo os.lo os_kv.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \ @@ -257,6 +257,7 @@ SRC = \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ $(TOP)/src/os_setup.h \ + $(TOP)/src/os_kv.c \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ $(TOP)/src/os_win.h \ @@ -377,7 +378,7 @@ SRC += \ opcodes.h \ parse.c \ parse.h \ - config.h \ + sqlite_cfg.h \ shell.c \ sqlite3.h @@ -434,6 +435,9 @@ TESTSRC = \ $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/ext/session/test_session.c \ $(TOP)/ext/session/sqlite3changebatch.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/test_recover.c \ $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions @@ -494,6 +498,7 @@ TESTSRC2 = \ $(TOP)/src/main.c \ $(TOP)/src/mem5.c \ $(TOP)/src/os.c \ + $(TOP)/src/os_kv.c \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ $(TOP)/src/pager.c \ @@ -557,7 +562,7 @@ HDR = \ $(TOP)/src/vdbeInt.h \ $(TOP)/src/vxworks.h \ $(TOP)/src/whereInt.h \ - config.h + sqlite_cfg.h # Header files used by extensions # @@ -623,7 +628,10 @@ SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC FUZZERSHELL_OPT = -FUZZCHECK_OPT = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ +FUZZCHECK_OPT += -I$(TOP)/test +FUZZCHECK_OPT += -I$(TOP)/ext/recover +FUZZCHECK_OPT += -DSQLITE_OMIT_LOAD_EXTENSION +FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4 @@ -633,8 +641,14 @@ FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB -FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c $(TOP)/test/fuzzinvariants.c +FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c +FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c +FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c +FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c +FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c +FUZZCHECK_SRC += $(TOP)/test/vt02.c DBFUZZ_OPT = +ST_OPT = -DSQLITE_OS_KV_OPTIONAL # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -683,7 +697,7 @@ fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZERSHELL_OPT) \ $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) -fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h +fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h @@ -940,6 +954,9 @@ pcache1.lo: $(TOP)/src/pcache1.c $(HDR) $(TOP)/src/pcache.h os.lo: $(TOP)/src/os.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os.c +os_kv.lo: $(TOP)/src/os_kv.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_kv.c + os_unix.lo: $(TOP)/src/os_unix.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_unix.c @@ -1104,6 +1121,9 @@ SHELL_SRC = \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)/src/test_windirent.c shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl @@ -1389,7 +1409,7 @@ LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS) -speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c +speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c Makefile $(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS) startup$(TEXE): $(TOP)/test/startup.c sqlite3.c @@ -1503,7 +1523,7 @@ clean: rm -f threadtest5 distclean: clean - rm -f config.h config.log config.status libtool Makefile sqlite3.pc \ + rm -f sqlite_cfg.h config.log config.status libtool Makefile sqlite3.pc \ $(TESTPROGS) # @@ -1524,143 +1544,8 @@ sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def $(TCC) -shared -o $@ sqlite3.def \ -Wl,"--strip-all" $(REAL_LIBOBJ) - # -# fiddle/wasm section +# Fiddle app # -# Maintenance reminder: we can/should move this into the wasm-specific -# GNU Make makefile, but we currently need it here for access to -# $(SHELL_OPT). The rest of the wasm-related bits are handled via GNU -# Make in ext/wasm/... -# -wasm_dir = ext/wasm -wasm_dir_abs = $(TOP)/ext/wasm -# ^^^ some emcc opts require absolute paths -fiddle_dir = $(wasm_dir)/fiddle -fiddle_dir_abs = $(TOP)/$(fiddle_dir) -fiddle_module_js = $(fiddle_dir)/fiddle-module.js -#emcc_opt = -O0 -#emcc_opt = -O1 -#emcc_opt = -O2 -#emcc_opt = -O3 -emcc_opt = -Oz -emcc_flags = $(emcc_opt) \ - -sALLOW_TABLE_GROWTH \ - -sABORTING_MALLOC \ - -sSTRICT_JS \ - -sENVIRONMENT=web \ - -sMODULARIZE \ - -sEXPORTED_RUNTIME_METHODS=@$(wasm_dir_abs)/EXPORTED_RUNTIME_METHODS.fiddle \ - -sDYNAMIC_EXECUTION=0 \ - --minify 0 \ - -I. $(SHELL_OPT) \ - -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_UTF16 -DSQLITE_OMIT_DEPRECATED -$(fiddle_module_js): Makefile sqlite3.c shell.c \ - $(wasm_dir)/EXPORTED_RUNTIME_METHODS.fiddle \ - $(wasm_dir)/EXPORTED_FUNCTIONS.fiddle - emcc -o $@ $(emcc_flags) \ - -sEXPORT_NAME=initFiddleModule \ - -sEXPORTED_FUNCTIONS=@$(wasm_dir_abs)/EXPORTED_FUNCTIONS.fiddle \ - -DSQLITE_SHELL_FIDDLE \ - sqlite3.c shell.c - gzip < $@ > $@.gz - gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz -$(fiddle_dir)/fiddle.js.gz: $(fiddle_dir)/fiddle.js - gzip < $< > $@ - -fiddle_generated = $(fiddle_module_js) $(fiddle_module_js).gz \ - $(fiddle_dir)/fiddle-module.wasm \ - $(fiddle_dir)/fiddle-module.wasm.gz \ - $(fiddle_dir)/fiddle.js.gz - -clean-fiddle: - rm -f $(fiddle_generated) -clean: clean-fiddle -fiddle: $(fiddle_module_js) $(fiddle_dir)/fiddle.js.gz -wasm: fiddle -######################################################################## -# Explanation of the emcc build flags follows. Full docs for these can -# be found at: -# -# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js -# -# -sENVIRONMENT=web: elides bootstrap code related to non-web JS -# environments like node.js. Removing this makes the output a tiny -# tick larger but hypothetically makes it more portable to -# non-browser JS environments. -# -# -sMODULARIZE: changes how the generated code is structured to avoid -# declaring a global Module object and instead installing a function -# which loads and initializes the module. The function is named... -# -# -sEXPORT_NAME=jsFunctionName (see -sMODULARIZE) -# -# -sEXPORTED_RUNTIME_METHODS=@/absolute/path/to/file: a file -# containing a list of emscripten-supplied APIs, one per line, which -# must be exported into the generated JS. Must be an absolute path! -# -# -sEXPORTED_FUNCTIONS=@/absolute/path/to/file: a file containing a -# list of C functions, one per line, which must be exported via wasm -# so they're visible to JS. C symbols names in that file must all -# start with an underscore for reasons known only to the emcc -# developers. e.g., _sqlite3_open_v2 and _sqlite3_finalize. Must be -# an absolute path! -# -# -sSTRICT_JS ensures that the emitted JS code includes the 'use -# strict' option. Note that -sSTRICT is more broadly-scoped and -# results in build errors. -# -# -sALLOW_TABLE_GROWTH is required for (at a minimum) the UDF-binding -# feature. Without it, JS functions cannot be made to proxy C-side -# callbacks. -# -# -sABORTING_MALLOC causes the JS-bound _malloc() to abort rather than -# return 0 on OOM. If set to 0 then all code which uses _malloc() -# must, just like in C, check the result before using it, else -# they're likely to corrupt the JS/WASM heap by writing to its -# address of 0. It is, as of this writing, enabled in Emscripten by -# default but we enable it explicitly in case that default changes. -# -# -sDYNAMIC_EXECUTION=0 disables eval() and the Function constructor. -# If the build runs without these, it's preferable to use this flag -# because certain execution environments disallow those constructs. -# This flag is not strictly necessary, however. -# -# -sWASM_BIGINT is UNTESTED but "should" allow the int64-using C APIs -# to work with JS/wasm, insofar as the JS environment supports the -# BigInt type. That support requires an extremely recent browser: -# Safari didn't get that support until late 2020. -# -# --no-entry: for compiling library code with no main(). If this is -# not supplied and the code has a main(), it is called as part of the -# module init process. Note that main() is #if'd out of shell.c -# (renamed) when building in wasm mode. -# -# --pre-js/--post-js=FILE relative or absolute paths to JS files to -# prepend/append to the emcc-generated bootstrapping JS. It's -# easier/faster to develop with separate JS files (reduces rebuilding -# requirements) but certain configurations, namely -sMODULARIZE, may -# require using at least a --pre-js file. They can be used -# individually and need not be paired. -# -# -O0..-O3 and -Oz: optimization levels affect not only C-style -# optimization but whether or not the resulting generated JS code -# gets minified. -O0 compiles _much_ more quickly than -O3 or -Oz, -# and doesn't minimize any JS code, so is recommended for -# development. -O3 or -Oz are recommended for deployment, but -# primarily because -Oz will shrink the wasm file notably. JS-side -# minification makes little difference in terms of overall -# distributable size. -# -# --minify 0: disables minification of the generated JS code, -# regardless of optimization level. Minification of the JS has -# minimal overall effect in the larger scheme of things and results -# in JS files which can neither be edited nor viewed as text files in -# Fossil (which flags them as binary because of their extreme line -# lengths). Interestingly, whether or not the comments in the -# generated JS file get stripped is unaffected by this setting and -# depends entirely on the optimization level. Higher optimization -# levels reduce the size of the JS considerably even without -# minification. -# -######################################################################## +fiddle: sqlite3.c shell.c + make -C ext/wasm fiddle emcc_opt=-Os diff --git a/Makefile.msc b/Makefile.msc index 902ae1775e..8c96cb1852 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1251,7 +1251,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memdb.lo memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ - notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ + notify.lo opcodes.lo os.lo os_kv.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \ @@ -1332,6 +1332,7 @@ SRC00 = \ $(TOP)\src\mutex_w32.c \ $(TOP)\src\notify.c \ $(TOP)\src\os.c \ + $(TOP)\src\os_kv.c \ $(TOP)\src\os_unix.c \ $(TOP)\src\os_win.c @@ -1588,6 +1589,9 @@ TESTEXT = \ $(TOP)\ext\misc\unionvtab.c \ $(TOP)\ext\misc\wholenumber.c \ $(TOP)\ext\rtree\test_rtreedoc.c \ + $(TOP)\ext\recover\sqlite3recover.c \ + $(TOP)\ext\recover\test_recover.c \ + $(TOP)\ext\recover\dbdata.c \ fts5.c # If use of zlib is enabled, add the "zipfile.c" source file. @@ -1698,15 +1702,25 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 # MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -FUZZCHECK_OPTS = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzcheck.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\ossfuzz.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c -FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c $(TOP)\test\fuzzinvariants.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ @@ -2054,6 +2068,9 @@ pcache1.lo: $(TOP)\src\pcache1.c $(HDR) $(TOP)\src\pcache.h os.lo: $(TOP)\src\os.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os.c +os_kv.lo: $(TOP)\src\os_kv.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os_kv.c + os_unix.lo: $(TOP)\src\os_unix.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os_unix.c @@ -2225,6 +2242,9 @@ SHELL_SRC = \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\misc\memtrace.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)\src\test_windirent.c # If use of zlib is enabled, add the "zipfile.c" source file. diff --git a/configure b/configure index e9f135f87e..4a32d5e561 100755 --- a/configure +++ b/configure @@ -11875,7 +11875,7 @@ fi ######### # Output the config header -ac_config_headers="$ac_config_headers config.h" +ac_config_headers="$ac_config_headers sqlite_cfg.h" ######### @@ -12838,7 +12838,7 @@ for ac_config_target in $ac_config_targets do case $ac_config_target in "libtool") CONFIG_COMMANDS="$CONFIG_COMMANDS libtool" ;; - "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;; + "sqlite_cfg.h") CONFIG_HEADERS="$CONFIG_HEADERS sqlite_cfg.h" ;; "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; "sqlite3.pc") CONFIG_FILES="$CONFIG_FILES sqlite3.pc" ;; diff --git a/configure.ac b/configure.ac index cc805a00d5..5f905db40c 100644 --- a/configure.ac +++ b/configure.ac @@ -806,7 +806,7 @@ AC_SUBST(AMALGAMATION_LINE_MACROS) ######### # Output the config header -AC_CONFIG_HEADERS(config.h) +AC_CONFIG_HEADERS(sqlite_cfg.h) ######### # Generate the output files. diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c index 8c340889fe..e7c2c9d5c0 100644 --- a/ext/misc/cksumvfs.c +++ b/ext/misc/cksumvfs.c @@ -47,7 +47,7 @@ ** ** sqlite3 *db; ** sqlite3_open(":memory:", &db); -** sqlite3_load_extention(db, "./cksumvfs"); +** sqlite3_load_extension(db, "./cksumvfs"); ** sqlite3_close(db); ** ** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and diff --git a/ext/misc/dbdata.c b/ext/recover/dbdata.c similarity index 84% rename from ext/misc/dbdata.c rename to ext/recover/dbdata.c index 7405e7c890..4132b83d71 100644 --- a/ext/misc/dbdata.c +++ b/ext/recover/dbdata.c @@ -71,16 +71,20 @@ ** It contains one entry for each b-tree pointer between a parent and ** child page in the database. */ + #if !defined(SQLITEINT_H) #include "sqlite3ext.h" typedef unsigned char u8; +typedef unsigned int u32; #endif SQLITE_EXTENSION_INIT1 #include #include +#ifndef SQLITE_OMIT_VIRTUALTABLE + #define DBDATA_PADDING_BYTES 100 typedef struct DbdataTable DbdataTable; @@ -102,11 +106,12 @@ struct DbdataCursor { /* Only for the sqlite_dbdata table */ u8 *pRec; /* Buffer containing current record */ - int nRec; /* Size of pRec[] in bytes */ - int nHdr; /* Size of header in bytes */ + sqlite3_int64 nRec; /* Size of pRec[] in bytes */ + sqlite3_int64 nHdr; /* Size of header in bytes */ int iField; /* Current field number */ u8 *pHdrPtr; u8 *pPtr; + u32 enc; /* Text encoding */ sqlite3_int64 iIntkey; /* Integer key value */ }; @@ -299,14 +304,14 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ /* ** Utility methods to decode 16 and 32-bit big-endian unsigned integers. */ -static unsigned int get_uint16(unsigned char *a){ +static u32 get_uint16(unsigned char *a){ return (a[0]<<8)|a[1]; } -static unsigned int get_uint32(unsigned char *a){ - return ((unsigned int)a[0]<<24) - | ((unsigned int)a[1]<<16) - | ((unsigned int)a[2]<<8) - | ((unsigned int)a[3]); +static u32 get_uint32(unsigned char *a){ + return ((u32)a[0]<<24) + | ((u32)a[1]<<16) + | ((u32)a[2]<<8) + | ((u32)a[3]); } /* @@ -321,7 +326,7 @@ static unsigned int get_uint32(unsigned char *a){ */ static int dbdataLoadPage( DbdataCursor *pCsr, /* Cursor object */ - unsigned int pgno, /* Page number of page to load */ + u32 pgno, /* Page number of page to load */ u8 **ppPage, /* OUT: pointer to page buffer */ int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ @@ -331,25 +336,27 @@ static int dbdataLoadPage( *ppPage = 0; *pnPage = 0; - sqlite3_bind_int64(pStmt, 2, pgno); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - int nCopy = sqlite3_column_bytes(pStmt, 0); - if( nCopy>0 ){ - u8 *pPage; - pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); - if( pPage==0 ){ - rc = SQLITE_NOMEM; - }else{ - const u8 *pCopy = sqlite3_column_blob(pStmt, 0); - memcpy(pPage, pCopy, nCopy); - memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); + if( pgno>0 ){ + sqlite3_bind_int64(pStmt, 2, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nCopy = sqlite3_column_bytes(pStmt, 0); + if( nCopy>0 ){ + u8 *pPage; + pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); + if( pPage==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); + } + *ppPage = pPage; + *pnPage = nCopy; } - *ppPage = pPage; - *pnPage = nCopy; } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; } - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = rc2; return rc; } @@ -358,17 +365,30 @@ static int dbdataLoadPage( ** Read a varint. Put the value in *pVal and return the number of bytes. */ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ - sqlite3_int64 v = 0; + sqlite3_uint64 u = 0; int i; for(i=0; i<8; i++){ - v = (v<<7) + (z[i]&0x7f); - if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + u = (u<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } } - v = (v<<8) + (z[i]&0xff); - *pVal = v; + u = (u<<8) + (z[i]&0xff); + *pVal = (sqlite3_int64)u; return 9; } +/* +** Like dbdataGetVarint(), but set the output to 0 if it is less than 0 +** or greater than 0xFFFFFFFF. This can be used for all varints in an +** SQLite database except for key values in intkey tables. +*/ +static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 val; + int nRet = dbdataGetVarint(z, &val); + if( val<0 || val>0xFFFFFFFF ) val = 0; + *pVal = val; + return nRet; +} + /* ** Return the number of bytes of space used by an SQLite value of type ** eType. @@ -405,6 +425,7 @@ static int dbdataValueBytes(int eType){ */ static void dbdataValue( sqlite3_context *pCtx, + u32 enc, int eType, u8 *pData, int nData @@ -449,7 +470,19 @@ static void dbdataValue( default: { int n = ((eType-12) / 2); if( eType % 2 ){ - sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); + switch( enc ){ +#ifndef SQLITE_OMIT_UTF16 + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; +#endif + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } }else{ sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); } @@ -477,6 +510,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); if( rc!=SQLITE_OK ) return rc; if( pCsr->aPage ) break; + if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; } pCsr->iCell = pTab->bPtr ? -2 : 0; @@ -540,7 +574,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( bNextPage || iOff>pCsr->nPage ){ bNextPage = 1; }else{ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); + iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); } /* If this is a leaf intkey cell, load the rowid */ @@ -587,7 +621,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Load content from overflow pages */ if( nPayload>nLocal ){ sqlite3_int64 nRem = nPayload - nLocal; - unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); while( nRem>0 ){ u8 *aOvfl = 0; int nOvfl = 0; @@ -607,7 +641,8 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ } } - iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); + iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr); + if( nHdr>nPayload ) nHdr = 0; pCsr->nHdr = nHdr; pCsr->pHdrPtr = &pCsr->pRec[iHdr]; pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; @@ -621,7 +656,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ bNextPage = 1; }else{ - pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType); + pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); pCsr->pPtr += dbdataValueBytes(iType); } } @@ -660,6 +695,18 @@ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ return pCsr->aPage==0; } +/* +** Return true if nul-terminated string zSchema ends in "()". Or false +** otherwise. +*/ +static int dbdataIsFunction(const char *zSchema){ + size_t n = strlen(zSchema); + if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){ + return (int)n-2; + } + return 0; +} + /* ** Determine the size in pages of database zSchema (where zSchema is ** "main", "temp" or the name of an attached database) and set @@ -670,10 +717,16 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; char *zSql = 0; int rc, rc2; + int nFunc = 0; sqlite3_stmt *pStmt = 0; - zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema); + }else{ + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + } if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -684,6 +737,25 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ return rc; } +/* +** Attempt to figure out the encoding of the database by retrieving page 1 +** and inspecting the header field. If successful, set the pCsr->enc variable +** and return SQLITE_OK. Otherwise, return an SQLite error code. +*/ +static int dbdataGetEncoding(DbdataCursor *pCsr){ + int rc = SQLITE_OK; + int nPg1 = 0; + u8 *aPg1 = 0; + rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); + assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 ); + if( rc==SQLITE_OK && nPg1>0 ){ + pCsr->enc = get_uint32(&aPg1[56]); + } + sqlite3_free(aPg1); + return rc; +} + + /* ** xFilter method for sqlite_dbdata and sqlite_dbptr. */ @@ -701,19 +773,28 @@ static int dbdataFilter( assert( pCsr->iPgno==1 ); if( idxNum & 0x01 ){ zSchema = (const char*)sqlite3_value_text(argv[0]); + if( zSchema==0 ) zSchema = ""; } if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); pCsr->bOnePage = 1; }else{ - pCsr->nPage = dbdataDbsize(pCsr, zSchema); rc = dbdataDbsize(pCsr, zSchema); } if( rc==SQLITE_OK ){ + int nFunc = 0; if( pTab->pStmt ){ pCsr->pStmt = pTab->pStmt; pTab->pStmt = 0; + }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } }else{ rc = sqlite3_prepare_v2(pTab->db, "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, @@ -726,13 +807,20 @@ static int dbdataFilter( }else{ pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } + + /* Try to determine the encoding of the db by inspecting the header + ** field on page 1. */ + if( rc==SQLITE_OK ){ + rc = dbdataGetEncoding(pCsr); + } + if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); } return rc; } -/* +/* ** Return a column for the sqlite_dbdata or sqlite_dbptr table. */ static int dbdataColumn( @@ -778,9 +866,10 @@ static int dbdataColumn( sqlite3_result_int64(ctx, pCsr->iIntkey); }else{ sqlite3_int64 iType; - dbdataGetVarint(pCsr->pHdrPtr, &iType); + dbdataGetVarintU32(pCsr->pHdrPtr, &iType); dbdataValue( - ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr + ctx, pCsr->enc, iType, pCsr->pPtr, + &pCsr->pRec[pCsr->nRec] - pCsr->pPtr ); } break; @@ -849,3 +938,5 @@ int sqlite3_dbdata_init( SQLITE_EXTENSION_INIT2(pApi); return sqlite3DbdataRegister(db); } + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test new file mode 100644 index 0000000000..75f5dba1ff --- /dev/null +++ b/ext/recover/recover1.test @@ -0,0 +1,277 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recover1 + +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } + + compare_result $db1 $db2 "PRAGMA page_size" + compare_result $db1 $db2 "PRAGMA auto_vacuum" + compare_result $db1 $db2 "PRAGMA encoding" + compare_result $db1 $db2 "PRAGMA user_version" + compare_result $db1 $db2 "PRAGMA application_id" +} + +proc do_recover_test {tn} { + forcedelete test.db2 + forcedelete rstate.db + + uplevel [list do_test $tn.1 { + set R [sqlite3_recover_init db main test.db2] + $R config testdb rstate.db + $R run + $R finish + } {}] + + sqlite3 db2 test.db2 + uplevel [list do_test $tn.2 [list compare_dbs db db2] {}] + db2 close + + forcedelete test.db2 + forcedelete rstate.db + + uplevel [list do_test $tn.3 { + set ::sqlhook [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R config testdb rstate.db + $R config rowids 1 + $R run + $R finish + } {}] + + sqlite3 db2 test.db2 + execsql [join $::sqlhook ";"] db2 + db2 close + sqlite3 db2 test.db2 + uplevel [list do_test $tn.4 [list compare_dbs db db2] {}] + db2 close +} + +proc my_sql_hook {sql} { + lappend ::sqlhook $sql + return 0 +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b) WITHOUT ROWID; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT i*2, hex(randomblob(250)) FROM s; + INSERT INTO t2 SELECT * FROM t1; +} + +do_recover_test 1 + +do_execsql_test 2.0 { + ALTER TABLE t1 ADD COLUMN c DEFAULT 'xyz' +} +do_recover_test 2 + +do_execsql_test 3.0 { + CREATE INDEX i1 ON t1(c); +} +do_recover_test 3 + +do_execsql_test 4.0 { + CREATE VIEW v1 AS SELECT * FROM t2; +} +do_recover_test 4 + +do_execsql_test 5.0 { + CREATE UNIQUE INDEX i2 ON t1(c, b); +} +do_recover_test 5 + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1( + a INTEGER PRIMARY KEY, + b INT, + c TEXT, + d INT GENERATED ALWAYS AS (a*abs(b)) VIRTUAL, + e TEXT GENERATED ALWAYS AS (substr(c,b,b+1)) STORED, + f TEXT GENERATED ALWAYS AS (substr(c,b,b+1)) STORED + ); + + INSERT INTO t1 VALUES(1, 2, 'hello world'); +} +do_recover_test 6 + +do_execsql_test 7.0 { + CREATE TABLE t2(i, j GENERATED ALWAYS AS (i+1) STORED, k); + INSERT INTO t2 VALUES(10, 'ten'); +} +do_execsql_test 7.1 { + SELECT * FROM t2 +} {10 11 ten} + +do_recover_test 7.2 + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2 + ) + INSERT INTO x1(b, c) SELECT hex(randomblob(100)), hex(randomblob(100)) FROM s; + + CREATE INDEX x1b ON x1(b); + CREATE INDEX x1cb ON x1(c, b); + DELETE FROM x1 WHERE a>50; + + ANALYZE; +} + +do_recover_test 8 + +#------------------------------------------------------------------------- +reset_db +ifcapable fts5 { + do_execsql_test 9.1 { + CREATE VIRTUAL TABLE ft5 USING fts5(a, b); + INSERT INTO ft5 VALUES('hello', 'world'); + } + do_recover_test 9 +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.1 { + CREATE TABLE x1(a PRIMARY KEY, str TEXT) WITHOUT ROWID; + INSERT INTO x1 VALUES(1, ' + \nhello\012world(\n0)(\n1) + '); + INSERT INTO x1 VALUES(2, ' + \nhello + '); +} +do_execsql_test 10.2 " + INSERT INTO x1 VALUES(3, '\012hello there\015world'); + INSERT INTO x1 VALUES(4, '\015hello there\015world'); +" +do_recover_test 10 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 11.1 { + PRAGMA page_size = 4096; + PRAGMA encoding='utf16'; + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 45; + PRAGMA application_id = 22; + + CREATE TABLE u1(u, v); + INSERT INTO u1 VALUES('edvin marton', 'bond'); + INSERT INTO u1 VALUES(1, 4.0); +} +do_execsql_test 11.1a { + PRAGMA auto_vacuum; +} {2} + +do_recover_test 11 + +do_test 12.1 { + set R [sqlite3_recover_init db "" test.db2] + $R config lostandfound "" + $R config invalid xyz +} {12} +do_test 12.2 { + $R run + $R run +} {0} + +do_test 12.3 { + $R finish +} {} + + + +#------------------------------------------------------------------------- +reset_db +file_control_reservebytes db 16 +do_execsql_test 12.1 { + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 45; + PRAGMA application_id = 22; + + CREATE TABLE u1(u, v); + CREATE UNIQUE INDEX i1 ON u1(u, v); + INSERT INTO u1 VALUES(1, 2), (3, 4); + + CREATE TABLE u2(u, v); + CREATE UNIQUE INDEX i2 ON u1(u, v); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(50000)), hex(randomblob(20000))); +} + +do_recover_test 12 + +#------------------------------------------------------------------------- +reset_db +sqlite3 db "" +do_recover_test 13 + +do_execsql_test 14.1 { + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 45; + PRAGMA application_id = 22; + + CREATE TABLE u1(u, v); + CREATE UNIQUE INDEX i1 ON u1(u, v); + INSERT INTO u1 VALUES(1, 2), (3, 4); + + CREATE TABLE u2(u, v); + CREATE UNIQUE INDEX i2 ON u1(u, v); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(50000)), hex(randomblob(20000))); +} +do_recover_test 14 + +#------------------------------------------------------------------------- +reset_db +execsql { + PRAGMA journal_mode=OFF; + PRAGMA mmap_size=10; +} +do_execsql_test 15.1 { + CREATE TABLE t1(x); +} {} +do_recover_test 15 + +finish_test + diff --git a/ext/recover/recover_common.tcl b/ext/recover/recover_common.tcl new file mode 100644 index 0000000000..fdf735ee75 --- /dev/null +++ b/ext/recover/recover_common.tcl @@ -0,0 +1,14 @@ + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl + +if {[info commands sqlite3_recover_init]==""} { + finish_test + return -code return +} + + + diff --git a/ext/recover/recoverclobber.test b/ext/recover/recoverclobber.test new file mode 100644 index 0000000000..e096b2e216 --- /dev/null +++ b/ext/recover/recoverclobber.test @@ -0,0 +1,50 @@ +# 2019 April 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the SQLITE_RECOVER_ROWIDS option. +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoverclobber + +proc recover {db output} { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} + +forcedelete test.db2 +do_execsql_test 1.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.x1(x, one); + INSERT INTO x1 VALUES(1, 'one'), (2, 'two'), (3, 'three'); + + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3), (4, 4); + + DETACH aux; +} + +breakpoint +do_test 1.1 { + recover db test.db2 +} {} + +do_execsql_test 1.2 { + ATTACH 'test.db2' AS aux; + SELECT * FROM aux.t1; +} {1 1 2 2 3 3 4 4} + +do_catchsql_test 1.3 { + SELECT * FROM aux.x1; +} {1 {no such table: aux.x1}} + +finish_test diff --git a/ext/recover/recovercorrupt.test b/ext/recover/recovercorrupt.test new file mode 100644 index 0000000000..eb6fe53add --- /dev/null +++ b/ext/recover/recovercorrupt.test @@ -0,0 +1,67 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recovercorrupt + +database_may_be_corrupt + +do_execsql_test 1.0 { + PRAGMA page_size = 512; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, hex(randomblob(100)), randomblob(200)); + CREATE INDEX i1 ON t1(b, c); + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, hex(randomblob(100)), randomblob(200)); + ANALYZE; + PRAGMA writable_schema = 1; + DELETE FROM sqlite_schema WHERE name='t2'; +} + +do_test 1.1 { + expr [file size test.db]>3072 +} {1} + +proc toggle_bit {blob bit} { + set byte [expr {$bit / 8}] + set bit [expr {$bit & 0x0F}] + binary scan $blob a${byte}ca* A x B + set x [expr {$x ^ (1 << $bit)}] + binary format a*ca* $A $x $B +} + + +db_save_and_close +for {set ii 0} {$ii < 10000} {incr ii} { + db_restore_and_reopen + db func toggle_bit toggle_bit + set bitsperpage [expr 512*8] + + set pg [expr {($ii / $bitsperpage)+1}] + set byte [expr {$ii % $bitsperpage}] + db eval { + UPDATE sqlite_dbpage SET data = toggle_bit(data, $byte) WHERE pgno=$pg + } + + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R run + do_test 1.2.$ii { + $R finish + } {} +} + + +finish_test + diff --git a/ext/recover/recovercorrupt2.test b/ext/recover/recovercorrupt2.test new file mode 100644 index 0000000000..7147c67e93 --- /dev/null +++ b/ext/recover/recovercorrupt2.test @@ -0,0 +1,289 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recovercorrupt2 + +do_execsql_test 1.0 { + PRAGMA page_size = 512; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, hex(randomblob(100)), randomblob(200)); + CREATE INDEX i1 ON t1(b, c); + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, hex(randomblob(100)), randomblob(200)); + ANALYZE; + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE INDEX i1 ON o(world)' WHERE name='i1'; + DELETE FROM sqlite_schema WHERE name='sqlite_stat4'; +} + +do_test 1.1 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} + +sqlite3 db2 test.db2 +do_execsql_test -db db2 1.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c)} + {CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID} + {CREATE TABLE sqlite_stat1(tbl,idx,stat)} +} +db2 close + +do_execsql_test 1.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE TABLE t2 syntax error!' WHERE name='t2'; +} + +do_test 1.4 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} + +sqlite3 db2 test.db2 +do_execsql_test -db db2 1.5 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c)} + {CREATE TABLE sqlite_stat1(tbl,idx,stat)} +} +db2 close + +#------------------------------------------------------------------------- +# +reset_db +do_test 2.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 8192 pagesize 4096 filename x3.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 02 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ................ +| 96: 00 2e 63 00 0d 00 00 00 01 0f d8 00 0f d8 00 00 ..c............. +| 4048: 00 00 00 00 00 00 00 00 26 01 06 17 11 11 01 39 ........&......9 +| 4064: 74 61 62 6c 65 74 31 74 31 02 43 52 45 41 54 45 tablet1t1.CREATE +| 4080: 20 54 41 42 4c 45 20 74 31 28 61 2c 62 2c 63 29 TABLE t1(a,b,c) +| page 2 offset 4096 +| 0: 0d 00 00 00 01 0f ce 00 0f ce 00 00 00 00 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff ..............(. +| 4048: ff ff ff ff ff ff ff 28 04 27 25 23 61 61 61 61 .........'%#aaaa +| 4064: 61 61 61 61 61 61 61 61 61 62 62 62 62 62 62 62 aaaaaaaaabbbbbbb +| 4080: 62 62 62 62 62 63 63 63 63 63 63 63 63 63 63 63 bbbbbccccccccccc +| end x3.db +}]} {} + +do_test 2.1 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} + +sqlite3 db2 test.db2 +do_execsql_test -db db2 2.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE t1(a,b,c)} +} +do_execsql_test -db db2 2.3 { + SELECT * FROM t1 +} {} +db2 close + +#------------------------------------------------------------------------- +# +reset_db +do_test 3.0 { + sqlite3 db {} + db deserialize [decode_hexdb { + .open --hexdb + | size 4096 pagesize 1024 filename corrupt032.txt.db + | page 1 offset 0 + | 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. + | 16: 04 00 01 01 08 40 20 20 00 00 00 02 00 00 00 03 .....@ ........ + | 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ + | 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ + | 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ................ + | 96: 00 2e 24 80 0d 00 00 00 01 03 d4 00 03 d4 00 00 ..$............. + | 976: 00 00 00 00 22 01 06 17 11 11 01 31 74 61 62 6c ...........1tabl + | 992: 65 74 31 74 31 02 43 52 45 41 54 45 20 54 41 42 et1t1.CREATE TAB + | 1008: 4c 45 20 74 31 28 78 29 00 00 00 00 00 00 00 00 LE t1(x)........ + | page 2 offset 1024 + | 0: 0d 00 00 00 01 02 06 00 02 06 00 00 00 00 00 00 ................ + | 512: 00 00 00 00 00 00 8b 60 01 03 97 46 00 00 00 00 .......`...F.... + | 1008: 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 ................ + | end corrupt032.txt.db +}]} {} + +do_test 3.1 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} + +#------------------------------------------------------------------------- +# +reset_db +do_test 4.0 { + sqlite3 db {} + db deserialize [decode_hexdb { + .open --hexdb + | size 4096 pagesize 4096 filename crash-00f2d3627f1b43.db + | page 1 offset 0 + | 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. + | 16: 00 01 01 02 00 40 20 20 01 00 ff 00 42 01 10 01 .....@ ....B... + | 32: ef 00 00 87 00 ff ff ff f0 01 01 10 ff ff 00 00 ................ + | end crash-00f2d3627f1b43.db +}]} {} + +do_test 4.1 { + set R [sqlite3_recover_init db main test.db2] + catch { $R run } + list [catch { $R finish } msg] $msg +} {1 {unable to open database file}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 5.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 16384 pagesize 4096 filename crash-7b75760a4c5f15.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 04 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 03 0f 4e 00 0f bc 0f 90 ..........N..... +| 112: 0f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .N.............. +| 3904: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 40 03 ..............@. +| 3920: 06 17 11 11 01 6d 74 61 62 6c 65 74 32 74 32 04 .....mtablet2t2. +| 3936: 43 52 45 41 54 45 20 54 41 42 4c 45 20 74 32 28 CREATE TABLE t2( +| 3952: 78 2c 79 2c 7a 20 50 52 49 4d 41 52 59 20 4b 45 x,y,z PRIMARY KE +| 3968: 59 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 Y) WITHOUT ROWID +| 3984: 2a 02 06 17 13 11 01 3f 69 6e 64 65 78 74 31 61 *......?indext1a +| 4000: 74 31 03 43 52 45 41 54 45 20 49 4e 44 45 58 20 t1.CREATE INDEX +| 4016: 74 31 61 20 4f 4e 20 74 31 28 61 29 42 01 06 17 t1a ON t1(a)B... +| 4032: 11 11 01 71 74 61 62 6c 65 74 31 74 31 02 43 52 ...qtablet1t1.CR +| 4048: 45 41 54 45 20 54 41 42 4c 45 20 74 31 28 61 20 EATE TABLE t1(a +| 4064: 49 4e 54 2c 62 20 54 45 58 54 2c 63 20 42 4c 4f INT,b TEXT,c BLO +| 4080: 42 2c 64 20 52 45 41 4c 29 20 53 54 52 49 43 54 B,d REAL) STRICT +| page 2 offset 4096 +| 0: 0d 00 00 00 14 0c ae 00 0f df 0f bd 0f 9a 0f 76 ...............v +| 16: 0f 51 0f 2b 0f 04 0e dc 0e b3 0e 89 0e 5e 0e 32 .Q.+.........^.2 +| 32: 0e 05 0d 1a 0d a8 0d 78 0d 47 0d 15 0c e2 00 00 .......x.G...... +| 3232: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 32 14 ..............2. +| 3248: 05 06 3f 34 07 15 f4 c9 23 af e2 b3 b6 61 62 63 ..?4....#....abc +| 3264: 30 32 30 78 79 7a 01 00 00 00 00 00 00 00 00 00 020xyz.......... +| 3280: 00 00 00 00 00 00 00 00 00 00 c3 b0 96 7e fb 4e .............~.N +| 3296: c5 4c 31 13 05 06 1f 32 07 dd f2 2a a5 7e b2 4d .L1....2...*.~.M +| 3312: 82 61 62 63 30 31 39 78 79 7a 01 00 00 00 00 00 .abc019xyz...... +| 3328: 00 00 00 00 00 00 00 00 00 00 00 00 00 c3 a3 d6 ................ +| 3344: e9 f1 c2 fd f3 30 12 05 06 1f 30 07 8f 8f f5 c4 .....0....0..... +| 3360: 35 b6 7f 8d 61 62 63 30 31 38 00 00 00 00 00 00 5...abc018...... +| 3376: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 ...............C +| 3392: b2 13 1f 9d 56 8a 47 21 b1 05 06 1f 2e 07 7f 46 ....V.G!.......F +| 3408: 91 03 3f 97 fb f7 61 62 63 30 00 00 00 00 00 00 ..?...abc0...... +| 3440: c3 bb d8 96 86 c2 e8 2b 2e 10 05 06 1f 2c 07 6d .......+.....,.m +| 3456: 85 7b ce d0 32 d2 54 61 62 63 30 00 00 00 00 00 ....2.Tabc0..... +| 3488: 43 a1 eb 44 14 dc 03 7b 2d 0f 05 06 1f 2a 07 d9 C..D....-....*.. +| 3504: ab ec bf 34 51 70 f3 61 62 63 30 31 35 78 79 7a ...4Qp.abc015xyz +| 3520: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 c3 ................ +| 3536: b6 3d f4 46 b1 6a af 2c 0e 05 06 1f 28 07 36 75 .=.F.j.,....(.6u +| 3552: e9 a2 bd 05 04 ea 61 62 63 30 31 34 78 79 7a 00 ......abc014xyz. +| 3568: 00 00 00 00 00 00 00 00 00 00 00 00 00 c3 ab 23 ...............# +| 3584: a7 6a 34 ca f8 2b 0d 05 06 1f 26 07 48 45 ab e0 .j4..+....&.HE.. +| 3600: 8c 7c ff 0c 61 62 63 30 31 33 78 79 7a 00 00 00 .|..abc013xyz... +| 3616: 00 00 00 00 0d d0 00 00 00 00 43 b8 d3 93 f4 92 ..........C..... +| 3632: 5b 7a 2a 0c 05 06 1f 24 07 be 6d 1e db 61 5d 80 [z*....$..m..a]. +| 3648: 9f 61 62 63 30 31 32 78 79 7a 00 00 00 00 00 00 .abc012xyz...... +| 3664: 00 00 00 00 00 00 43 b5 a1 a4 af 7b c6 60 29 0b ......C......`). +| 3680: 05 06 1f 22 07 6e a2 a3 64 68 d4 a6 bd 61 62 63 .....n..dh...abc +| 3696: 30 31 31 78 79 7a 00 00 00 00 00 00 00 00 00 00 011xyz.......... +| 3712: 00 c3 c4 1e ff 0f fc e6 ff 28 0a 05 06 1f 20 07 .........(.... . +| 3728: 50 f9 4a bb a5 7a 1e ca 61 62 63 30 31 30 78 79 P.J..z..abc010xy +| 3744: 7a 00 00 00 00 00 00 00 00 00 00 c3 a7 90 ed d9 z............... +| 3760: 5c 2c d5 27 09 05 06 1f 1e 07 90 8e 1d d9 1c 3a .,.'...........: +| 3776: e8 c1 61 62 63 30 30 39 78 79 7a 00 00 00 00 00 ..abc009xyz..... +| 3792: 00 00 00 00 43 a7 97 87 cf b0 ff 79 26 08 05 06 ....C......y&... +| 3808: 1f 1c 07 86 65 f6 7c 50 7a 2c 76 61 62 63 30 30 ....e.|Pz,vabc00 +| 3824: 38 78 79 7a 00 00 00 00 00 00 00 00 c3 b0 e3 4c 8xyz...........L +| 3840: 4f d3 41 b5 25 07 05 06 1f 1a 07 8b 20 e5 68 11 O.A.%....... .h. +| 3856: 13 55 87 61 62 63 30 30 37 78 79 7a 00 00 00 00 .U.abc007xyz.... +| 3872: 00 00 00 c3 b6 a3 74 f1 9c 33 f8 24 06 05 06 1f ......t..3.$.... +| 3888: 18 07 97 3c bc 34 49 94 54 ab 61 62 63 30 30 36 ...<.4I.T.abc006 +| 3904: 78 79 7a 00 00 00 00 00 00 c3 88 00 c2 ca 4c 4d xyz...........LM +| 3920: d3 23 05 05 06 1f 16 07 59 37 11 10 e9 e5 3d d5 .#......Y7....=. +| 3936: 61 62 63 30 30 35 78 79 7a 00 00 00 00 00 c3 c0 abc005xyz....... +| 3952: 15 12 67 ed 4b 79 22 04 05 06 1f 14 07 93 39 01 ..g.Ky........9. +| 3968: 7f b8 c7 99 58 61 62 63 30 30 34 78 79 7a 00 00 ....Xabc004xyz.. +| 3984: 09 c0 43 bf e0 e7 6d 70 fd 61 21 03 05 06 1f 12 ..C...mp.a!..... +| 4000: 07 b6 df 8d 8b 27 08 22 5a 61 62 63 30 30 33 78 .....'..Zabc003x +| 4016: 79 7a 00 00 00 c3 c7 ea 0f dc dd 32 22 20 02 05 yz.........2. .. +| 4032: 06 1f 10 07 2f a6 da 71 df 66 b3 b5 61 62 63 30 ..../..q.f..abc0 +| 4048: 30 32 78 79 7a 00 00 c3 ce d9 8d e9 ec 20 45 1f 02xyz........ E. +| 4064: 01 05 06 1f 0e 07 5a 47 53 20 3b 48 8f c0 61 62 ......ZGS ;H..ab +| 4080: 63 30 30 31 78 79 7a 00 c3 c9 e6 81 f8 d9 24 04 c001xyz.......$. +| page 3 offset 8192 +| 0: 0a 00 00 00 14 0e fd 00 0f f3 0f e6 0f d9 0f cc ................ +| 16: 0f bf 0f b2 0f a5 0f 98 0f 8b 0f 7e 0f 71 0f 64 ...........~.q.d +| 32: 0f 57 0f 4a 0f 3d 0f 30 0f 24 00 00 00 00 00 00 .W.J.=.0.$...... +| 3824: 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 03 06 ................ +| 3840: 01 7f 46 91 03 3f 97 fb f7 11 0c 03 06 01 6e a2 ..F..?........n. +| 3856: a3 64 68 d4 a6 bd 0b 0c 03 06 01 6d 85 7b ce d0 .dh........m.... +| 3872: 32 d2 54 10 0b 03 06 09 5a 47 53 20 3b 48 8f c0 2.T.....ZGS ;H.. +| 3888: 0c 03 06 01 59 37 11 10 e9 e5 3d d5 05 0c 03 06 ....Y7....=..... +| 3904: 01 50 f9 4a bb a5 7a 1e ca 0a 0c 03 06 01 48 45 .P.J..z.......HE +| 3920: ab e0 8c 7c ff 0c 0d 0c 03 06 01 36 75 e9 a2 bd ...|.......6u... +| 3936: 05 04 ea 0e 0c 03 06 01 2f a6 da 71 df 66 b3 b5 ......../..q.f.. +| 3952: 02 0c 03 06 01 15 f4 c9 23 af e2 b3 b6 14 0c 03 ........#....... +| 3968: 06 01 dd f2 2a a5 7e b2 4d 82 13 0c 03 06 01 d9 ....*.~.M....... +| 3984: ab ec bf 34 51 70 f3 0f 0c 03 06 01 be 6d 1e db ...4Qp.......m.. +| 4000: 61 5d 80 9f 0c 0c 03 06 01 b6 df 8d 8b 27 08 22 a]...........'.. +| 4016: 5a 03 0c 03 06 01 97 3c bc 34 49 94 54 ab 06 0c Z......<.4I.T... +| 4032: 03 06 01 93 39 01 7f b8 c7 99 58 04 0c 03 06 01 ....9.....X..... +| 4048: 90 8e 1d d9 1c 3a e8 c1 09 0c 03 06 01 8f 8f f5 .....:.......... +| 4064: c4 35 b6 7f 8d 12 0c 03 06 01 8b 20 e5 68 11 13 .5......... .h.. +| 4080: 55 87 07 0c 03 06 01 86 65 f6 7c 50 7a 2b 06 08 U.......e.|Pz+.. +| page 4 offset 12288 +| 0: 0a 00 00 00 14 0f 62 00 0f 7a 0f a1 0f c9 0f d9 ......b..z...... +| 16: 0f 81 0f d1 0f f1 0f f9 0f e1 0f 89 0e 6a 0f c1 .............j.. +| 32: 0f 91 0f 99 0f b9 0f 72 0f 62 0f e9 0f b1 0f a9 .......r.b...... +| 3936: 00 00 07 04 01 01 01 11 0e 9e 07 04 01 01 01 0b ................ +| 3952: 31 16 07 04 01 01 01 10 37 36 06 04 09 01 01 ab 1.......76...... +| 3968: 58 07 04 01 01 01 05 1c 28 07 04 01 01 01 0a 10 X.......(....... +| 3984: cf 07 04 01 01 01 0d b2 e3 07 04 01 01 01 0e d3 ................ +| 4000: f2 07 04 01 01 01 02 41 ad 07 04 01 01 01 14 3e .......A.......> +| 4016: 22 07 04 01 01 01 13 27 45 07 04 01 01 01 0f ad .......'E....... +| 4032: dd 07 04 01 01 01 0c 2e a1 07 04 01 01 01 03 df ................ +| 4048: e1 07 04 01 01 01 06 59 a7 07 04 01 01 01 04 27 .......Y.......' +| 4064: bd 07 04 01 01 01 09 d0 e0 07 04 01 01 01 12 39 ...............9 +| 4080: 4f 07 04 01 01 01 07 c4 11 06 04 00 00 00 00 00 O............... +| end crash-7b75760a4c5f15.db +}]} {} + +do_test 5.1 { + set R [sqlite3_recover_init db main test.db2] + catch { $R run } + list [catch { $R finish } msg] $msg +} {0 {}} + +finish_test + diff --git a/ext/recover/recoverfault.test b/ext/recover/recoverfault.test new file mode 100644 index 0000000000..30bb65527d --- /dev/null +++ b/ext/recover/recoverfault.test @@ -0,0 +1,84 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoverfault + + +#-------------------------------------------------------------------------- +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} +#-------------------------------------------------------------------------- + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, hex(randomblob(1000)), randomblob(2000)); + CREATE INDEX i1 ON t1(b, c); + ANALYZE; +} +faultsim_save_and_close + +do_faultsim_test 1 -faults oom* -prep { + catch { db2 close } + faultsim_restore_and_reopen +} -body { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} -test { + faultsim_test_result {0 {}} {1 {}} + if {$testrc==0} { + sqlite3 db2 test.db2 + compare_dbs db db2 + db2 close + } +} + +faultsim_restore_and_reopen +do_execsql_test 2.0 { + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, hex(randomblob(1000)), hex(randomblob(2000))); + PRAGMA writable_schema = 1; + DELETE FROM sqlite_schema WHERE name='t2'; +} +faultsim_save_and_close + +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R run + $R finish +} -test { + faultsim_test_result {0 {}} {1 {}} +} + +finish_test + diff --git a/ext/recover/recoverfault2.test b/ext/recover/recoverfault2.test new file mode 100644 index 0000000000..e80d480ce1 --- /dev/null +++ b/ext/recover/recoverfault2.test @@ -0,0 +1,102 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoverfault2 + + +#-------------------------------------------------------------------------- +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} +#-------------------------------------------------------------------------- + +do_execsql_test 1.0 " + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(2, '\012hello\015world\012today\n'); +" +faultsim_save_and_close + +proc my_sql_hook {sql} { + lappend ::lSql $sql + return 0 +} + +do_faultsim_test 1 -faults oom* -prep { + catch { db2 close } + faultsim_restore_and_reopen + set ::lSql [list] +} -body { + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R run + $R finish +} -test { + faultsim_test_result {0 {}} {1 {}} + if {$testrc==0} { + sqlite3 db2 "" + db2 eval [join $::lSql ";"] + compare_dbs db db2 + db2 close + } +} + +ifcapable utf16 { + reset_db + do_execsql_test 2.0 " + PRAGMA encoding='utf-16'; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(2, '\012hello\015world\012today\n'); + " + faultsim_save_and_close + + proc my_sql_hook {sql} { + lappend ::lSql $sql + return 0 + } + + do_faultsim_test 2 -faults oom-t* -prep { + catch { db2 close } + faultsim_restore_and_reopen + set ::lSql [list] + } -body { + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R run + $R finish + } -test { + faultsim_test_result {0 {}} {1 {}} + if {$testrc==0} { + sqlite3 db2 "" + db2 eval [join $::lSql ";"] + compare_dbs db db2 + db2 close + } + } +} + + + +finish_test + diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test new file mode 100644 index 0000000000..c6acbb2f42 --- /dev/null +++ b/ext/recover/recoverold.test @@ -0,0 +1,189 @@ +# 2019 April 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoverold + +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "sql: $sql" + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} + +proc do_recover_test {tn {tsql {}} {res {}}} { + forcedelete test.db2 + forcedelete rstate.db + + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R run + $R finish + + sqlite3 db2 test.db2 + + if {$tsql==""} { + uplevel [list do_test $tn.1 [list compare_dbs db db2] {}] + } else { + uplevel [list do_execsql_test -db db2 $tn.1 $tsql $res] + } + db2 close + + forcedelete test.db2 + forcedelete rstate.db + + set ::sqlhook [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R config lostandfound lost_and_found + $R run + $R finish + + sqlite3 db2 test.db2 + db2 eval [join $::sqlhook ";"] + + + db cache flush + if {$tsql==""} { + compare_dbs db db2 + uplevel [list do_test $tn.sql [list compare_dbs db db2] {}] + } else { + uplevel [list do_execsql_test -db db2 $tn.sql $tsql $res] + } + db2 close +} + +proc my_sql_hook {sql} { + lappend ::sqlhook $sql + return 0 +} + + +set doc { + hello + world +} +do_execsql_test 1.1.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 4, X'1234567800'); + INSERT INTO t1 VALUES(2, 'test', 8.1); + INSERT INTO t1 VALUES(3, $doc, 8.4); +} +do_recover_test 1.1.2 + +do_execsql_test 1.2.1 " + DELETE FROM t1; + INSERT INTO t1 VALUES(13, 'hello\r\nworld', 13); +" +do_recover_test 1.2.2 + +do_execsql_test 1.3.1 " + CREATE TABLE t2(i INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + INSERT INTO t2 VALUES(NULL, 1, 2); + INSERT INTO t2 VALUES(NULL, 3, 4); + INSERT INTO t2 VALUES(NULL, 5, 6); + CREATE TABLE t3(i INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + INSERT INTO t3 VALUES(NULL, 1, 2); + INSERT INTO t3 VALUES(NULL, 3, 4); + INSERT INTO t3 VALUES(NULL, 5, 6); + DELETE FROM t2; +" +do_recover_test 1.3.2 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.1.0 { + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); +} + +do_recover_test 2.1.1 + +do_execsql_test 2.2.0 { + PRAGMA writable_schema = 1; + DELETE FROM sqlite_master WHERE name='t1'; +} +do_recover_test 2.2.1 { + SELECT name FROM sqlite_master +} {lost_and_found} + +do_execsql_test 2.3.0 { + CREATE TABLE lost_and_found(a, b, c); +} +do_recover_test 2.3.1 { + SELECT name FROM sqlite_master +} {lost_and_found lost_and_found_0} + +do_execsql_test 2.4.0 { + CREATE TABLE lost_and_found_0(a, b, c); +} +do_recover_test 2.4.1 { + SELECT name FROM sqlite_master; + SELECT * FROM lost_and_found_1; +} {lost_and_found lost_and_found_0 lost_and_found_1 + 2 2 3 {} 2 3 1 + 2 2 3 {} 5 6 4 + 2 2 3 {} 8 9 7 +} + +do_execsql_test 2.5 { + CREATE TABLE x1(a, b, c); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO x1 SELECT i, i, hex(randomblob(500)) FROM s; + DROP TABLE x1; +} +do_recover_test 2.5.1 { + SELECT name FROM sqlite_master; + SELECT * FROM lost_and_found_1; +} {lost_and_found lost_and_found_0 lost_and_found_1 + 2 2 3 {} 2 3 1 + 2 2 3 {} 5 6 4 + 2 2 3 {} 8 9 7 +} + +ifcapable !secure_delete { + do_test 2.6 { + forcedelete test.db2 + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R config freelistcorrupt 1 + $R run + $R finish + sqlite3 db2 test.db2 + execsql { SELECT count(*) FROM lost_and_found_1; } db2 + } {103} + db2 close +} + +#------------------------------------------------------------------------- +breakpoint +reset_db +do_recover_test 3.0 + +finish_test diff --git a/ext/recover/recoverpgsz.test b/ext/recover/recoverpgsz.test new file mode 100644 index 0000000000..1a91f08459 --- /dev/null +++ b/ext/recover/recoverpgsz.test @@ -0,0 +1,100 @@ +# 2022 October 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] + +db close +sqlite3_test_control_pending_byte 0x1000000 + +set testprefix recoverpgsz + +foreach {pgsz bOverflow} { + 512 0 1024 0 2048 0 4096 0 8192 0 16384 0 32768 0 65536 0 + 512 1 1024 1 2048 1 4096 1 8192 1 16384 1 32768 1 65536 1 +} { + reset_db + execsql "PRAGMA page_size = $pgsz" + execsql "PRAGMA auto_vacuum = 0" + do_execsql_test 1.$pgsz.$bOverflow.1 { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(b, a, c); + INSERT INTO t1(a, b) VALUES(1, 2), (3, 4), (5, 6); + DELETE FROM t1 WHERE a=3; + } + if {$bOverflow} { + do_execsql_test 1.$pgsz.$bOverflow.1a { + UPDATE t1 SET c = randomblob(100000); + } + } + db close + + + set fd [open test.db] + fconfigure $fd -encoding binary -translation binary + seek $fd $pgsz + set pg1 [read $fd $pgsz] + set pg2 [read $fd $pgsz] + close $fd + + set fd2 [open test.db2 w] + fconfigure $fd2 -encoding binary -translation binary + seek $fd2 $pgsz + puts -nonewline $fd2 $pg1 + close $fd2 + + sqlite3 db2 test.db2 + do_test 1.$pgsz.$bOverflow.2 { + set R [sqlite3_recover_init db2 main test.db3] + $R run + $R finish + } {} + + sqlite3 db3 test.db3 + do_test 1.$pgsz.$bOverflow.3 { + db3 eval { SELECT * FROM sqlite_schema } + db3 eval { PRAGMA page_size } + } $pgsz + + db2 close + db3 close + + forcedelete test.db3 + forcedelete test.db2 + + set fd2 [open test.db2 w] + fconfigure $fd2 -encoding binary -translation binary + seek $fd2 $pgsz + puts -nonewline $fd2 $pg2 + close $fd2 + + sqlite3 db2 test.db2 + do_test 1.$pgsz.$bOverflow.4 { + set R [sqlite3_recover_init db2 main test.db3] + $R run + $R finish + } {} + + sqlite3 db3 test.db3 + do_test 1.$pgsz.$bOverflow.5 { + db3 eval { SELECT * FROM sqlite_schema } + db3 eval { PRAGMA page_size } + } $pgsz + + db2 close + db3 close +} + + +finish_test + + + diff --git a/ext/recover/recoverrowid.test b/ext/recover/recoverrowid.test new file mode 100644 index 0000000000..5855e84fa6 --- /dev/null +++ b/ext/recover/recoverrowid.test @@ -0,0 +1,50 @@ +# 2022 September 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the SQLITE_RECOVER_ROWIDS option. +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoverrowid + +proc recover {db bRowids output} { + forcedelete $output + + set R [sqlite3_recover_init db main test.db2] + $R config rowids $bRowids + $R run + $R finish +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3), (4, 4); + DELETE FROM t1 WHERE a IN (1, 3); +} + +do_test 1.1 { + recover db 0 test.db2 + sqlite3 db2 test.db2 + execsql { SELECT rowid, a, b FROM t1 ORDER BY rowid} db2 +} {1 2 2 2 4 4} + +do_test 1.2 { + db2 close + recover db 1 test.db2 + sqlite3 db2 test.db2 + execsql { SELECT rowid, a, b FROM t1 ORDER BY rowid} db2 +} {2 2 2 4 4 4} +db2 close + + + + +finish_test diff --git a/ext/recover/recoverslowidx.test b/ext/recover/recoverslowidx.test new file mode 100644 index 0000000000..269105113d --- /dev/null +++ b/ext/recover/recoverslowidx.test @@ -0,0 +1,87 @@ +# 2022 September 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the SQLITE_RECOVER_SLOWINDEXES option. +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoverslowidx + +do_execsql_test 1.0 { + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3), (4, 4); +} + +proc my_sql_hook {sql} { + lappend ::lSql $sql + return 0 +} + +do_test 1.1 { + set lSql [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + while {[$R step]==0} { } + $R finish +} {} + +do_test 1.2 { + set lSql +} [list {*}{ + {BEGIN} + {PRAGMA writable_schema = on} + {PRAGMA encoding = 'UTF-8'} + {PRAGMA page_size = '1024'} + {PRAGMA auto_vacuum = '0'} + {PRAGMA user_version = '0'} + {PRAGMA application_id = '0'} + {CREATE TABLE t1(a, b)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (1, 1, 1)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (2, 2, 2)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (3, 3, 3)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (4, 4, 4)} + {CREATE INDEX i1 ON t1(a)} + {PRAGMA writable_schema = off} + {COMMIT} +}] + +do_test 1.3 { + set lSql [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R config slowindexes 1 + while {[$R step]==0} { } + $R finish +} {} + +do_test 1.4 { + set lSql +} [list {*}{ + {BEGIN} + {PRAGMA writable_schema = on} + {PRAGMA encoding = 'UTF-8'} + {PRAGMA page_size = '1024'} + {PRAGMA auto_vacuum = '0'} + {PRAGMA user_version = '0'} + {PRAGMA application_id = '0'} + {CREATE TABLE t1(a, b)} + {CREATE INDEX i1 ON t1(a)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (1, 1, 1)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (2, 2, 2)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (3, 3, 3)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (4, 4, 4)} + {PRAGMA writable_schema = off} + {COMMIT} +}] + + +finish_test + diff --git a/ext/recover/recoversql.test b/ext/recover/recoversql.test new file mode 100644 index 0000000000..0a6726727d --- /dev/null +++ b/ext/recover/recoversql.test @@ -0,0 +1,52 @@ +# 2022 September 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoversql + +do_execsql_test 1.0 { + CREATE TABLE "x.1" (x, y); + INSERT INTO "x.1" VALUES(1, 1), (2, 2), (3, 3); + CREATE INDEX "i.1" ON "x.1"(y, x); +} + +proc sql_hook {sql} { + incr ::iSqlHook + if {$::iSqlHook==$::sql_hook_cnt} { return 4 } + return 0 +} + +do_test 1.1 { + set ::sql_hook_cnt -1 + set ::iSqlHook 0 + set R [sqlite3_recover_init_sql db main sql_hook] + $R run + $R finish +} {} + +set nSqlCall $iSqlHook + +for {set ii 1} {$ii<$nSqlCall} {incr ii} { + set iSqlHook 0 + set sql_hook_cnt $ii + do_test 1.$ii.a { + set R [sqlite3_recover_init_sql db main sql_hook] + $R run + } {1} + do_test 1.$ii.b { + list [catch { $R finish } msg] $msg + } {1 {callback returned an error - 4}} +} + + +finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c new file mode 100644 index 0000000000..1dd63fb3f4 --- /dev/null +++ b/ext/recover/sqlite3recover.c @@ -0,0 +1,2851 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + + +#include "sqlite3recover.h" +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Declaration for public API function in file dbdata.c. This may be called +** with NULL as the final two arguments to register the sqlite_dbptr and +** sqlite_dbdata virtual tables with a database handle. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +typedef unsigned int u32; +typedef unsigned char u8; +typedef sqlite3_int64 i64; + +typedef struct RecoverTable RecoverTable; +typedef struct RecoverColumn RecoverColumn; + +/* +** When recovering rows of data that can be associated with table +** definitions recovered from the sqlite_schema table, each table is +** represented by an instance of the following object. +** +** iRoot: +** The root page in the original database. Not necessarily (and usually +** not) the same in the recovered database. +** +** zTab: +** Name of the table. +** +** nCol/aCol[]: +** aCol[] is an array of nCol columns. In the order in which they appear +** in the table. +** +** bIntkey: +** Set to true for intkey tables, false for WITHOUT ROWID. +** +** iRowidBind: +** Each column in the aCol[] array has associated with it the index of +** the bind parameter its values will be bound to in the INSERT statement +** used to construct the output database. If the table does has a rowid +** but not an INTEGER PRIMARY KEY column, then iRowidBind contains the +** index of the bind paramater to which the rowid value should be bound. +** Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY +** KEY column, then the rowid value should be bound to the index associated +** with the column. +** +** pNext: +** All RecoverTable objects used by the recovery operation are allocated +** and populated as part of creating the recovered database schema in +** the output database, before any non-schema data are recovered. They +** are then stored in a singly-linked list linked by this variable beginning +** at sqlite3_recover.pTblList. +*/ +struct RecoverTable { + u32 iRoot; /* Root page in original database */ + char *zTab; /* Name of table */ + int nCol; /* Number of columns in table */ + RecoverColumn *aCol; /* Array of columns */ + int bIntkey; /* True for intkey, false for without rowid */ + int iRowidBind; /* If >0, bind rowid to INSERT here */ + RecoverTable *pNext; +}; + +/* +** Each database column is represented by an instance of the following object +** stored in the RecoverTable.aCol[] array of the associated table. +** +** iField: +** The index of the associated field within database records. Or -1 if +** there is no associated field (e.g. for virtual generated columns). +** +** iBind: +** The bind index of the INSERT statement to bind this columns values +** to. Or 0 if there is no such index (iff (iField<0)). +** +** bIPK: +** True if this is the INTEGER PRIMARY KEY column. +** +** zCol: +** Name of column. +** +** eHidden: +** A RECOVER_EHIDDEN_* constant value (see below for interpretation of each). +*/ +struct RecoverColumn { + int iField; /* Field in record on disk */ + int iBind; /* Binding to use in INSERT */ + int bIPK; /* True for IPK column */ + char *zCol; + int eHidden; +}; + +#define RECOVER_EHIDDEN_NONE 0 /* Normal database column */ +#define RECOVER_EHIDDEN_HIDDEN 1 /* Column is __HIDDEN__ */ +#define RECOVER_EHIDDEN_VIRTUAL 2 /* Virtual generated column */ +#define RECOVER_EHIDDEN_STORED 3 /* Stored generated column */ + +/* +** Bitmap object used to track pages in the input database. Allocated +** and manipulated only by the following functions: +** +** recoverBitmapAlloc() +** recoverBitmapFree() +** recoverBitmapSet() +** recoverBitmapQuery() +** +** nPg: +** Largest page number that may be stored in the bitmap. The range +** of valid keys is 1 to nPg, inclusive. +** +** aElem[]: +** Array large enough to contain a bit for each key. For key value +** iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32]. +** In other words, the following is true if bit iKey is set, or +** false if it is clear: +** +** (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0 +*/ +typedef struct RecoverBitmap RecoverBitmap; +struct RecoverBitmap { + i64 nPg; /* Size of bitmap */ + u32 aElem[1]; /* Array of 32-bit bitmasks */ +}; + +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data for tables identified in the recovered schema (state +** RECOVER_STATE_WRITING). +*/ +typedef struct RecoverStateW1 RecoverStateW1; +struct RecoverStateW1 { + sqlite3_stmt *pTbls; + sqlite3_stmt *pSel; + sqlite3_stmt *pInsert; + int nInsert; + + RecoverTable *pTab; /* Table currently being written */ + int nMax; /* Max column count in any schema table */ + sqlite3_value **apVal; /* Array of nMax values */ + int nVal; /* Number of valid entries in apVal[] */ + int bHaveRowid; + i64 iRowid; + i64 iPrevPage; + int iPrevCell; +}; + +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data destined for the lost and found table (states +** RECOVER_STATE_LOSTANDFOUND[123]). +*/ +typedef struct RecoverStateLAF RecoverStateLAF; +struct RecoverStateLAF { + RecoverBitmap *pUsed; + i64 nPg; /* Size of db in pages */ + sqlite3_stmt *pAllAndParent; + sqlite3_stmt *pMapInsert; + sqlite3_stmt *pMaxField; + sqlite3_stmt *pUsedPages; + sqlite3_stmt *pFindRoot; + sqlite3_stmt *pInsert; /* INSERT INTO lost_and_found ... */ + sqlite3_stmt *pAllPage; + sqlite3_stmt *pPageData; + sqlite3_value **apVal; + int nMaxField; +}; + +/* +** Main recover handle structure. +*/ +struct sqlite3_recover { + /* Copies of sqlite3_recover_init[_sql]() parameters */ + sqlite3 *dbIn; /* Input database */ + char *zDb; /* Name of input db ("main" etc.) */ + char *zUri; /* URI for output database */ + void *pSqlCtx; /* SQL callback context */ + int (*xSql)(void*,const char*); /* Pointer to SQL callback function */ + + /* Values configured by sqlite3_recover_config() */ + char *zStateDb; /* State database to use (or NULL) */ + char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ + int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ + int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ + int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ + + int pgsz; + int detected_pgsz; + int nReserve; + u8 *pPage1Disk; + u8 *pPage1Cache; + + /* Error code and error message */ + int errCode; /* For sqlite3_recover_errcode() */ + char *zErrMsg; /* For sqlite3_recover_errmsg() */ + + int eState; + int bCloseTransaction; + + /* Variables used with eState==RECOVER_STATE_WRITING */ + RecoverStateW1 w1; + + /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */ + RecoverStateLAF laf; + + /* Fields used within sqlite3_recover_run() */ + sqlite3 *dbOut; /* Output database */ + sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ + RecoverTable *pTblList; /* List of tables recovered from schema */ +}; + +/* +** The various states in which an sqlite3_recover object may exist: +** +** RECOVER_STATE_INIT: +** The object is initially created in this state. sqlite3_recover_step() +** has yet to be called. This is the only state in which it is permitted +** to call sqlite3_recover_config(). +** +** RECOVER_STATE_WRITING: +** +** RECOVER_STATE_LOSTANDFOUND1: +** State to populate the bitmap of pages used by other tables or the +** database freelist. +** +** RECOVER_STATE_LOSTANDFOUND2: +** Populate the recovery.map table - used to figure out a "root" page +** for each lost page from in the database from which records are +** extracted. +** +** RECOVER_STATE_LOSTANDFOUND3: +** Populate the lost-and-found table itself. +*/ +#define RECOVER_STATE_INIT 0 +#define RECOVER_STATE_WRITING 1 +#define RECOVER_STATE_LOSTANDFOUND1 2 +#define RECOVER_STATE_LOSTANDFOUND2 3 +#define RECOVER_STATE_LOSTANDFOUND3 4 +#define RECOVER_STATE_SCHEMA2 5 +#define RECOVER_STATE_DONE 6 + + +/* +** Global variables used by this extension. +*/ +typedef struct RecoverGlobal RecoverGlobal; +struct RecoverGlobal { + const sqlite3_io_methods *pMethods; + sqlite3_recover *p; +}; +static RecoverGlobal recover_g; + +/* +** Use this static SQLite mutex to protect the globals during the +** first call to sqlite3_recover_step(). +*/ +#define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2 + + +/* +** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). +*/ +#define RECOVER_ROWID_DEFAULT 1 + +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 +# define recoverEnterMutex() +# define recoverLeaveMutex() +# define recoverAssertMutexHeld() +#else +static void recoverEnterMutex(void){ + sqlite3_mutex_enter(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); +} +static void recoverLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); +} +static void recoverAssertMutexHeld(void){ + assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) ); +} +#endif + + +/* +** Like strlen(). But handles NULL pointer arguments. +*/ +static int recoverStrlen(const char *zStr){ + int nRet = 0; + if( zStr ){ + while( zStr[nRet] ) nRet++; + } + return nRet; +} + +/* +** This function is a no-op if the recover handle passed as the first +** argument already contains an error (if p->errCode!=SQLITE_OK). +** +** Otherwise, an attempt is made to allocate, zero and return a buffer nByte +** bytes in size. If successful, a pointer to the new buffer is returned. Or, +** if an OOM error occurs, NULL is returned and the handle error code +** (p->errCode) set to SQLITE_NOMEM. +*/ +static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ + void *pRet = 0; + assert( nByte>0 ); + if( p->errCode==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + p->errCode = SQLITE_NOMEM; + } + } + return pRet; +} + +/* +** Set the error code and error message for the recover handle passed as +** the first argument. The error code is set to the value of parameter +** errCode. +** +** Parameter zFmt must be a printf() style formatting string. The handle +** error message is set to the result of using any trailing arguments for +** parameter substitutions in the formatting string. +** +** For example: +** +** recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename); +*/ +static int recoverError( + sqlite3_recover *p, + int errCode, + const char *zFmt, ... +){ + char *z = 0; + va_list ap; + va_start(ap, zFmt); + if( zFmt ){ + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + } + sqlite3_free(p->zErrMsg); + p->zErrMsg = z; + p->errCode = errCode; + return errCode; +} + + +/* +** This function is a no-op if p->errCode is initially other than SQLITE_OK. +** In this case it returns NULL. +** +** Otherwise, an attempt is made to allocate and return a bitmap object +** large enough to store a bit for all page numbers between 1 and nPg, +** inclusive. The bitmap is initially zeroed. +*/ +static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ + int nElem = (nPg+1+31) / 32; + int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); + RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte); + + if( pRet ){ + pRet->nPg = nPg; + } + return pRet; +} + +/* +** Free a bitmap object allocated by recoverBitmapAlloc(). +*/ +static void recoverBitmapFree(RecoverBitmap *pMap){ + sqlite3_free(pMap); +} + +/* +** Set the bit associated with page iPg in bitvec pMap. +*/ +static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + pMap->aElem[iElem] |= (((u32)1) << iBit); + } +} + +/* +** Query bitmap object pMap for the state of the bit associated with page +** iPg. Return 1 if it is set, or 0 otherwise. +*/ +static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ + int ret = 1; + if( iPg<=pMap->nPg && iPg>0 ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; + } + return ret; +} + +/* +** Set the recover handle error to the error code and message returned by +** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database +** handle db. +*/ +static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ + return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, it attempts to prepare the SQL statement in zSql against +** database handle db. If successful, the statement handle is returned. +** Or, if an error occurs, NULL is returned and an error left in the +** recover handle. +*/ +static sqlite3_stmt *recoverPrepare( + sqlite3_recover *p, + sqlite3 *db, + const char *zSql +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ + recoverDbError(p, db); + } + } + return pStmt; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zFmt is used as a printf() style format string, +** along with any trailing arguments, to create an SQL statement. This +** SQL statement is prepared against database handle db and, if successful, +** the statment handle returned. Or, if an error occurs - either during +** the printf() formatting or when preparing the resulting SQL - an +** error code and message are left in the recover handle. +*/ +static sqlite3_stmt *recoverPreparePrintf( + sqlite3_recover *p, + sqlite3 *db, + const char *zFmt, ... +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + p->errCode = SQLITE_NOMEM; + }else{ + pStmt = recoverPrepare(p, db, z); + sqlite3_free(z); + } + } + return pStmt; +} + +/* +** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +** +** This function returns a copy of the statement handle pointer passed +** as the second argument. +*/ +static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){ + recoverDbError(p, sqlite3_db_handle(pStmt)); + } + return pStmt; +} + +/* +** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +*/ +static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, db); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this +** case. +** +** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK. +** Or, if an error occurs, leave an error code and message in the recover +** handle and return a copy of the error code. +*/ +static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc ){ + recoverDbError(p, db); + } + } + return p->errCode; +} + +/* +** Bind the value pVal to parameter iBind of statement pStmt. Leave an +** error in the recover handle passed as the first argument if an error +** (e.g. an OOM) occurs. +*/ +static void recoverBindValue( + sqlite3_recover *p, + sqlite3_stmt *pStmt, + int iBind, + sqlite3_value *pVal +){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_bind_value(pStmt, iBind, pVal); + if( rc ) recoverError(p, rc, 0); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). NULL is returned in this case. +** +** Otherwise, an attempt is made to interpret zFmt as a printf() style +** formatting string and the result of using the trailing arguments for +** parameter substitution with it written into a buffer obtained from +** sqlite3_malloc(). If successful, a pointer to the buffer is returned. +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +** +** Or, if an error occurs, an error code and message is left in the recover +** handle and NULL returned. +*/ +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( p->errCode==SQLITE_OK ){ + if( z==0 ) p->errCode = SQLITE_NOMEM; + }else{ + sqlite3_free(z); + z = 0; + } + return z; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). Zero is returned in this case. +** +** Otherwise, execute "PRAGMA page_count" against the input database. If +** successful, return the integer result. Or, if an error occurs, leave an +** error code and error message in the sqlite3_recover handle and return +** zero. +*/ +static i64 recoverPageCount(sqlite3_recover *p){ + i64 nPg = 0; + if( p->errCode==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + if( pStmt ){ + sqlite3_step(pStmt); + nPg = sqlite3_column_int64(pStmt, 0); + } + recoverFinalize(p, pStmt); + } + return nPg; +} + +/* +** Implementation of SQL scalar function "read_i32". The first argument to +** this function must be a blob. The second a non-negative integer. This +** function reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +** +** SELECT read_i32(, ) +*/ +static void recoverReadI32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + assert( argc==2 ); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]) & 0xFFFF; + + if( (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + i64 iVal = ((i64)a[0]<<24) + + ((i64)a[1]<<16) + + ((i64)a[2]<< 8) + + ((i64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + +/* +** Implementation of SQL scalar function "page_is_used". This function +** is used as part of the procedure for locating orphan rows for the +** lost-and-found table, and it depends on those routines having populated +** the sqlite3_recover.laf.pUsed variable. +** +** The only argument to this function is a page-number. It returns true +** if the page has already been used somehow during data recovery, or false +** otherwise. +** +** SELECT page_is_used(); +*/ +static void recoverPageIsUsed( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + assert( nArg==1 ); + sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno)); +} + +/* +** The implementation of a user-defined SQL function invoked by the +** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages +** of the database being recovered. +** +** This function always takes a single integer argument. If the argument +** is zero, then the value returned is the number of pages in the db being +** recovered. If the argument is greater than zero, it is a page number. +** The value returned in this case is an SQL blob containing the data for +** the identified page of the db being recovered. e.g. +** +** SELECT getpage(0); -- return number of pages in db +** SELECT getpage(4); -- return page 4 of db as a blob of data +*/ +static void recoverGetPage( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + sqlite3_stmt *pStmt = 0; + + assert( nArg==1 ); + if( pgno==0 ){ + i64 nPg = recoverPageCount(p); + sqlite3_result_int64(pCtx, nPg); + return; + }else{ + if( p->pGetPage==0 ){ + pStmt = p->pGetPage = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); + }else if( p->errCode==SQLITE_OK ){ + pStmt = p->pGetPage; + } + + if( pStmt ){ + sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + const u8 *aPg; + int nPg; + assert( p->errCode==SQLITE_OK ); + aPg = sqlite3_column_blob(pStmt, 0); + nPg = sqlite3_column_bytes(pStmt, 0); + if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ + aPg = p->pPage1Disk; + } + sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT); + } + recoverReset(p, pStmt); + } + } + + if( p->errCode ){ + if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); + sqlite3_result_error_code(pCtx, p->errCode); + } +} + +/* +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *recoverUnusedString( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + +/* +** Implementation of scalar SQL function "escape_crnl". The argument passed to +** this function is the output of built-in function quote(). If the first +** character of the input is "'", indicating that the value passed to quote() +** was a text value, then this function searches the input for "\n" and "\r" +** characters and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy of the input +** is returned. +*/ +static void recoverEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + if( zText && zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, attempt to populate temporary table "recovery.schema" with the +** parts of the database schema that can be extracted from the input database. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. It is not considered an error if part of all of +** the database schema cannot be recovered due to corruption. +*/ +static int recoverCacheSchema(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "WITH RECURSIVE pages(p) AS (" + " SELECT 1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" + ")" + "INSERT INTO recovery.schema SELECT" + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" + " SELECT p FROM pages" + ") GROUP BY pgno, cell" + ); +} + +/* +** If this recover handle is not in SQL callback mode (i.e. was not created +** using sqlite3_recover_init_sql()) of if an error has already occurred, +** this function is a no-op. Otherwise, issue a callback with SQL statement +** zSql as the parameter. +** +** If the callback returns non-zero, set the recover handle error code to +** the value returned (so that the caller will abandon processing). +*/ +static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ + if( p->errCode==SQLITE_OK && p->xSql ){ + int res = p->xSql(p->pSqlCtx, zSql); + if( res ){ + recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); + } + } +} + +/* +** Transfer the following settings from the input database to the output +** database: +** +** + page-size, +** + auto-vacuum settings, +** + database encoding, +** + user-version (PRAGMA user_version), and +** + application-id (PRAGMA application_id), and +*/ +static void recoverTransferSettings(sqlite3_recover *p){ + const char *aPragma[] = { + "encoding", + "page_size", + "auto_vacuum", + "user_version", + "application_id" + }; + int ii; + + /* Truncate the output database to 0 pages in size. This is done by + ** opening a new, empty, temp db, then using the backup API to clobber + ** any existing output db with a copy of it. */ + if( p->errCode==SQLITE_OK ){ + sqlite3 *db2 = 0; + int rc = sqlite3_open("", &db2); + if( rc!=SQLITE_OK ){ + recoverDbError(p, db2); + return; + } + + for(ii=0; iidbIn, "PRAGMA %Q.%s", p->zDb, zPrag); + if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ + const char *zArg = (const char*)sqlite3_column_text(p1, 0); + char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg); + recoverSqlCallback(p, z2); + recoverExec(p, db2, z2); + sqlite3_free(z2); + if( zArg==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + } + recoverFinalize(p, p1); + } + recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;"); + + if( p->errCode==SQLITE_OK ){ + sqlite3 *db = p->dbOut; + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); + if( pBackup ){ + sqlite3_backup_step(pBackup, -1); + p->errCode = sqlite3_backup_finish(pBackup); + }else{ + recoverDbError(p, db); + } + } + + sqlite3_close(db2); + } +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, an attempt is made to open the output database, attach +** and create the schema of the temporary database used to store +** intermediate data, and to register all required user functions and +** virtual table modules with the output handle. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. +*/ +static int recoverOpenOutput(sqlite3_recover *p){ + struct Func { + const char *zName; + int nArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value **); + } aFunc[] = { + { "getpage", 1, recoverGetPage }, + { "page_is_used", 1, recoverPageIsUsed }, + { "read_i32", 2, recoverReadI32 }, + { "escape_crnl", 1, recoverEscapeCrnl }, + }; + + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + sqlite3 *db = 0; /* New database handle */ + int ii; /* For iterating through aFunc[] */ + + assert( p->dbOut==0 ); + + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ + recoverDbError(p, db); + } + + /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. + ** These two are registered with the output database handle - this + ** module depends on the input handle supporting the sqlite_dbpage + ** virtual table only. */ + if( p->errCode==SQLITE_OK ){ + p->errCode = sqlite3_dbdata_init(db, 0, 0); + } + + /* Register the custom user-functions with the output handle. */ + for(ii=0; p->errCode==SQLITE_OK && iierrCode = sqlite3_create_function(db, aFunc[ii].zName, + aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 + ); + } + + p->dbOut = db; + return p->errCode; +} + +/* +** Attach the auxiliary database 'recovery' to the output database handle. +** This temporary database is used during the recovery process and then +** discarded. +*/ +static void recoverOpenRecovery(sqlite3_recover *p){ + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); + recoverExec(p, p->dbOut, zSql); + recoverExec(p, p->dbOut, + "PRAGMA writable_schema = 1;" + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + ); + sqlite3_free(zSql); +} + + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zName must be the name of a table that has just been +** created in the output database. This function queries the output db +** for the schema of said table, and creates a RecoverTable object to +** store the schema in memory. The new RecoverTable object is linked into +** the list at sqlite3_recover.pTblList. +** +** Parameter iRoot must be the root page of table zName in the INPUT +** database. +*/ +static void recoverAddTable( + sqlite3_recover *p, + const char *zName, /* Name of table created in output db */ + i64 iRoot /* Root page of same table in INPUT db */ +){ + sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, + "PRAGMA table_xinfo(%Q)", zName + ); + + if( pStmt ){ + int iPk = -1; + int iBind = 1; + RecoverTable *pNew = 0; + int nCol = 0; + int nName = recoverStrlen(zName); + int nByte = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol++; + nByte += (sqlite3_column_bytes(pStmt, 1)+1); + } + nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; + recoverReset(p, pStmt); + + pNew = recoverMalloc(p, nByte); + if( pNew ){ + int i = 0; + int iField = 0; + char *csr = 0; + pNew->aCol = (RecoverColumn*)&pNew[1]; + pNew->zTab = csr = (char*)&pNew->aCol[nCol]; + pNew->nCol = nCol; + pNew->iRoot = iRoot; + memcpy(csr, zName, nName); + csr += nName+1; + + for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int iPKF = sqlite3_column_int(pStmt, 5); + int n = sqlite3_column_bytes(pStmt, 1); + const char *z = (const char*)sqlite3_column_text(pStmt, 1); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + int eHidden = sqlite3_column_int(pStmt, 6); + + if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i; + if( iPKF>1 ) iPk = -2; + pNew->aCol[i].zCol = csr; + pNew->aCol[i].eHidden = eHidden; + if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){ + pNew->aCol[i].iField = -1; + }else{ + pNew->aCol[i].iField = iField++; + } + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + pNew->aCol[i].iBind = iBind++; + } + memcpy(csr, z, n); + csr += (n+1); + } + + pNew->pNext = p->pTblList; + p->pTblList = pNew; + pNew->bIntkey = 1; + } + + recoverFinalize(p, pStmt); + + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); + while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + int iField = sqlite3_column_int(pStmt, 0); + int iCol = sqlite3_column_int(pStmt, 1); + + assert( iFieldnCol && iColnCol ); + pNew->aCol[iCol].iField = iField; + + pNew->bIntkey = 0; + iPk = -2; + } + recoverFinalize(p, pStmt); + + if( p->errCode==SQLITE_OK ){ + if( iPk>=0 ){ + pNew->aCol[iPk].bIPK = 1; + }else if( pNew->bIntkey ){ + pNew->iRowidBind = iBind++; + } + } + } +} + +/* +** This function is called after recoverCacheSchema() has cached those parts +** of the input database schema that could be recovered in temporary table +** "recovery.schema". This function creates in the output database copies +** of all parts of that schema that must be created before the tables can +** be populated. Specifically, this means: +** +** * all tables that are not VIRTUAL, and +** * UNIQUE indexes. +** +** If the recovery handle uses SQL callbacks, then callbacks containing +** the associated "CREATE TABLE" and "CREATE INDEX" statements are made. +** +** Additionally, records are added to the sqlite_schema table of the +** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE +** records are written directly to sqlite_schema, not actually executed. +** If the handle is in SQL callback mode, then callbacks are invoked +** with equivalent SQL statements. +*/ +static int recoverWriteSchema1(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + sqlite3_stmt *pTblname = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" + " SELECT rootpage, name, sql, " + " type='table', " + " sql LIKE 'create virtual%'," + " (type='index' AND (sql LIKE '%unique%' OR ?1))" + " FROM recovery.schema" + ")" + "SELECT rootpage, tbl, isVirtual, name, sql" + " FROM dbschema " + " WHERE tbl OR isIndex" + " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" + ); + + pTblname = recoverPrepare(p, p->dbOut, + "SELECT name FROM sqlite_schema " + "WHERE type='table' ORDER BY rowid DESC LIMIT 1" + ); + + if( pSelect ){ + sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + int bTable = sqlite3_column_int(pSelect, 1); + int bVirtual = sqlite3_column_int(pSelect, 2); + const char *zName = (const char*)sqlite3_column_text(pSelect, 3); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 4); + char *zFree = 0; + int rc = SQLITE_OK; + + if( bVirtual ){ + zSql = (const char*)(zFree = recoverMPrintf(p, + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + )); + } + rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + if( bTable && !bVirtual ){ + if( SQLITE_ROW==sqlite3_step(pTblname) ){ + const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); + recoverAddTable(p, zTbl, iRoot); + } + recoverReset(p, pTblname); + } + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + sqlite3_free(zFree); + } + } + recoverFinalize(p, pSelect); + recoverFinalize(p, pTblname); + + return p->errCode; +} + +/* +** This function is called after the output database has been populated. It +** adds all recovered schema elements that were not created in the output +** database by recoverWriteSchema1() - everything except for tables and +** UNIQUE indexes. Specifically: +** +** * views, +** * triggers, +** * non-UNIQUE indexes. +** +** If the recover handle is in SQL callback mode, then equivalent callbacks +** are issued to create the schema elements. +*/ +static int recoverWriteSchema2(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + + pSelect = recoverPrepare(p, p->dbOut, + p->bSlowIndexes ? + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND type!='index'" + : + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + ); + + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } + } + recoverFinalize(p, pSelect); + + return p->errCode; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). In this case it returns NULL. +** +** Otherwise, if the recover handle is configured to create an output +** database (was created by sqlite3_recover_init()), then this function +** prepares and returns an SQL statement to INSERT a new record into table +** pTab, assuming the first nField fields of a record extracted from disk +** are valid. +** +** For example, if table pTab is: +** +** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e); +** +** And nField is 4, then the SQL statement prepared and returned is: +** +** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3); +** +** In this case even though 4 values were extracted from the input db, +** only 3 are written to the output, as the generated STORED column +** cannot be written. +** +** If the recover handle is in SQL callback mode, then the SQL statement +** prepared is such that evaluating it returns a single row containing +** a single text value - itself an SQL statement similar to the above, +** except with SQL literals in place of the variables. For example: +** +** SELECT 'INSERT INTO (a, c, d) VALUES (' +** || quote(?1) || ', ' +** || quote(?2) || ', ' +** || quote(?3) || ')'; +** +** In either case, it is the responsibility of the caller to eventually +** free the statement handle using sqlite3_finalize(). +*/ +static sqlite3_stmt *recoverInsertStmt( + sqlite3_recover *p, + RecoverTable *pTab, + int nField +){ + sqlite3_stmt *pRet = 0; + const char *zSep = ""; + const char *zSqlSep = ""; + char *zSql = 0; + char *zFinal = 0; + char *zBind = 0; + int ii; + int bSql = p->xSql ? 1 : 0; + + if( nField<=0 ) return 0; + + assert( nField<=pTab->nCol ); + + zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); + + if( pTab->iRowidBind ){ + assert( pTab->bIntkey ); + zSql = recoverMPrintf(p, "%z_rowid_", zSql); + if( bSql ){ + zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind); + }else{ + zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); + } + zSqlSep = "||', '||"; + zSep = ", "; + } + + for(ii=0; iiaCol[ii].eHidden; + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); + zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); + + if( bSql ){ + zBind = recoverMPrintf(p, + "%z%sescape_crnl(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind + ); + zSqlSep = "||', '||"; + }else{ + zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); + } + zSep = ", "; + } + } + + if( bSql ){ + zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", + zSql, zBind + ); + }else{ + zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind); + } + + pRet = recoverPrepare(p, p->dbOut, zFinal); + sqlite3_free(zSql); + sqlite3_free(zBind); + sqlite3_free(zFinal); + + return pRet; +} + + +/* +** Search the list of RecoverTable objects at p->pTblList for one that +** has root page iRoot in the input database. If such an object is found, +** return a pointer to it. Otherwise, return NULL. +*/ +static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ + RecoverTable *pRet = 0; + for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); + return pRet; +} + +/* +** This function attempts to create a lost and found table within the +** output db. If successful, it returns a pointer to a buffer containing +** the name of the new table. It is the responsibility of the caller to +** eventually free this buffer using sqlite3_free(). +** +** If an error occurs, NULL is returned and an error code and error +** message left in the recover handle. +*/ +static char *recoverLostAndFoundCreate( + sqlite3_recover *p, /* Recover object */ + int nField /* Number of column fields in new table */ +){ + char *zTbl = 0; + sqlite3_stmt *pProbe = 0; + int ii = 0; + + pProbe = recoverPrepare(p, p->dbOut, + "SELECT 1 FROM sqlite_schema WHERE name=?" + ); + for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ + int bFail = 0; + if( ii<0 ){ + zTbl = recoverMPrintf(p, "%s", p->zLostAndFound); + }else{ + zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii); + } + + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pProbe) ){ + bFail = 1; + } + recoverReset(p, pProbe); + } + + if( bFail ){ + sqlite3_clear_bindings(pProbe); + sqlite3_free(zTbl); + zTbl = 0; + } + } + recoverFinalize(p, pProbe); + + if( zTbl ){ + const char *zSep = 0; + char *zField = 0; + char *zSql = 0; + + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; + for(ii=0; p->errCode==SQLITE_OK && iidbOut, zSql); + recoverSqlCallback(p, zSql); + sqlite3_free(zSql); + }else if( p->errCode==SQLITE_OK ){ + recoverError( + p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound + ); + } + + return zTbl; +} + +/* +** Synthesize and prepare an INSERT statement to write to the lost_and_found +** table in the output database. The name of the table is zTab, and it has +** nField c* fields. +*/ +static sqlite3_stmt *recoverLostAndFoundInsert( + sqlite3_recover *p, + const char *zTab, + int nField +){ + int nTotal = nField + 4; + int ii; + char *zBind = 0; + sqlite3_stmt *pRet = 0; + + if( p->xSql==0 ){ + for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind + ); + }else{ + const char *zSep = ""; + for(ii=0; iidbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind + ); + } + + sqlite3_free(zBind); + return pRet; +} + +/* +** Input database page iPg contains data that will be written to the +** lost-and-found table of the output database. This function attempts +** to identify the root page of the tree that page iPg belonged to. +** If successful, it sets output variable (*piRoot) to the page number +** of the root page and returns SQLITE_OK. Otherwise, if an error occurs, +** an SQLite error code is returned and the final value of *piRoot +** undefined. +*/ +static int recoverLostAndFoundFindRoot( + sqlite3_recover *p, + i64 iPg, + i64 *piRoot +){ + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->pFindRoot==0 ){ + pLaf->pFindRoot = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE p(pgno) AS (" + " SELECT ?" + " UNION" + " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno" + ") " + "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno " + " AND m.parent IS NULL" + ); + } + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg); + if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){ + *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0); + }else{ + *piRoot = iPg; + } + recoverReset(p, pLaf->pFindRoot); + } + return p->errCode; +} + +/* +** Recover data from page iPage of the input database and write it to +** the lost-and-found table in the output database. +*/ +static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_value **apVal = pLaf->apVal; + sqlite3_stmt *pPageData = pLaf->pPageData; + sqlite3_stmt *pInsert = pLaf->pInsert; + + int nVal = -1; + int iPrevCell = 0; + i64 iRoot = 0; + int bHaveRowid = 0; + i64 iRowid = 0; + int ii = 0; + + if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return; + sqlite3_bind_int64(pPageData, 1, iPage); + while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){ + int iCell = sqlite3_column_int64(pPageData, 0); + int iField = sqlite3_column_int64(pPageData, 1); + + if( iPrevCell!=iCell && nVal>=0 ){ + /* Insert the new row */ + sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */ + sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */ + sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ + if( bHaveRowid ){ + sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ + } + for(ii=0; iinMaxField ){ + sqlite3_value *pVal = sqlite3_column_value(pPageData, 2); + apVal[iField] = sqlite3_value_dup(pVal); + assert( iField==nVal || (nVal==-1 && iField==0) ); + nVal = iField+1; + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + } + + iPrevCell = iCell; + } + recoverReset(p, pPageData); + + for(ii=0; iilaf; + if( p->errCode==SQLITE_OK ){ + if( pLaf->pInsert==0 ){ + return SQLITE_DONE; + }else{ + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllPage); + if( res==SQLITE_ROW ){ + i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0); + if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){ + recoverLostAndFoundOnePage(p, iPage); + } + }else{ + recoverReset(p, pLaf->pAllPage); + return SQLITE_DONE; + } + } + } + } + return SQLITE_OK; +} + +/* +** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 +** state - during which the lost-and-found table of the output database +** is populated with recovered data that can not be assigned to any +** recovered schema object. +*/ +static void recoverLostAndFound3Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->nMaxField>0 ){ + char *zTab = 0; /* Name of lost_and_found table */ + + zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField); + pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField); + sqlite3_free(zTab); + + pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT ii FROM seq" , p->laf.nPg + ); + pLaf->pPageData = recoverPrepare(p, p->dbOut, + "SELECT cell, field, value " + "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? " + "UNION ALL " + "SELECT -1, -1, -1" + ); + + pLaf->apVal = (sqlite3_value**)recoverMalloc(p, + pLaf->nMaxField*sizeof(sqlite3_value*) + ); + } +} + +/* +** Initialize resources required in RECOVER_STATE_WRITING state - during which +** tables recovered from the schema of the input database are populated with +** recovered data. +*/ +static int recoverWriteDataInit(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + RecoverTable *pTbl = 0; + int nByte = 0; + + /* Figure out the maximum number of columns for any table in the schema */ + assert( p1->nMax==0 ); + for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ + if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; + } + + /* Allocate an array of (sqlite3_value*) in which to accumulate the values + ** that will be written to the output database in a single row. */ + nByte = sizeof(sqlite3_value*) * (p1->nMax+1); + p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte); + if( p1->apVal==0 ) return p->errCode; + + /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT + ** to loop through cells that appear to belong to a single table (pSel). */ + p1->pTbls = recoverPrepare(p, p->dbOut, + "SELECT rootpage FROM recovery.schema " + " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" + " ORDER BY (tbl_name='sqlite_sequence') ASC" + ); + p1->pSel = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE pages(page) AS (" + " SELECT ?1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages " + " WHERE pgno=page" + ") " + "SELECT page, cell, field, value " + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " + "UNION ALL " + "SELECT 0, 0, 0, 0" + ); + + return p->errCode; +} + +/* +** Clean up resources allocated by recoverWriteDataInit() (stuff in +** sqlite3_recover.w1). +*/ +static void recoverWriteDataCleanup(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + int ii; + for(ii=0; iinVal; ii++){ + sqlite3_value_free(p1->apVal[ii]); + } + sqlite3_free(p1->apVal); + recoverFinalize(p, p1->pInsert); + recoverFinalize(p, p1->pTbls); + recoverFinalize(p, p1->pSel); + memset(p1, 0, sizeof(*p1)); +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_WRITING state - during which tables recovered from the +** schema of the input database are populated with recovered data. +*/ +static int recoverWriteDataStep(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + sqlite3_stmt *pSel = p1->pSel; + sqlite3_value **apVal = p1->apVal; + + if( p->errCode==SQLITE_OK && p1->pTab==0 ){ + if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); + p1->pTab = recoverFindTable(p, iRoot); + + recoverFinalize(p, p1->pInsert); + p1->pInsert = 0; + + /* If this table is unknown, return early. The caller will invoke this + ** function again and it will move on to the next table. */ + if( p1->pTab==0 ) return p->errCode; + + /* If this is the sqlite_sequence table, delete any rows added by + ** earlier INSERT statements on tables with AUTOINCREMENT primary + ** keys before recovering its contents. The p1->pTbls SELECT statement + ** is rigged to deliver "sqlite_sequence" last of all, so we don't + ** worry about it being modified after it is recovered. */ + if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){ + recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); + recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); + } + + /* Bind the root page of this table within the original database to + ** SELECT statement p1->pSel. The SELECT statement will then iterate + ** through cells that look like they belong to table pTab. */ + sqlite3_bind_int64(pSel, 1, iRoot); + + p1->nVal = 0; + p1->bHaveRowid = 0; + p1->iPrevPage = -1; + p1->iPrevCell = -1; + }else{ + return SQLITE_DONE; + } + } + assert( p->errCode!=SQLITE_OK || p1->pTab ); + + if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + RecoverTable *pTab = p1->pTab; + + i64 iPage = sqlite3_column_int64(pSel, 0); + int iCell = sqlite3_column_int(pSel, 1); + int iField = sqlite3_column_int(pSel, 2); + sqlite3_value *pVal = sqlite3_column_value(pSel, 3); + int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); + + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); + + if( bNewCell ){ + int ii = 0; + if( p1->nVal>=0 ){ + if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){ + recoverFinalize(p, p1->pInsert); + p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal); + p1->nInsert = p1->nVal; + } + if( p1->nVal>0 ){ + sqlite3_stmt *pInsert = p1->pInsert; + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; + int iBind = pCol->iBind; + if( iBind>0 ){ + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, iBind, p1->iRowid); + }else if( pCol->iFieldnVal ){ + recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); + } + } + } + if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid); + } + if( SQLITE_ROW==sqlite3_step(pInsert) ){ + const char *z = (const char*)sqlite3_column_text(pInsert, 0); + recoverSqlCallback(p, z); + } + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); + } + } + + for(ii=0; iinVal; ii++){ + sqlite3_value_free(apVal[ii]); + apVal[ii] = 0; + } + p1->nVal = -1; + p1->bHaveRowid = 0; + } + + if( iPage!=0 ){ + if( iField<0 ){ + p1->iRowid = sqlite3_column_int64(pSel, 3); + assert( p1->nVal==-1 ); + p1->nVal = 0; + p1->bHaveRowid = 1; + }else if( iFieldnCol ){ + assert( apVal[iField]==0 ); + apVal[iField] = sqlite3_value_dup( pVal ); + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + p1->nVal = iField+1; + } + p1->iPrevCell = iCell; + p1->iPrevPage = iPage; + } + }else{ + recoverReset(p, pSel); + p1->pTab = 0; + } + + return p->errCode; +} + +/* +** Initialize resources required by sqlite3_recover_step() in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static void recoverLostAndFound1Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_stmt *pStmt = 0; + + assert( p->laf.pUsed==0 ); + pLaf->nPg = recoverPageCount(p); + pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg); + + /* Prepare a statement to iterate through all pages that are part of any tree + ** in the recoverable part of the input database schema to the bitmap. And, + ** if !p->bFreelistCorrupt, add all pages that appear to be part of the + ** freelist. */ + pStmt = recoverPrepare( + p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")," + "" + "roots(r) AS (" + " SELECT 1 UNION ALL" + " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + ")," + "used(page) AS (" + " SELECT r FROM roots" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), used " + " WHERE pgno=page" + ") " + "SELECT page FROM used" + " UNION ALL " + "SELECT freepgno FROM freelist WHERE NOT ?" + ); + if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); + pLaf->pUsedPages = pStmt; +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static int recoverLostAndFound1Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + int rc = p->errCode; + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pLaf->pUsedPages); + if( rc==SQLITE_ROW ){ + i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0); + recoverBitmapSet(pLaf->pUsed, iPg); + rc = SQLITE_OK; + }else{ + recoverFinalize(p, pLaf->pUsedPages); + pLaf->pUsedPages = 0; + } + } + return rc; +} + +/* +** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 +** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1 +** are sorted into sets that likely belonged to the same database tree. +*/ +static void recoverLostAndFound2Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + + assert( p->laf.pAllAndParent==0 ); + assert( p->laf.pMapInsert==0 ); + assert( p->laf.pMaxField==0 ); + assert( p->laf.nMaxField==0 ); + + pLaf->pMapInsert = recoverPrepare(p, p->dbOut, + "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)" + ); + pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT pgno, child FROM sqlite_dbptr('getpage()') " + " UNION ALL " + "SELECT NULL, ii FROM seq", p->laf.nPg + ); + pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut, + "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?" + ); +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified +** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged +** to the same database tree. +*/ +static int recoverLostAndFound2Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllAndParent); + if( res==SQLITE_ROW ){ + i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1); + if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){ + sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild); + sqlite3_bind_value(pLaf->pMapInsert, 2, + sqlite3_column_value(pLaf->pAllAndParent, 0) + ); + sqlite3_step(pLaf->pMapInsert); + recoverReset(p, pLaf->pMapInsert); + sqlite3_bind_int64(pLaf->pMaxField, 1, iChild); + if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){ + int nMax = sqlite3_column_int(pLaf->pMaxField, 0); + if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax; + } + recoverReset(p, pLaf->pMaxField); + } + }else{ + recoverFinalize(p, pLaf->pAllAndParent); + pLaf->pAllAndParent =0; + return SQLITE_DONE; + } + } + return p->errCode; +} + +/* +** Free all resources allocated as part of sqlite3_recover_step() calls +** in one of the RECOVER_STATE_LOSTANDFOUND[123] states. +*/ +static void recoverLostAndFoundCleanup(sqlite3_recover *p){ + recoverBitmapFree(p->laf.pUsed); + p->laf.pUsed = 0; + sqlite3_finalize(p->laf.pUsedPages); + sqlite3_finalize(p->laf.pAllAndParent); + sqlite3_finalize(p->laf.pMapInsert); + sqlite3_finalize(p->laf.pMaxField); + sqlite3_finalize(p->laf.pFindRoot); + sqlite3_finalize(p->laf.pInsert); + sqlite3_finalize(p->laf.pAllPage); + sqlite3_finalize(p->laf.pPageData); + p->laf.pUsedPages = 0; + p->laf.pAllAndParent = 0; + p->laf.pMapInsert = 0; + p->laf.pMaxField = 0; + p->laf.pFindRoot = 0; + p->laf.pInsert = 0; + p->laf.pAllPage = 0; + p->laf.pPageData = 0; + sqlite3_free(p->laf.apVal); + p->laf.apVal = 0; +} + +/* +** Free all resources allocated as part of sqlite3_recover_step() calls. +*/ +static void recoverFinalCleanup(sqlite3_recover *p){ + RecoverTable *pTab = 0; + RecoverTable *pNext = 0; + + recoverWriteDataCleanup(p); + recoverLostAndFoundCleanup(p); + + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + p->pTblList = 0; + sqlite3_finalize(p->pGetPage); + p->pGetPage = 0; + + { +#ifndef NDEBUG + int res = +#endif + sqlite3_close(p->dbOut); + assert( res==SQLITE_OK ); + } + p->dbOut = 0; +} + +/* +** Decode and return an unsigned 16-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU16(const u8 *a){ + return (((u32)a[0])<<8) + ((u32)a[1]); +} + +/* +** Decode and return an unsigned 32-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU32(const u8 *a){ + return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); +} + +/* +** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal) +** and return the number of bytes consumed. +*/ +static int recoverGetVarint(const u8 *a, i64 *pVal){ + sqlite3_uint64 u = 0; + int i; + for(i=0; i<8; i++){ + u = (u<<7) + (a[i]&0x7f); + if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } + } + u = (u<<8) + (a[i]&0xff); + *pVal = (sqlite3_int64)u; + return 9; +} + +/* +** The second argument points to a buffer n bytes in size. If this buffer +** or a prefix thereof appears to contain a well-formed SQLite b-tree page, +** return the page-size in bytes. Otherwise, if the buffer does not +** appear to contain a well-formed b-tree page, return 0. +*/ +static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ + u8 *aUsed = aTmp; + int nFrag = 0; + int nActual = 0; + int iFree = 0; + int nCell = 0; /* Number of cells on page */ + int iCellOff = 0; /* Offset of cell array in page */ + int iContent = 0; + int eType = 0; + int ii = 0; + + eType = (int)a[0]; + if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0; + + iFree = (int)recoverGetU16(&a[1]); + nCell = (int)recoverGetU16(&a[3]); + iContent = (int)recoverGetU16(&a[5]); + if( iContent==0 ) iContent = 65536; + nFrag = (int)a[7]; + + if( iContent>n ) return 0; + + memset(aUsed, 0, n); + memset(aUsed, 0xFF, iContent); + + /* Follow the free-list. This is the same format for all b-tree pages. */ + if( iFree && iFree<=iContent ) return 0; + while( iFree ){ + int iNext = 0; + int nByte = 0; + if( iFree>(n-4) ) return 0; + iNext = recoverGetU16(&a[iFree]); + nByte = recoverGetU16(&a[iFree+2]); + if( iFree+nByte>n ) return 0; + if( iNext && iNextiContent ) return 0; + for(ii=0; iin ){ + return 0; + } + if( eType==0x05 || eType==0x02 ) nByte += 4; + nByte += recoverGetVarint(&a[iOff+nByte], &nPayload); + if( eType==0x0D ){ + i64 dummy = 0; + nByte += recoverGetVarint(&a[iOff+nByte], &dummy); + } + if( eType!=0x05 ){ + int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23); + int M = ((n-12)*32/255)-23; + int K = M+((nPayload-M)%(n-4)); + + if( nPayloadn ){ + return 0; + } + for(iByte=iOff; iByte<(iOff+nByte); iByte++){ + if( aUsed[iByte]!=0 ){ + return 0; + } + aUsed[iByte] = 0xFF; + } + } + + nActual = 0; + for(ii=0; iipMethods!=&recover_methods ); + return pFd->pMethods->xClose(pFd); +} + +/* +** Write value v to buffer a[] as a 16-bit big-endian unsigned integer. +*/ +static void recoverPutU16(u8 *a, u32 v){ + a[0] = (v>>8) & 0x00FF; + a[1] = (v>>0) & 0x00FF; +} + +/* +** Write value v to buffer a[] as a 32-bit big-endian unsigned integer. +*/ +static void recoverPutU32(u8 *a, u32 v){ + a[0] = (v>>24) & 0x00FF; + a[1] = (v>>16) & 0x00FF; + a[2] = (v>>8) & 0x00FF; + a[3] = (v>>0) & 0x00FF; +} + +/* +** Detect the page-size of the database opened by file-handle pFd by +** searching the first part of the file for a well-formed SQLite b-tree +** page. If parameter nReserve is non-zero, then as well as searching for +** a b-tree page with zero reserved bytes, this function searches for one +** with nReserve reserved bytes at the end of it. +** +** If successful, set variable p->detected_pgsz to the detected page-size +** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page +** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or, +** if an error occurs (e.g. an IO or OOM error), then an SQLite error code +** is returned. The final value of p->detected_pgsz is undefined in this +** case. +*/ +static int recoverVfsDetectPagesize( + sqlite3_recover *p, /* Recover handle */ + sqlite3_file *pFd, /* File-handle open on input database */ + u32 nReserve, /* Possible nReserve value */ + i64 nSz /* Size of database file in bytes */ +){ + int rc = SQLITE_OK; + const int nMin = 512; + const int nMax = 65536; + const int nMaxBlk = 4; + u32 pgsz = 0; + int iBlk = 0; + u8 *aPg = 0; + u8 *aTmp = 0; + int nBlk = 0; + + aPg = (u8*)sqlite3_malloc(2*nMax); + if( aPg==0 ) return SQLITE_NOMEM; + aTmp = &aPg[nMax]; + + nBlk = (nSz+nMax-1)/nMax; + if( nBlk>nMaxBlk ) nBlk = nMaxBlk; + + do { + for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); + memset(aPg, 0, nMax); + rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); + if( rc==SQLITE_OK ){ + int pgsz2; + for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ + int iOff; + for(iOff=0; iOff(u32)p->detected_pgsz ){ + p->detected_pgsz = pgsz; + p->nReserve = nReserve; + } + if( nReserve==0 ) break; + nReserve = 0; + }while( 1 ); + + p->detected_pgsz = pgsz; + sqlite3_free(aPg); + return rc; +} + +/* +** The xRead() method of the wrapper VFS. This is used to intercept calls +** to read page 1 of the input database. +*/ +static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ + int rc = SQLITE_OK; + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + if( nByte==16 ){ + sqlite3_randomness(16, aBuf); + }else + if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){ + /* Ensure that the database has a valid header file. The only fields + ** that really matter to recovery are: + ** + ** + Database page size (16-bits at offset 16) + ** + Size of db in pages (32-bits at offset 28) + ** + Database encoding (32-bits at offset 56) + ** + ** Also preserved are: + ** + ** + first freelist page (32-bits at offset 32) + ** + size of freelist (32-bits at offset 36) + ** + ** We also try to preserve the auto-vacuum, incr-value, user-version + ** and application-id fields - all 32 bit quantities at offsets + ** 52, 60, 64 and 68. All other fields are set to known good values. + ** + ** Byte offset 105 should also contain the page-size as a 16-bit + ** integer. + */ + const int aPreserve[] = {32, 36, 52, 60, 64, 68}; + u8 aHdr[108] = { + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, + 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2e, 0x5b, 0x30, + + 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00 + }; + u8 *a = (u8*)aBuf; + + u32 pgsz = recoverGetU16(&a[16]); + u32 nReserve = a[20]; + u32 enc = recoverGetU32(&a[56]); + u32 dbsz = 0; + i64 dbFileSize = 0; + int ii; + sqlite3_recover *p = recover_g.p; + + if( pgsz==0x01 ) pgsz = 65536; + rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); + + if( rc==SQLITE_OK && p->detected_pgsz==0 ){ + rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize); + } + if( p->detected_pgsz ){ + pgsz = p->detected_pgsz; + nReserve = p->nReserve; + } + + if( pgsz ){ + dbsz = dbFileSize / pgsz; + } + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ + enc = SQLITE_UTF8; + } + + sqlite3_free(p->pPage1Cache); + p->pPage1Cache = 0; + p->pPage1Disk = 0; + + p->pgsz = nByte; + p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2); + if( p->pPage1Cache ){ + p->pPage1Disk = &p->pPage1Cache[nByte]; + memcpy(p->pPage1Disk, aBuf, nByte); + + recoverPutU32(&aHdr[28], dbsz); + recoverPutU32(&aHdr[56], enc); + recoverPutU16(&aHdr[105], pgsz-nReserve); + if( pgsz==65536 ) pgsz = 1; + recoverPutU16(&aHdr[16], pgsz); + aHdr[20] = nReserve; + for(ii=0; iipPage1Cache, aBuf, nByte); + }else{ + rc = p->errCode; + } + + } + pFd->pMethods = &recover_methods; + }else{ + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + } + return rc; +} + +/* +** Used to make sqlite3_io_methods wrapper methods less verbose. +*/ +#define RECOVER_VFS_WRAPPER(code) \ + int rc = SQLITE_OK; \ + if( pFd->pMethods==&recover_methods ){ \ + pFd->pMethods = recover_g.pMethods; \ + rc = code; \ + pFd->pMethods = &recover_methods; \ + }else{ \ + rc = code; \ + } \ + return rc; + +/* +** Methods of the wrapper VFS. All methods except for xRead() and xClose() +** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent +** method on the lower level VFS, then reinstall the wrapper before returning. +** Those that return an integer value use the RECOVER_VFS_WRAPPER macro. +*/ +static int recoverVfsWrite( + sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff) + ); +} +static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xTruncate(pFd, size) + ); +} +static int recoverVfsSync(sqlite3_file *pFd, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSync(pFd, flags) + ); +} +static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xFileSize(pFd, pSize) + ); +} +static int recoverVfsLock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xLock(pFd, eLock) + ); +} +static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xUnlock(pFd, eLock) + ); +} +static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xCheckReservedLock(pFd, pResOut) + ); +} +static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){ + RECOVER_VFS_WRAPPER ( + (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND) + ); +} +static int recoverVfsSectorSize(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSectorSize(pFd) + ); +} +static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xDeviceCharacteristics(pFd) + ); +} +static int recoverVfsShmMap( + sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp) + ); +} +static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmLock(pFd, offset, n, flags) + ); +} +static void recoverVfsShmBarrier(sqlite3_file *pFd){ + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + pFd->pMethods->xShmBarrier(pFd); + pFd->pMethods = &recover_methods; + }else{ + pFd->pMethods->xShmBarrier(pFd); + } +} +static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmUnmap(pFd, deleteFlag) + ); +} + +static int recoverVfsFetch( + sqlite3_file *pFd, + sqlite3_int64 iOff, + int iAmt, + void **pp +){ + *pp = 0; + return SQLITE_OK; +} +static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){ + return SQLITE_OK; +} + +/* +** Install the VFS wrapper around the file-descriptor open on the input +** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held +** when this function is called. +*/ +static void recoverInstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + assert( recover_g.pMethods==0 ); + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + assert( pFd==0 || pFd->pMethods!=&recover_methods ); + if( pFd && pFd->pMethods ){ + int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0); + recover_g.pMethods = pFd->pMethods; + recover_g.p = p; + recover_methods.iVersion = iVersion; + pFd->pMethods = &recover_methods; + } +} + +/* +** Uninstall the VFS wrapper that was installed around the file-descriptor open +** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be +** held when this function is called. +*/ +static void recoverUninstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); + if( pFd && pFd->pMethods ){ + pFd->pMethods = recover_g.pMethods; + recover_g.pMethods = 0; + recover_g.p = 0; + } +} + +/* +** This function does the work of a single sqlite3_recover_step() call. It +** is guaranteed that the handle is not in an error state when this +** function is called. +*/ +static void recoverStep(sqlite3_recover *p){ + assert( p && p->errCode==SQLITE_OK ); + switch( p->eState ){ + case RECOVER_STATE_INIT: + /* This is the very first call to sqlite3_recover_step() on this object. + */ + recoverSqlCallback(p, "BEGIN"); + recoverSqlCallback(p, "PRAGMA writable_schema = on"); + + recoverEnterMutex(); + recoverInstallWrapper(p); + + /* Open the output database. And register required virtual tables and + ** user functions with the new handle. */ + recoverOpenOutput(p); + + /* Open transactions on both the input and output databases. */ + recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); + recoverExec(p, p->dbIn, "BEGIN"); + if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; + recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema"); + recoverTransferSettings(p); + recoverOpenRecovery(p); + recoverCacheSchema(p); + + recoverUninstallWrapper(p); + recoverLeaveMutex(); + + recoverExec(p, p->dbOut, "BEGIN"); + + recoverWriteSchema1(p); + p->eState = RECOVER_STATE_WRITING; + break; + + case RECOVER_STATE_WRITING: { + if( p->w1.pTbls==0 ){ + recoverWriteDataInit(p); + } + if( SQLITE_DONE==recoverWriteDataStep(p) ){ + recoverWriteDataCleanup(p); + if( p->zLostAndFound ){ + p->eState = RECOVER_STATE_LOSTANDFOUND1; + }else{ + p->eState = RECOVER_STATE_SCHEMA2; + } + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND1: { + if( p->laf.pUsed==0 ){ + recoverLostAndFound1Init(p); + } + if( SQLITE_DONE==recoverLostAndFound1Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND2; + } + break; + } + case RECOVER_STATE_LOSTANDFOUND2: { + if( p->laf.pAllAndParent==0 ){ + recoverLostAndFound2Init(p); + } + if( SQLITE_DONE==recoverLostAndFound2Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND3; + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND3: { + if( p->laf.pInsert==0 ){ + recoverLostAndFound3Init(p); + } + if( SQLITE_DONE==recoverLostAndFound3Step(p) ){ + p->eState = RECOVER_STATE_SCHEMA2; + } + break; + } + + case RECOVER_STATE_SCHEMA2: { + int rc = SQLITE_OK; + + recoverWriteSchema2(p); + p->eState = RECOVER_STATE_DONE; + + /* If no error has occurred, commit the write transaction on the output + ** database. Regardless of whether or not an error has occurred, make + ** an attempt to end the read transaction on the input database. */ + recoverExec(p, p->dbOut, "COMMIT"); + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + + recoverSqlCallback(p, "PRAGMA writable_schema = off"); + recoverSqlCallback(p, "COMMIT"); + p->eState = RECOVER_STATE_DONE; + recoverFinalCleanup(p); + break; + }; + + case RECOVER_STATE_DONE: { + /* no-op */ + break; + }; + } +} + + +/* +** This is a worker function that does the heavy lifting for both init +** functions: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** All this function does is allocate space for the recover handle and +** take copies of the input parameters. All the real work is done within +** sqlite3_recover_run(). +*/ +sqlite3_recover *recoverInit( + sqlite3* db, + const char *zDb, + const char *zUri, /* Output URI for _recover_init() */ + int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */ + void *pSqlCtx /* Context arg for _recover_init_sql() */ +){ + sqlite3_recover *pRet = 0; + int nDb = 0; + int nUri = 0; + int nByte = 0; + + if( zDb==0 ){ zDb = "main"; } + + nDb = recoverStrlen(zDb); + nUri = recoverStrlen(zUri); + + nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; + pRet = (sqlite3_recover*)sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->dbIn = db; + pRet->zDb = (char*)&pRet[1]; + pRet->zUri = &pRet->zDb[nDb+1]; + memcpy(pRet->zDb, zDb, nDb); + if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri); + pRet->xSql = xSql; + pRet->pSqlCtx = pSqlCtx; + pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; + } + + return pRet; +} + +/* +** Initialize a recovery handle that creates a new database containing +** the recovered data. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +){ + return recoverInit(db, zDb, zUri, 0, 0); +} + +/* +** Initialize a recovery handle that returns recovered data in the +** form of SQL statements via a callback. +*/ +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pSqlCtx +){ + return recoverInit(db, zDb, 0, xSql, pSqlCtx); +} + +/* +** Return the handle error message, if any. +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover *p){ + return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory"; +} + +/* +** Return the handle error code. +*/ +int sqlite3_recover_errcode(sqlite3_recover *p){ + return p ? p->errCode : SQLITE_NOMEM; +} + +/* +** Configure the handle. +*/ +int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ + int rc = SQLITE_OK; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else if( p->eState!=RECOVER_STATE_INIT ){ + rc = SQLITE_MISUSE; + }else{ + switch( op ){ + case 789: + /* This undocumented magic configuration option is used to set the + ** name of the auxiliary database that is ATTACH-ed to the database + ** connection and used to hold state information during the + ** recovery process. This option is for debugging use only and + ** is subject to change or removal at any time. */ + sqlite3_free(p->zStateDb); + p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); + break; + + case SQLITE_RECOVER_LOST_AND_FOUND: { + const char *zArg = (const char*)pArg; + sqlite3_free(p->zLostAndFound); + if( zArg ){ + p->zLostAndFound = recoverMPrintf(p, "%s", zArg); + }else{ + p->zLostAndFound = 0; + } + break; + } + + case SQLITE_RECOVER_FREELIST_CORRUPT: + p->bFreelistCorrupt = *(int*)pArg; + break; + + case SQLITE_RECOVER_ROWIDS: + p->bRecoverRowid = *(int*)pArg; + break; + + case SQLITE_RECOVER_SLOWINDEXES: + p->bSlowIndexes = *(int*)pArg; + break; + + default: + rc = SQLITE_NOTFOUND; + break; + } + } + + return rc; +} + +/* +** Do a unit of work towards the recovery job. Return SQLITE_OK if +** no error has occurred but database recovery is not finished, SQLITE_DONE +** if database recovery has been successfully completed, or an SQLite +** error code if an error has occurred. +*/ +int sqlite3_recover_step(sqlite3_recover *p){ + if( p==0 ) return SQLITE_NOMEM; + if( p->errCode==SQLITE_OK ) recoverStep(p); + if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){ + return SQLITE_DONE; + } + return p->errCode; +} + +/* +** Do the configured recovery operation. Return SQLITE_OK if successful, or +** else an SQLite error code. +*/ +int sqlite3_recover_run(sqlite3_recover *p){ + while( SQLITE_OK==sqlite3_recover_step(p) ); + return sqlite3_recover_errcode(p); +} + + +/* +** Free all resources associated with the recover handle passed as the only +** argument. The results of using a handle with any sqlite3_recover_** +** API function after it has been passed to this function are undefined. +** +** A copy of the value returned by the first call made to sqlite3_recover_run() +** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has +** not been called on this handle. +*/ +int sqlite3_recover_finish(sqlite3_recover *p){ + int rc; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + recoverFinalCleanup(p); + if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + } + rc = p->errCode; + sqlite3_free(p->zErrMsg); + sqlite3_free(p->zStateDb); + sqlite3_free(p->zLostAndFound); + sqlite3_free(p->pPage1Cache); + sqlite3_free(p); + } + return rc; +} + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h new file mode 100644 index 0000000000..7a1cd1cd87 --- /dev/null +++ b/ext/recover/sqlite3recover.h @@ -0,0 +1,249 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the public interface to the "recover" extension - +** an SQLite extension designed to recover data from corrupted database +** files. +*/ + +/* +** OVERVIEW: +** +** To use the API to recover data from a corrupted database, an +** application: +** +** 1) Creates an sqlite3_recover handle by calling either +** sqlite3_recover_init() or sqlite3_recover_init_sql(). +** +** 2) Configures the new handle using one or more calls to +** sqlite3_recover_config(). +** +** 3) Executes the recovery by repeatedly calling sqlite3_recover_step() on +** the handle until it returns something other than SQLITE_OK. If it +** returns SQLITE_DONE, then the recovery operation completed without +** error. If it returns some other non-SQLITE_OK value, then an error +** has occurred. +** +** 4) Retrieves any error code and English language error message using the +** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs, +** respectively. +** +** 5) Destroys the sqlite3_recover handle and frees all resources +** using sqlite3_recover_finish(). +** +** The application may abandon the recovery operation at any point +** before it is finished by passing the sqlite3_recover handle to +** sqlite3_recover_finish(). This is not an error, but the final state +** of the output database, or the results of running the partial script +** delivered to the SQL callback, are undefined. +*/ + +#ifndef _SQLITE_RECOVER_H +#define _SQLITE_RECOVER_H + +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An instance of the sqlite3_recover object represents a recovery +** operation in progress. +** +** Constructors: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** Destructor: +** +** sqlite3_recover_finish() +** +** Methods: +** +** sqlite3_recover_config() +** sqlite3_recover_errcode() +** sqlite3_recover_errmsg() +** sqlite3_recover_run() +** sqlite3_recover_step() +*/ +typedef struct sqlite3_recover sqlite3_recover; + +/* +** These two APIs attempt to create and return a new sqlite3_recover object. +** In both cases the first two arguments identify the (possibly +** corrupt) database to recover data from. The first argument is an open +** database handle and the second the name of a database attached to that +** handle (i.e. "main", "temp" or the name of an attached database). +** +** If sqlite3_recover_init() is used to create the new sqlite3_recover +** handle, then data is recovered into a new database, identified by +** string parameter zUri. zUri may be an absolute or relative file path, +** or may be an SQLite URI. If the identified database file already exists, +** it is overwritten. +** +** If sqlite3_recover_init_sql() is invoked, then any recovered data will +** be returned to the user as a series of SQL statements. Executing these +** SQL statements results in the same database as would have been created +** had sqlite3_recover_init() been used. For each SQL statement in the +** output, the callback function passed as the third argument (xSql) is +** invoked once. The first parameter is a passed a copy of the fourth argument +** to this function (pCtx) as its first parameter, and a pointer to a +** nul-terminated buffer containing the SQL statement formated as UTF-8 as +** the second. If the xSql callback returns any value other than SQLITE_OK, +** then processing is immediately abandoned and the value returned used as +** the recover handle error code (see below). +** +** If an out-of-memory error occurs, NULL may be returned instead of +** a valid handle. In all other cases, it is the responsibility of the +** application to avoid resource leaks by ensuring that +** sqlite3_recover_finish() is called on all allocated handles. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +); +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pCtx +); + +/* +** Configure an sqlite3_recover object that has just been created using +** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function +** may only be called before the first call to sqlite3_recover_step() +** or sqlite3_recover_run() on the object. +** +** The second argument passed to this function must be one of the +** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument +** depend on the specific SQLITE_RECOVER_* symbol in use. +** +** SQLITE_OK is returned if the configuration operation was successful, +** or an SQLite error code otherwise. +*/ +int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); + +/* +** SQLITE_RECOVER_LOST_AND_FOUND: +** The pArg argument points to a string buffer containing the name +** of a "lost-and-found" table in the output database, or NULL. If +** the argument is non-NULL and the database contains seemingly +** valid pages that cannot be associated with any table in the +** recovered part of the schema, data is extracted from these +** pages to add to the lost-and-found table. +** +** SQLITE_RECOVER_FREELIST_CORRUPT: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1) and a lost-and-found table has been configured using +** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is +** corrupt and an attempt is made to recover records from pages that +** appear to be linked into the freelist. Otherwise, pages on the freelist +** are ignored. Setting this option can recover more data from the +** database, but often ends up "recovering" deleted records. The default +** value is 0 (clear). +** +** SQLITE_RECOVER_ROWIDS: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1), then an attempt is made to recover rowid values +** that are not also INTEGER PRIMARY KEY values. If this option is +** clear, then new rowids are assigned to all recovered rows. The +** default value is 1 (set). +** +** SQLITE_RECOVER_SLOWINDEXES: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is clear +** (argument is 0), then when creating an output database, the recover +** module creates and populates non-UNIQUE indexes right at the end of the +** recovery operation - after all recoverable data has been inserted +** into the new database. This is faster overall, but means that the +** final call to sqlite3_recover_step() for a recovery operation may +** be need to create a large number of indexes, which may be very slow. +** +** Or, if this option is set (argument is 1), then non-UNIQUE indexes +** are created in the output database before it is populated with +** recovered data. This is slower overall, but avoids the slow call +** to sqlite3_recover_step() at the end of the recovery operation. +** +** The default option value is 0. +*/ +#define SQLITE_RECOVER_LOST_AND_FOUND 1 +#define SQLITE_RECOVER_FREELIST_CORRUPT 2 +#define SQLITE_RECOVER_ROWIDS 3 +#define SQLITE_RECOVER_SLOWINDEXES 4 + +/* +** Perform a unit of work towards the recovery operation. This function +** must normally be called multiple times to complete database recovery. +** +** If no error occurs but the recovery operation is not completed, this +** function returns SQLITE_OK. If recovery has been completed successfully +** then SQLITE_DONE is returned. If an error has occurred, then an SQLite +** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not +** considered an error if some or all of the data cannot be recovered +** due to database corruption. +** +** Once sqlite3_recover_step() has returned a value other than SQLITE_OK, +** all further such calls on the same recover handle are no-ops that return +** the same non-SQLITE_OK value. +*/ +int sqlite3_recover_step(sqlite3_recover*); + +/* +** Run the recovery operation to completion. Return SQLITE_OK if successful, +** or an SQLite error code otherwise. Calling this function is the same +** as executing: +** +** while( SQLITE_OK==sqlite3_recover_step(p) ); +** return sqlite3_recover_errcode(p); +*/ +int sqlite3_recover_run(sqlite3_recover*); + +/* +** If an error has been encountered during a prior call to +** sqlite3_recover_step(), then this function attempts to return a +** pointer to a buffer containing an English language explanation of +** the error. If no error message is available, or if an out-of memory +** error occurs while attempting to allocate a buffer in which to format +** the error message, NULL is returned. +** +** The returned buffer remains valid until the sqlite3_recover handle is +** destroyed using sqlite3_recover_finish(). +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover*); + +/* +** If this function is called on an sqlite3_recover handle after +** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK. +*/ +int sqlite3_recover_errcode(sqlite3_recover*); + +/* +** Clean up a recovery object created by a call to sqlite3_recover_init(). +** The results of using a recovery object with any API after it has been +** passed to this function are undefined. +** +** This function returns the same value as sqlite3_recover_errcode(). +*/ +int sqlite3_recover_finish(sqlite3_recover*); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_RECOVER_H */ diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c new file mode 100644 index 0000000000..1c333df8e0 --- /dev/null +++ b/ext/recover/test_recover.c @@ -0,0 +1,311 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + +#include "sqlite3recover.h" +#include "sqliteInt.h" + +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +typedef struct TestRecover TestRecover; +struct TestRecover { + sqlite3_recover *p; + Tcl_Interp *interp; + Tcl_Obj *pScript; +}; + +static int xSqlCallback(void *pSqlArg, const char *zSql){ + TestRecover *p = (TestRecover*)pSqlArg; + Tcl_Obj *pEval = 0; + int res = 0; + + pEval = Tcl_DuplicateObj(p->pScript); + Tcl_IncrRefCount(pEval); + + res = Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zSql, -1)); + if( res==TCL_OK ){ + res = Tcl_EvalObjEx(p->interp, pEval, 0); + } + + Tcl_DecrRefCount(pEval); + if( res ){ + Tcl_BackgroundError(p->interp); + return TCL_ERROR; + }else{ + Tcl_Obj *pObj = Tcl_GetObjResult(p->interp); + if( Tcl_GetCharLength(pObj)==0 ){ + res = 0; + }else if( Tcl_GetIntFromObj(p->interp, pObj, &res) ){ + Tcl_BackgroundError(p->interp); + return TCL_ERROR; + } + } + return res; +} + +static int getDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ + Tcl_CmdInfo info; + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + return TCL_ERROR; + } + *pDb = *(sqlite3 **)info.objClientData; + return TCL_OK; +} + +/* +** Implementation of the command created by [sqlite3_recover_init]: +** +** $cmd config OP ARG +** $cmd run +** $cmd errmsg +** $cmd errcode +** $cmd finalize +*/ +static int testRecoverCmd( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static struct RecoverSub { + const char *zSub; + int nArg; + const char *zMsg; + } aSub[] = { + { "config", 2, "ARG" }, /* 0 */ + { "run", 0, "" }, /* 1 */ + { "errmsg", 0, "" }, /* 2 */ + { "errcode", 0, "" }, /* 3 */ + { "finish", 0, "" }, /* 4 */ + { "step", 0, "" }, /* 5 */ + { 0 } + }; + int rc = TCL_OK; + int iSub = 0; + TestRecover *pTest = (TestRecover*)clientData; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ..."); + return TCL_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( (objc-2)!=aSub[iSub].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + + switch( iSub ){ + case 0: assert( sqlite3_stricmp("config", aSub[iSub].zSub)==0 ); { + const char *aOp[] = { + "testdb", /* 0 */ + "lostandfound", /* 1 */ + "freelistcorrupt", /* 2 */ + "rowids", /* 3 */ + "slowindexes", /* 4 */ + "invalid", /* 5 */ + 0 + }; + int iOp = 0; + int res = 0; + if( Tcl_GetIndexFromObj(interp, objv[2], aOp, "option", 0, &iOp) ){ + return TCL_ERROR; + } + switch( iOp ){ + case 0: + res = sqlite3_recover_config(pTest->p, + 789, (void*)Tcl_GetString(objv[3]) /* MAGIC NUMBER! */ + ); + break; + case 1: { + const char *zStr = Tcl_GetString(objv[3]); + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_LOST_AND_FOUND, (void*)(zStr[0] ? zStr : 0) + ); + break; + } + case 2: { + int iVal = 0; + if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_FREELIST_CORRUPT, (void*)&iVal + ); + break; + } + case 3: { + int iVal = 0; + if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_ROWIDS, (void*)&iVal + ); + break; + } + case 4: { + int iVal = 0; + if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_SLOWINDEXES, (void*)&iVal + ); + break; + } + case 5: { + res = sqlite3_recover_config(pTest->p, 12345, 0); + break; + } + } + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } + case 1: assert( sqlite3_stricmp("run", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_run(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } + case 2: assert( sqlite3_stricmp("errmsg", aSub[iSub].zSub)==0 ); { + const char *zErr = sqlite3_recover_errmsg(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + break; + } + case 3: assert( sqlite3_stricmp("errcode", aSub[iSub].zSub)==0 ); { + int errCode = sqlite3_recover_errcode(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(errCode)); + break; + } + case 4: assert( sqlite3_stricmp("finish", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_errcode(pTest->p); + int res2; + if( res!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + } + res2 = sqlite3_recover_finish(pTest->p); + assert( res2==res ); + if( res ) return TCL_ERROR; + break; + } + case 5: assert( sqlite3_stricmp("step", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_step(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } + } + + return TCL_OK; +} + +/* +** sqlite3_recover_init DB DBNAME URI +*/ +static int test_sqlite3_recover_init( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static int iTestRecoverCmd = 1; + + TestRecover *pNew = 0; + sqlite3 *db = 0; + const char *zDb = 0; + const char *zUri = 0; + char zCmd[128]; + int bSql = clientData ? 1 : 0; + + if( objc!=4 ){ + const char *zErr = (bSql ? "DB DBNAME SCRIPT" : "DB DBNAME URI"); + Tcl_WrongNumArgs(interp, 1, objv, zErr); + return TCL_ERROR; + } + if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; + zDb = Tcl_GetString(objv[2]); + if( zDb[0]=='\0' ) zDb = 0; + + pNew = ckalloc(sizeof(TestRecover)); + if( bSql==0 ){ + zUri = Tcl_GetString(objv[3]); + pNew->p = sqlite3_recover_init(db, zDb, zUri); + }else{ + pNew->interp = interp; + pNew->pScript = objv[3]; + Tcl_IncrRefCount(pNew->pScript); + pNew->p = sqlite3_recover_init_sql(db, zDb, xSqlCallback, (void*)pNew); + } + + sprintf(zCmd, "sqlite_recover%d", iTestRecoverCmd++); + Tcl_CreateObjCommand(interp, zCmd, testRecoverCmd, (void*)pNew, 0); + + Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1)); + return TCL_OK; +} + +/* +** Declaration for public API function in file dbdata.c. This may be called +** with NULL as the final two arguments to register the sqlite_dbptr and +** sqlite_dbdata virtual tables with a database handle. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +/* +** sqlite3_recover_init DB DBNAME URI +*/ +static int test_sqlite3_dbdata_init( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; + sqlite3_dbdata_init(db, 0, 0); + + Tcl_ResetResult(interp); + return TCL_OK; +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int TestRecover_Init(Tcl_Interp *interp){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + struct Cmd { + const char *zCmd; + Tcl_ObjCmdProc *xProc; + void *pArg; + } aCmd[] = { + { "sqlite3_recover_init", test_sqlite3_recover_init, 0 }, + { "sqlite3_recover_init_sql", test_sqlite3_recover_init, (void*)1 }, + { "sqlite3_dbdata_init", test_sqlite3_dbdata_init, (void*)1 }, + }; + int i; + + for(i=0; izCmd, p->xProc, p->pArg, 0); + } +#endif + return TCL_OK; +} + diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 49053a2bcc..7daae9b1cb 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -3235,7 +3235,7 @@ static int rtreeUpdate( rtreeReference(pRtree); assert(nData>=1); - cell.iRowid = 0; /* Used only to suppress a compiler warning */ + memset(&cell, 0, sizeof(cell)); /* Constraint handling. A write operation on an r-tree table may return ** SQLITE_CONSTRAINT for two reasons: diff --git a/ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle b/ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle deleted file mode 100644 index af2c48a3f4..0000000000 --- a/ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle +++ /dev/null @@ -1,14 +0,0 @@ -FS -addFunction -allocateUTF8OnStack -ccall -cwrap -getValue -intArrayFromString -lengthBytesUTF8 -removeFunction -setValue -stackAlloc -stackRestore -stackSave -stringToUTF8Array diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index ee8ade74a3..039dff410f 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -1,37 +1,114 @@ -# This GNU makefile exists primarily to simplify/speed up development -# of the sqlite3 WASM components. It is not part of the canonical -# build process. +####################################################################### +# This GNU makefile drives the build of the sqlite3 WASM +# components. It is not part of the canonical build process. # -# Maintenance notes: the fiddle build is currently performed in the -# top-level ../../Makefile.in. It may be moved into this file at some -# point, as GNU Make has been deemed acceptable for the WASM-related -# components (whereas POSIX Make is required for the more conventional -# components). +# This build assumes a Linux platform and is not intended for +# general-purpose client-level use, except for creating builds with +# custom configurations. It is primarily intended for the sqlite +# project's own development of the JS/WASM components. +# +# Primary targets: +# +# default, all = build in dev mode +# +# o0, o1, o2, o3, os, oz = full clean/rebuild with the -Ox level indicated +# by the target name. Rebuild is necessary for all components to get +# the desired optimization level. +# +# dist = create end user deliverables. Add dist.build=oX to build +# with a specific optimization level, where oX is one of the +# above-listed o? target names. +# +# clean = clean up +######################################################################## SHELL := $(shell which bash 2>/dev/null) -all: - -.PHONY: fiddle -ifneq (,$(wildcard /home/stephan)) - fiddle_opt ?= -O0 -else - fiddle_opt = -Os -endif -fiddle: - $(MAKE) -C ../.. fiddle -e emcc_opt=$(fiddle_opt) - -clean: - $(MAKE) -C ../../ clean-fiddle - -rm -f $(CLEAN_FILES) - MAKEFILE := $(lastword $(MAKEFILE_LIST)) +CLEAN_FILES := +DISTCLEAN_FILES := ./--dummy-- +default: all +release: oz + +# Emscripten SDK home dir and related binaries... +EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/emsdk $(HOME)/src/emsdk)) +emcc.bin ?= $(word 1,$(wildcard $(EMSDK_HOME)/upstream/emscripten/emcc) $(shell which emcc)) +ifeq (,$(emcc.bin)) + $(error Cannot find emcc.) +endif + +wasm-strip ?= $(shell which wasm-strip 2>/dev/null) +ifeq (,$(filter clean,$(MAKECMDGOALS))) +ifeq (,$(wasm-strip)) + $(info WARNING: *******************************************************************) + $(info WARNING: builds using -O2/-O3/-Os/-Oz will minify WASM-exported names,) + $(info WARNING: breaking _All The Things_. The workaround for that is to build) + $(info WARNING: with -g3 (which explodes the file size) and then strip the debug) + $(info WARNING: info after compilation, using wasm-strip, to shrink the wasm file.) + $(info WARNING: wasm-strip was not found in the PATH so we cannot strip those.) + $(info WARNING: If this build uses any optimization level higher than -O1 then) + $(info WARNING: the ***resulting JS code WILL NOT BE USABLE***.) + $(info WARNING: wasm-strip is part of the wabt package:) + $(info WARNING: https://github.com/WebAssembly/wabt) + $(info WARNING: on Ubuntu-like systems it can be installed with:) + $(info WARNING: sudo apt install wabt) + $(info WARNING: *******************************************************************) +endif +endif # 'make clean' check + +ifeq (,$(wasm-strip)) + maybe-wasm-strip = echo "not wasm-stripping" +else + maybe-wasm-strip = $(wasm-strip) +endif + dir.top := ../.. -# Reminder: some Emscripten flags require absolute paths -dir.wasm := $(patsubst %/,%,$(dir $(abspath $(MAKEFILE)))) +# Reminder: some Emscripten flags require absolute paths but we want +# relative paths for most stuff simply to reduce noise. The +# $(abspath...) GNU make function can transform relative paths to +# absolute. +dir.wasm := $(patsubst %/,%,$(dir $(MAKEFILE))) dir.api := api dir.jacc := jaccwabyt dir.common := common -CLEAN_FILES := *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ +dir.fiddle := fiddle +dir.tool := $(dir.top)/tool +######################################################################## +# dir.dout = output dir for deliverables. +# +# MAINTENANCE REMINDER: the output .js and .wasm files of emcc must be +# in _this_ dir, rather than a subdir, or else parts of the generated +# code get confused and cannot load property. Specifically, when X.js +# loads X.wasm, whether or not X.js uses the correct path for X.wasm +# depends on how it's loaded: an HTML script tag will resolve it +# intuitively, whereas a Worker's call to importScripts() will not. +# That's a fundamental incompatibility with how URL resolution in +# JS happens between those two contexts. See: +# +# https://zzz.buzz/2017/03/14/relative-uris-in-web-development/ +# +# We unfortunately have no way, from Worker-initiated code, to +# automatically resolve the path from X.js to X.wasm. +# +# We have an "only slightly unsightly" solution for our main builds +# but it does not work for the WASMFS builds, so those builds have to +# be built to _this_ directory and can only run when the client app is +# loaded from the same directory. +dir.dout := $(dir.wasm)/jswasm +# dir.tmp = output dir for intermediary build files, as opposed to +# end-user deliverables. +dir.tmp := $(dir.wasm)/bld +CLEAN_FILES += $(dir.tmp)/* $(dir.dout)/* +ifeq (,$(wildcard $(dir.dout))) + dir._tmp := $(shell mkdir -p $(dir.dout)) +endif +ifeq (,$(wildcard $(dir.tmp))) + dir._tmp := $(shell mkdir -p $(dir.tmp)) +endif +cflags.common := -I. -I.. -I$(dir.top) +CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ +emcc.WASM_BIGINT ?= 1 +sqlite3.c := $(dir.top)/sqlite3.c +sqlite3.h := $(dir.top)/sqlite3.h SQLITE_OPT = \ -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_RTREE \ @@ -45,21 +122,31 @@ SQLITE_OPT = \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED \ -DSQLITE_OMIT_UTF16 \ - -DSQLITE_THREADSAFE=0 -#SQLITE_OPT += -DSQLITE_ENABLE_MEMSYS5 -$(dir.top)/sqlite3.c: - $(MAKE) -C $(dir.top) sqlite3.c - -# SQLITE_OMIT_LOAD_EXTENSION: if this is true, sqlite3_vfs::xDlOpen -# and friends may be NULL. + -DSQLITE_OMIT_SHARED_CACHE \ + -DSQLITE_OMIT_WAL \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_TEMP_STORE=3 \ + -DSQLITE_OS_KV_OPTIONAL=1 \ + '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ + -DSQLITE_USE_URI=1 \ + -DSQLITE_WASM_ENABLE_C_TESTS +# ^^^ most flags are set in sqlite3-wasm.c but we need them +# made explicit here for building speedtest1.c. +ifneq (,$(filter release,$(MAKECMDGOALS))) +emcc_opt ?= -Oz -flto +else emcc_opt ?= -O0 -.PHONY: release -release: - $(MAKE) 'emcc_opt=-Os -g3' -# ^^^^^ target-specific vars, e.g.: -# release: emcc_opt=... -# apparently only work for file targets, not PHONY targets? +# ^^^^ build times for -O levels higher than 0 are painful at +# dev-time. +endif +# When passing emcc_opt from the CLI, += and re-assignment have no +# effect, so emcc_opt+=-g3 doesn't work. So... +emcc_opt_full := $(emcc_opt) -g3 +# ^^^ ALWAYS use -g3. See below for why. +# +# ^^^ -flto improves runtime speed at -O0 considerably but doubles +# build time. # # ^^^^ -O3, -Oz, -Os minify symbol names and there appears to be no # way around that except to use -g3, but -g3 causes the binary file @@ -77,33 +164,19 @@ release: # -Os: not quite 1% in some completely unscientific tests. Runtime # speed for the unit tests is all over the place either way so it's # difficult to say whether -Os gives any speed benefit over -Oz. +# +# (Much later: -O2 consistently gives the best speeds.) ######################################################################## -# Emscripten SDK home dir and related binaries... -EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/src/emsdk $(HOME)/emsdk)) -emcc.bin ?= $(word 1,$(wildcard $(shell which emcc) $(EMSDK_HOME)/upstream/emscripten/emcc)) -ifeq (,$(emcc.bin)) - $(error Cannot find emcc.) -endif -wasm-strip ?= $(shell which wasm-strip 2>/dev/null) -ifeq (,$(filter clean,$(MAKECMDGOALS))) -ifeq (,$(wasm-strip)) - $(info WARNING: *******************************************************************) - $(info WARNING: builds using -O3/-Os/-Oz will minify WASM-exported names,) - $(info WARNING: breaking _All The Things_. The workaround for that is to build) - $(info WARNING: with -g3 (which explodes the file size) and then strip the debug) - $(info WARNING: info after compilation, using wasm-strip, to shrink the wasm file.) - $(info WARNING: wasm-strip was not found in the PATH so we cannot strip those.) - $(info WARNING: If this build uses any optimization level higher than -O2 then) - $(info WARNING: the ***resulting WASM binary WILL NOT BE USABLE***.) - $(info WARNING: wasm-strip is part of the wabt package:) - $(info WARNING: https://github.com/WebAssembly/wabt) - $(info WARNING: on Ubuntu-like systems it can be installed with:) - $(info WARNING: sudo apt install wabt) - $(info WARNING: *******************************************************************) -endif -endif # 'make clean' check +$(sqlite3.c) $(sqlite3.h): + $(MAKE) -C $(dir.top) sqlite3.c + +.PHONY: clean distclean +clean: + -rm -f $(CLEAN_FILES) +distclean: clean + -rm -f $(DISTCLEAN_FILES) ifeq (release,$(filter release,$(MAKECMDGOALS))) ifeq (,$(wasm-strip)) @@ -114,25 +187,54 @@ else $(info Development build. Use '$(MAKE) release' for a smaller release build.) endif -EXPORTED_FUNCTIONS.api.in := $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api \ - $(dir.jacc)/jaccwabyt_test.exports +bin.version-info := $(dir.wasm)/version-info +# ^^^^ NOT in $(dir.tmp) because we need it to survive the cleanup +# process for the dist build to work properly. +$(bin.version-info): $(dir.wasm)/version-info.c $(sqlite3.h) $(MAKEFILE) + $(CC) -O0 -I$(dir.top) -o $@ $< +DISTCLEAN_FILES += $(bin.version-info) -EXPORTED_FUNCTIONS.api: $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) +bin.stripccomments := $(dir.tool)/stripccomments +$(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) + $(CC) -o $@ $< +DISTCLEAN_FILES += $(bin.stripccomments) + +EXPORTED_FUNCTIONS.api.in := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api +$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) cat $(EXPORTED_FUNCTIONS.api.in) > $@ -CLEAN_FILES += EXPORTED_FUNCTIONS.api -sqlite3-api.jses := \ - $(dir.api)/sqlite3-api-prologue.js \ - $(dir.common)/whwasmutil.js \ - $(dir.jacc)/jaccwabyt.js \ - $(dir.api)/sqlite3-api-glue.js \ - $(dir.api)/sqlite3-api-oo1.js \ - $(dir.api)/sqlite3-api-worker.js \ - $(dir.api)/sqlite3-api-opfs.js \ - $(dir.api)/sqlite3-api-cleanup.js +sqlite3-license-version.js := $(dir.tmp)/sqlite3-license-version.js +sqlite3-license-version-header.js := $(dir.api)/sqlite3-license-version-header.js +sqlite3-api-build-version.js := $(dir.tmp)/sqlite3-api-build-version.js +# sqlite3-api.jses = the list of JS files which make up $(sqlite3-api.js), in +# the order they need to be assembled. +sqlite3-api.jses := $(sqlite3-license-version.js) +sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js +sqlite3-api.jses += $(dir.common)/whwasmutil.js +sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js +sqlite3-api.jses += $(sqlite3-api-build-version.js) +sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js -sqlite3-api.js := $(dir.api)/sqlite3-api.js -CLEAN_FILES += $(sqlite3-api.js) +# "External" API files which are part of our distribution +# but not part of the sqlite3-api.js amalgamation. +SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js +sqlite3-worker1.js := $(dir.api)/sqlite3-worker1.js +sqlite3-worker1-promiser.js := $(dir.api)/sqlite3-worker1-promiser.js +define CP_XAPI +sqlite3-api.ext.jses += $$(dir.dout)/$$(notdir $(1)) +$$(dir.dout)/$$(notdir $(1)): $(1) $$(MAKEFILE) + cp $$< $$@ +endef +$(foreach X,$(SOAP.js) $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js),\ + $(eval $(call CP_XAPI,$(X)))) +all: $(sqlite3-api.ext.jses) + +sqlite3-api.js := $(dir.tmp)/sqlite3-api.js $(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) @echo "Making $@..." @for i in $(sqlite3-api.jses); do \ @@ -141,13 +243,25 @@ $(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) echo "/* END FILE: $$i */"; \ done > $@ -post-js.js := $(dir.api)/post-js.js -CLEAN_FILES += $(post-js.js) +$(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) + @echo "Making $@..." + @{ \ + echo 'self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){'; \ + echo -n ' sqlite3.version = '; \ + $(bin.version-info) --json; \ + echo ';'; \ + echo '});'; \ + } > $@ + +######################################################################## +# --post-js and --pre-js are emcc flags we use to append/prepend JS to +# the generated emscripten module file. +pre-js.js := $(dir.api)/pre-js.js +post-js.js := $(dir.tmp)/post-js.js post-jses := \ $(dir.api)/post-js-header.js \ $(sqlite3-api.js) \ $(dir.api)/post-js-footer.js - $(post-js.js): $(post-jses) $(MAKEFILE) @echo "Making $@..." @for i in $(post-jses); do \ @@ -155,133 +269,387 @@ $(post-js.js): $(post-jses) $(MAKEFILE) cat $$i; \ echo "/* END FILE: $$i */"; \ done > $@ - +extern-post-js.js := $(dir.api)/extern-post-js.js +extern-pre-js.js := $(dir.api)/extern-pre-js.js +pre-post-common.flags := \ + --post-js=$(post-js.js) \ + --extern-post-js=$(extern-post-js.js) \ + --extern-pre-js=$(sqlite3-license-version.js) +pre-post-jses.deps := $(post-js.js) \ + $(extern-post-js.js) $(extern-pre-js.js) $(sqlite3-license-version.js) +$(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) $(MAKEFILE) + @echo "Making $@..."; { \ + cat $(sqlite3-license-version-header.js); \ + echo '/*'; \ + echo '** This code was built from sqlite3 version...'; \ + echo "** "; \ + awk -e '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' \ + -e '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ + echo '*/'; \ + } > $@ ######################################################################## -# emcc flags for .c/.o/.wasm. -emcc.flags = +# call-make-pre-js creates rules for pre-js-$(1).js. $1 = the base +# name of the JS file on whose behalf this pre-js is for. +define call-make-pre-js +pre-post-$(1).flags ?= +$$(dir.tmp)/pre-js-$(1).js: $$(pre-js.js) $$(MAKEFILE) + cp $$(pre-js.js) $$@ + @if [ sqlite3-wasmfs = $(1) ]; then \ + echo "delete Module[xNameOfInstantiateWasm] /*for WASMFS build*/;"; \ + elif [ sqlite3 != $(1) ]; then \ + echo "Module[xNameOfInstantiateWasm].uri = '$(1).wasm';"; \ + fi >> $$@ +pre-post-$(1).deps := $$(pre-post-jses.deps) $$(dir.tmp)/pre-js-$(1).js +pre-post-$(1).flags += --pre-js=$$(dir.tmp)/pre-js-$(1).js +endef +#$(error $(call call-make-pre-js,sqlite3-wasmfs)) +# /post-js and pre-js +######################################################################## + +######################################################################## +# emcc flags for .c/.o/.wasm/.js. +emcc.flags := #emcc.flags += -v # _very_ loud but also informative about what it's doing +# -g3 is needed to keep -O2 and higher from creating broken JS via +# minification. ######################################################################## # emcc flags for .c/.o. emcc.cflags := emcc.cflags += -std=c99 -fPIC # -------------^^^^^^^^ we currently need c99 for WASM-specific sqlite3 APIs. -emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT) +emcc.cflags += -I. -I$(dir.top) ######################################################################## # emcc flags specific to building the final .js/.wasm file... emcc.jsflags := -fPIC +emcc.jsflags += --minify 0 emcc.jsflags += --no-entry -emcc.jsflags += -sENVIRONMENT=web emcc.jsflags += -sMODULARIZE emcc.jsflags += -sSTRICT_JS emcc.jsflags += -sDYNAMIC_EXECUTION=0 emcc.jsflags += -sNO_POLYFILL -emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api -emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory # wasmMemory==>for -sIMPORTED_MEMORY +emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.api) +emcc.exportedRuntimeMethods := \ + -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory + # FS ==> stdio/POSIX I/O proxies + # wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY +emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY -#emcc.jsflags += -sINITIAL_MEMORY=13107200 +emcc.environment := -sENVIRONMENT=web,worker +######################################################################## +# -sINITIAL_MEMORY: How much memory we need to start with is governed +# at least in part by whether -sALLOW_MEMORY_GROWTH is enabled. If so, +# we can start with less. If not, we need as much as we'll ever +# possibly use (which, of course, we can't know for sure). Note, +# however, that speedtest1 shows that performance for even moderate +# workloads MAY suffer considerably if we start small and have to grow +# at runtime. e.g. OPFS-backed (speedtest1 --size 75) take MAY take X +# time with 16mb+ memory and 3X time when starting with 8MB. However, +# such test results are inconsistent due to browser internals which +# are opaque to us. +emcc.jsflags += -sALLOW_MEMORY_GROWTH +emcc.INITIAL_MEMORY.128 := 13107200 +emcc.INITIAL_MEMORY.96 := 100663296 +emcc.INITIAL_MEMORY.64 := 64225280 +emcc.INITIAL_MEMORY.32 := 33554432 +emcc.INITIAL_MEMORY.16 := 16777216 +emcc.INITIAL_MEMORY.8 := 8388608 +emcc.INITIAL_MEMORY ?= 16 +ifeq (,$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY))) +$(error emcc.INITIAL_MEMORY must be one of: 8, 16, 32, 64, 96, 128 (megabytes)) +endif +emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) +# /INITIAL_MEMORY +######################################################################## + +emcc.jsflags += $(emcc.environment) #emcc.jsflags += -sTOTAL_STACK=4194304 -emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule + +sqlite3.js.init-func := sqlite3InitModule +# ^^^^ $(sqlite3.js.init-func) symbol name is hard-coded in +# $(extern-post-js.js) as well as in numerous docs. If changed, it +# needs to be globally modified in *.js and all related documentation. + +emcc.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. -emcc.jsflags +=--post-js=$(post-js.js) #emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() #emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS #emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API #emcc.jsflags += -sABORTING_MALLOC -emcc.jsflags += -sALLOW_MEMORY_GROWTH emcc.jsflags += -sALLOW_TABLE_GROWTH +# -sALLOW_TABLE_GROWTH is required for installing new SQL UDFs emcc.jsflags += -Wno-limited-postlink-optimizations # ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. -#emcc.jsflags += -sMALLOC=emmalloc -#emcc.jsflags += -sMALLOC=dlmalloc # a good 8k larger than emmalloc #emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why -#emcc.jsflags += --import=foo_bar -#emcc.jsflags += --no-gc-sections # https://lld.llvm.org/WebAssembly.html emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 emcc.jsflags += -sLLD_REPORT_UNDEFINED #emcc.jsflags += --allow-undefined -emcc.jsflags += --import-undefined +#emcc.jsflags += --import-undefined #emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic -#emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined +#emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined #emcc.jsflags += --unresolved-symbols=ignore-all -enable_bigint ?= 1 -ifneq (0,$(enable_bigint)) -emcc.jsflags += -sWASM_BIGINT -endif -emcc.jsflags += -sMEMORY64=0 -# ^^^^ MEMORY64=1 fails to load, erroring with: +emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) + +######################################################################## +# -sMEMORY64=1 fails to load, erroring with: # invalid memory limits flags 0x5 # (enable via --experimental-wasm-memory64) # # ^^^^ MEMORY64=2 builds and loads but dies when we do things like: # -# new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) +# new Uint8Array(wasm.heap8u().buffer, ptr, n) # # because ptr is now a BigInt, so is invalid for passing to arguments -# which have strict must-be-a-number requirements. +# which have strict must-be-a-Number requirements. ######################################################################## -sqlite3.js := $(dir.api)/sqlite3.js -sqlite3.wasm := $(dir.api)/sqlite3.wasm -$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT) -$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c -$(dir.api)/wasm_util.o: emcc.cflags += $(SQLITE_OPT) -sqlite3.wasm.c := $(dir.api)/sqlite3-wasm.c \ - $(dir.jacc)/jaccwabyt_test.c -# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test -# apps. However, we want to test the release builds with those apps, -# so we cannot simply elide that file in release builds. That -# component is critical to the VFS bindings so needs to be tested -# along with the core APIs. -define WASM_C_COMPILE -$(1).o := $$(subst .c,.o,$(1)) -sqlite3.wasm.obj += $$($(1).o) -$$($(1).o): $$(MAKEFILE) $(1) - $$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@ -CLEAN_FILES += $$($(1).o) -endef -$(foreach c,$(sqlite3.wasm.c),$(eval $(call WASM_C_COMPILE,$(c)))) -$(sqlite3.js): -$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \ - EXPORTED_FUNCTIONS.api \ - $(post-js.js) - $(emcc.bin) -o $@ $(emcc_opt) $(emcc.flags) $(emcc.jsflags) $(sqlite3.wasm.obj) - chmod -x $(sqlite3.wasm) -ifneq (,$(wasm-strip)) - $(wasm-strip) $(sqlite3.wasm) -endif - @ls -la $@ $(sqlite3.wasm) +######################################################################## +# -sSINGLE_FILE: +# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js#L1704 +# -sSINGLE_FILE=1 would be really nice but we have to build with -g3 +# for -O2 and higher to work (else minification breaks the code) and +# cannot wasm-strip the binary before it gets encoded into the JS +# file. The result is that the generated JS file is, because of the -g3 +# debugging info, _huge_. +######################################################################## +######################################################################## +# AN EXPERIMENT: undocumented Emscripten feature: if the target file +# extension is "mjs", it defaults to ES6 module builds: +# https://github.com/emscripten-core/emscripten/issues/14383 +ifeq (,$(filter esm,$(MAKECMDGOALS))) +sqlite3.js.ext := js +else +esm.deps := $(filter-out esm,$(MAKECMDGOALS)) +esm: $(if $(esm.deps),$(esm.deps),all) +sqlite3.js.ext := mjs +endif +# /esm +######################################################################## +sqlite3.js := $(dir.dout)/sqlite3.$(sqlite3.js.ext) +sqlite3.wasm := $(dir.dout)/sqlite3.wasm +sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c +# sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter +# (predictably) results in a slightly faster binary, but we're close +# enough to the target speed requirements that the 500ms makes a +# difference. Thus we build all binaries against sqlite3-wasm.c +# instead of building a shared copy of sqlite3-wasm.o. +$(eval $(call call-make-pre-js,sqlite3)) +$(sqlite3.js): +$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \ + $(EXPORTED_FUNCTIONS.api) \ + $(pre-post-sqlite3.deps) + @echo "Building $@ ..." + $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ + $(emcc.jsflags) $(pre-post-common.flags) $(pre-post-sqlite3.flags) \ + $(cflags.common) $(SQLITE_OPT) $(sqlite3-wasm.c) + chmod -x $(sqlite3.wasm) + $(maybe-wasm-strip) $(sqlite3.wasm) + @ls -la $@ $(sqlite3.wasm) +$(sqlite3.wasm): $(sqlite3.js) CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm) all: $(sqlite3.js) +wasm: $(sqlite3.js) # End main Emscripten-based module build ######################################################################## +######################################################################## +# batch-runner.js... +dir.sql := sql +speedtest1 := ../../speedtest1 +speedtest1.c := ../../test/speedtest1.c +speedtest1.sql := $(dir.sql)/speedtest1.sql +speedtest1.cliflags := --size 25 --big-transactions +$(speedtest1): + $(MAKE) -C ../.. speedtest1 +$(speedtest1.sql): $(speedtest1) $(MAKEFILE) + $(speedtest1) $(speedtest1.cliflags) --script $@ +batch-runner.list: $(MAKEFILE) $(speedtest1.sql) $(dir.sql)/000-mandelbrot.sql + bash split-speedtest1-script.sh $(dir.sql)/speedtest1.sql + ls -1 $(dir.sql)/*.sql | grep -v speedtest1.sql | sort > $@ +clean-batch: + rm -f batch-runner.list $(dir.sql)/speedtest1*.sql +# ^^^ we don't do this along with 'clean' because we clean/rebuild on +# a regular basis with different -Ox flags and rebuilding the batch +# pieces each time is an unnecessary time sink. +batch: batch-runner.list +all: batch +# end batch-runner.js +######################################################################## +# speedtest1.js... +# speedtest1-common.eflags = emcc flags used by multiple builds of speedtest1 +# speedtest1.eflags = emcc flags used by main build of speedtest1 +speedtest1-common.eflags := $(emcc_opt_full) +speedtest1.eflags := +speedtest1.eflags += -sENVIRONMENT=web +speedtest1.eflags += -sALLOW_MEMORY_GROWTH +speedtest1.eflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) +speedtest1-common.eflags += -sINVOKE_RUN=0 +speedtest1-common.eflags += --no-entry +#speedtest1-common.eflags += -flto +speedtest1-common.eflags += -sABORTING_MALLOC +speedtest1-common.eflags += -sSTRICT_JS +speedtest1-common.eflags += -sMODULARIZE +speedtest1-common.eflags += -Wno-limited-postlink-optimizations +EXPORTED_FUNCTIONS.speedtest1 := $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) +speedtest1-common.eflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.speedtest1) +speedtest1-common.eflags += $(emcc.exportedRuntimeMethods) +speedtest1-common.eflags += -sALLOW_TABLE_GROWTH +speedtest1-common.eflags += -sDYNAMIC_EXECUTION=0 +speedtest1-common.eflags += --minify 0 +speedtest1-common.eflags += -sEXPORT_NAME=$(sqlite3.js.init-func) +speedtest1-common.eflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) +speedtest1-common.eflags += $(pre-post-common.flags) +speedtest1.exit-runtime0 := -sEXIT_RUNTIME=0 +speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 +# Re -sEXIT_RUNTIME=1 vs 0: if it's 1 and speedtest1 crashes, we get +# this error from emscripten: +# +# > native function `free` called after runtime exit (use +# NO_EXIT_RUNTIME to keep it alive after main() exits)) +# +# If it's 0 and it crashes, we get: +# +# > stdio streams had content in them that was not flushed. you should +# set EXIT_RUNTIME to 1 (see the FAQ), or make sure to emit a newline +# when you printf etc. +# +# and pending output is not flushed because it didn't end with a +# newline (by design). The lesser of the two evils seems to be +# -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app +# which runs speedtest1 multiple times. + +$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) + @echo "Making $@ ..." + @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ +speedtest1.js := $(dir.dout)/speedtest1.js +speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js)) +speedtest1.cflags := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM +speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.c) +$(eval $(call call-make-pre-js,speedtest1)) +$(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ + $(pre-post-speedtest1.deps) \ + $(EXPORTED_FUNCTIONS.speedtest1) + @echo "Building $@ ..." + $(emcc.bin) \ + $(speedtest1.eflags) $(speedtest1-common.eflags) $(speedtest1.cflags) \ + $(pre-post-speedtest1.flags) \ + $(SQLITE_OPT) \ + $(speedtest1.exit-runtime0) \ + -o $@ $(speedtest1.cses) -lm + $(maybe-wasm-strip) $(speedtest1.wasm) + ls -la $@ $(speedtest1.wasm) + +speedtest1: $(speedtest1.js) +all: speedtest1 +CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) +# end speedtest1.js +######################################################################## ######################################################################## -# fiddle_remote is the remote destination for the fiddle app. It -# must be a [user@]HOST:/path for rsync. -# Note that the target "should probably" contain a symlink of -# index.html -> fiddle.html. -fiddle_remote ?= -ifeq (,$(fiddle_remote)) -ifneq (,$(wildcard /home/stephan)) - fiddle_remote = wh:www/wh/sqlite3/. -else ifneq (,$(wildcard /home/drh)) - #fiddle_remote = if appropriate, add that user@host:/path here -endif -endif -$(fiddle_files): default -push-fiddle: $(fiddle_files) - @if [ x = "x$(fiddle_remote)" ]; then \ - echo "fiddle_remote must be a [user@]HOST:/path for rsync"; \ - exit 1; \ - fi - rsync -va fiddle/ $(fiddle_remote) -# end fiddle remote push +# Convenience rules to rebuild with various -Ox levels. Much +# experimentation shows -O2 to be the clear winner in terms of speed. +# Note that build times with anything higher than -O0 are somewhat +# painful. + +.PHONY: o0 o1 o2 o3 os oz +o-xtra := -flto +# ^^^^ -flto can have a considerably performance boost at -O0 but +# doubles the build time and seems to have negligible effect on +# higher optimization levels. +o0: clean + $(MAKE) -e "emcc_opt=-O0" +o1: clean + $(MAKE) -e "emcc_opt=-O1 $(o-xtra)" +o2: clean + $(MAKE) -e "emcc_opt=-O2 $(o-xtra)" +o3: clean + $(MAKE) -e "emcc_opt=-O3 $(o-xtra)" +os: clean + @echo "WARNING: -Os can result in a build with mysteriously missing pieces!" + $(MAKE) -e "emcc_opt=-Os $(o-xtra)" +oz: clean + $(MAKE) -e "emcc_opt=-Oz $(o-xtra)" + +######################################################################## +# Sub-makes... + +include fiddle.make + +# Only add wasmfs if wasmfs.enable=1 or we're running (dist)clean +wasmfs.enable ?= $(if $(filter %clean,$(MAKECMDGOALS)),1,0) +ifeq (1,$(wasmfs.enable)) +# wasmfs build disabled 2022-10-19 per /chat discussion. +# OPFS-over-wasmfs was initially a stopgap measure and a convenient +# point of comparison for the OPFS sqlite3_vfs's performance, but it +# currently doubles our deliverables and build maintenance burden for +# little, if any, benefit. +# +######################################################################## +# Some platforms do not support the WASMFS build. Raspberry Pi OS is one +# of them. As such platforms are discovered, add their (uname -m) name +# to PLATFORMS_WITH_NO_WASMFS to exclude the wasmfs build parts. +PLATFORMS_WITH_NO_WASMFS := aarch64 # add any others here +THIS_ARCH := $(shell /usr/bin/uname -m) +ifneq (,$(filter $(THIS_ARCH),$(PLATFORMS_WITH_NO_WASMFS))) +$(info This platform does not support the WASMFS build.) +HAVE_WASMFS := 0 +else +HAVE_WASMFS := 1 +include wasmfs.make +endif +endif +# /wasmfs +######################################################################## + +######################################################################## +# Create deliverables: +ifneq (,$(filter dist,$(MAKECMDGOALS))) +include dist.make +endif + +######################################################################## +# Push files to public wasm-testing.sqlite.org server +wasm-testing.include = $(dir.dout) *.js *.html \ + batch-runner.list $(dir.sql) $(dir.common) $(dir.fiddle) $(dir.jacc) +wasm-testing.exclude = sql/speedtest1.sql +wasm-testing.dir = /jail/sites/wasm-testing +wasm-testing.dest ?= wasm-testing:$(wasm-testing.dir) +# ---------------------^^^^^^^^^^^^ ssh alias +.PHONY: push-testing +push-testing: + rsync -z -e ssh --ignore-times --chown=stephan:www-data --group -r \ + $(patsubst %,--exclude=%,$(wasm-testing.exclude)) \ + $(wasm-testing.include) $(wasm-testing.dest) + @echo "Updating gzipped copies..."; \ + ssh wasm-testing 'cd $(wasm-testing.dir) && bash .gzip' || \ + echo "SSH failed: it's likely that stale content will be served via old gzip files." + +######################################################################## +# If we find a copy of the sqlite.org/wasm docs checked out, copy +# certain files over to it, noting that some need automatable edits... +WDOCS.home ?= ../../../wdoc +.PHONY: update-docs +ifneq (,$(wildcard $(WDOCS.home)/api-index.md)) +WDOCS.jswasm := $(WDOCS.home)/jswasm +update-docs: $(bin.stripccomments) $(sqlite3.js) $(sqlite3.wasm) + @echo "Copying files to the /wasm docs. Be sure to use an -Oz build for this!" + cp $(sqlite3.wasm) $(WDOCS.jswasm)/. + $(bin.stripccomments) -k -k < $(sqlite3.js) \ + | sed -e '/^[ \t]*$$/d' > $(WDOCS.jswasm)/sqlite3.js + cp demo-123.js demo-123.html demo-123-worker.html $(WDOCS.home) + sed -n -e '/EXTRACT_BEGIN/,/EXTRACT_END/p' \ + module-symbols.html > $(WDOCS.home)/module-symbols.html +else +update-docs: + @echo "Cannot find wasm docs checkout."; \ + echo "Pass WDOCS.home=/path/to/wasm/docs/checkout or edit this makefile to suit."; \ + exit 127 +endif +# end /wasm docs ######################################################################## diff --git a/ext/wasm/README-dist.txt b/ext/wasm/README-dist.txt new file mode 100644 index 0000000000..ca6bef93e8 --- /dev/null +++ b/ext/wasm/README-dist.txt @@ -0,0 +1,23 @@ +This is the README for the sqlite3 WASM/JS distribution. + +Main project page: https://sqlite.org + +Documentation: https://sqlite.org/wasm + +This archive contains the sqlite3.js and sqlite3.wasm file which make +up the sqlite3 WASM/JS build. + +The jswasm directory contains the core sqlite3 deliverables and the +top-level directory contains demonstration and test apps. Browsers +will not serve WASM files from file:// URLs, so the demo/test apps +require a web server and that server must include the following +headers in its response when serving the files: + + Cross-Origin-Opener-Policy: same-origin + Cross-Origin-Embedder-Policy: require-corp + +One simple way to get the demo apps up and running on Unix-style +systems is to install althttpd (https://sqlite.org/althttpd) and run: + + althttpd --enable-sab --page index.html + diff --git a/ext/wasm/README.md b/ext/wasm/README.md index 1702e1f427..e8d66865d8 100644 --- a/ext/wasm/README.md +++ b/ext/wasm/README.md @@ -10,6 +10,7 @@ below for Linux environments: ``` # Clone the emscripten repository: +$ sudo apt install git $ git clone https://github.com/emscripten-core/emsdk.git $ cd emsdk @@ -24,6 +25,7 @@ Those parts only need to be run once, but the SDK can be updated using: ``` $ git pull +$ ./emsdk install latest $ ./emsdk activate latest ``` @@ -55,48 +57,48 @@ $ cd ext/wasm $ make ``` -That will generate the fiddle application under -[ext/fiddle](/dir/ext/wasm/fiddle), as `fiddle.html`. That application -cannot, due to XMLHttpRequest security limitations, run if the HTML -file is opened directly in the browser (i.e. if it is opened using a -`file://` URL), so it needs to be served via an HTTP server. For -example, using [althttpd][]: +That will generate the a number of files required for a handful of +test and demo applications which can be accessed via +`index.html`. WASM content cannot, due to XMLHttpRequest security +limitations, be loaded if the containing HTML file is opened directly +in the browser (i.e. if it is opened using a `file://` URL), so it +needs to be served via an HTTP server. For example, using +[althttpd][]: ``` -$ cd ext/wasm/fiddle -$ althttpd -page fiddle.html +$ cd ext/wasm +$ althttpd --enable-sab --max-age 1 --page index.html ``` -That will open the system's browser and run the fiddle app's page. +That will open the system's browser and run the index page, from which +all of the test and demo applications can be accessed. Note that when serving this app via [althttpd][], it must be a version -from 2022-05-17 or newer so that it recognizes the `.wasm` file -extension and responds with the mimetype `application/wasm`, as the -WASM loader is pedantic about that detail. +from 2022-09-26 or newer so that it recognizes the `--enable-sab` +flag, which causes althttpd to emit two HTTP response headers which +are required to enable JavaScript's `SharedArrayBuffer` and `Atomics` +APIs. Those APIs are required in order to enable the OPFS-related +features in the apps which use them. +# Testing on a remote machine that is accessed via SSH -# Known Quirks and Limitations +*NB: The following are developer notes, last validated on 2022-08-18* -Some "impedence mismatch" between C and WASM/JavaScript is to be -expected. + * Remote: Install git, emsdk, and althttpd + * Use a [version of althttpd][althttpd] from + September 26, 2022 or newer. + * Remote: Install the SQLite source tree. CD to ext/wasm + * Remote: "`make`" to build WASM + * Remote: `althttpd --enable-sab --port 8080 --popup` + * Local: `ssh -L 8180:localhost:8080 remote` + * Local: Point your web-browser at http://localhost:8180/index.html -## No I/O - -sqlite3 shell commands which require file I/O or pipes are disabled in -the WASM build. - -## `exit()` Triggered from C - -When C code calls `exit()`, as happens (for example) when running an -"unsafe" command when safe mode is active, WASM's connection to the -sqlite3 shell environment has no sensible choice but to shut down -because `exit()` leaves it in a state we can no longer recover -from. The JavaScript-side application attempts to recognize this and -warn the user that restarting the application is necessary. Currently -the only way to restart it is to reload the page. Restructuring the -shell code such that it could be "rebooted" without restarting the -JS app would require some invasive changes which are not currently -on any TODO list but have not been entirely ruled out long-term. +In order to enable [SharedArrayBuffers](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer), +the web-browser requires that the two extra Cross-Origin lines be present +in HTTP reply headers and that the request must come from "localhost". +Since the web-server is on a different machine from +the web-broser, the localhost requirement means that the connection must be tunneled +using SSH. [emscripten]: https://emscripten.org diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 43d2b0dd5d..6440eba996 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -23,8 +23,6 @@ The overall idea is that the following files get concatenated together, in the listed order, the resulting file is loaded by a browser client: -- `post-js-header.js`\ - Emscripten-specific header for the `--post-js` input. - `sqlite3-api-prologue.js`\ Contains the initial bootstrap setup of the sqlite3 API objects. This is exposed as a function, rather than objects, so that @@ -47,46 +45,55 @@ browser client: independent spinoff project, conceived for the sqlite3 project but maintained separately. - `sqlite3-api-glue.js`\ - Invokes the function exposed by `sqlite3-api-prologue.js`, passing - it a configuration object to configure it for the current WASM - toolchain (noting that it currently requires Emscripten), then - removes that function from the global scope. The result of this file - is a global-scope `sqlite3` object which acts as a namespace for the - API's functionality. This object gets removed from the global scope - after the following files have attached their own features to it. + Invokes functionality exposed by the previous two files to + flesh out low-level parts of `sqlite3-api-prologue.js`. Most of + these pieces related to the `sqlite3.capi.wasm` object. +- `sqlite3-api-build-version.js`\ + Gets created by the build process and populates the + `sqlite3.version` object. This part is not critical, but records the + version of the library against which this module was built. - `sqlite3-api-oo1.js`\ Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. That said, it is not a "required component" and can be elided from builds which do not want it. -- `sqlite3-api-worker.js`\ +- `sqlite3-api-worker1.js`\ A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - `sqlite3-worker.js`\ + - `sqlite3-worker1.js`\ Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module - and runs the Worker API which is implemented in - `sqlite3-api-worker.js`. + and runs the Worker #1 API which is implemented in + `sqlite3-api-worker1.js`. + - `sqlite3-worker1-promiser.js`\ + Is likewise not part of the amalgamated sources and provides + a Promise-based interface into the Worker #1 API. This is + a far user-friendlier way to interface with databases running + in a Worker thread. - `sqlite3-api-opfs.js`\ - is an in-development/experimental sqlite3 VFS wrapper, the goal of - which being to use Google Chrome's Origin-Private FileSystem (OPFS) - storage layer to provide persistent storage for database files in a - browser. It is far from complete. -- `sqlite3-api-cleanup.js`\ - the previous files temporarily create global objects in order to - communicate their state to the files which follow them, and _this_ - file connects any final components together and cleans up those - globals. As of this writing, this code ensures that the previous - files leave no global symbols installed, and it moves the sqlite3 - namespace object into the in-scope Emscripten module. Abstracting - this for other WASM toolchains is TODO. -- `post-js-footer.js`\ - Emscripten-specific footer for the `--post-js` input. This closes - off the lexical scope opened by `post-js-header.js`. + is an sqlite3 VFS implementation which supports Google Chrome's + Origin-Private FileSystem (OPFS) as a storage layer to provide + persistent storage for database files in a browser. It requires... + - `sqlite3-opfs-async-proxy.js`\ + is the asynchronous backend part of the OPFS proxy. It speaks + directly to the (async) OPFS API and channels those results back + to its synchronous counterpart. This file, because it must be + started in its own Worker, is not part of the amalgamation. +- **`api/sqlite3-api-cleanup.js`**\ + The previous files do not immediately extend the library. Instead + they add callback functions to be called during its + bootstrapping. Some also temporarily create global objects in order + to communicate their state to the files which follow them. This file + cleans up any dangling globals and runs the API bootstrapping + process, which is what finally executes the initialization code + installed by the previous files. As of this writing, this code + ensures that the previous files leave no more than a single global + symbol installed. When adapting the API for non-Emscripten + toolchains, this "should" be the only file where changes are needed. The build process glues those files together, resulting in `sqlite3-api.js`, which is everything except for the `post-js-*.js` @@ -99,3 +106,28 @@ The non-JS outlier file is `sqlite3-wasm.c`: it is a proxy for WASM-specific helper functions, at least one of which requires access to private/static `sqlite3.c` internals. `sqlite3.wasm` is compiled from this file rather than `sqlite3.c`. + +The following files are part of the build process but are injected +into the build-generated `sqlite3.js` along with `sqlite3-api.js`. + +- `extern-pre-js.js`\ + Emscripten-specific header for Emscripten's `--extern-pre-js` + flag. As of this writing, that file is only used for experimentation + purposes and holds no code relevant to the production deliverables. +- `pre-js.js`\ + Emscripten-specific header for Emscripten's `--pre-js` flag. This + file is intended as a place to override certain Emscripten behavior + before it starts up, but corner-case Emscripten bugs keep that from + being a reality. +- `post-js-header.js`\ + Emscripten-specific header for the `--post-js` input. It opens up + a lexical scope by starting a post-run handler for Emscripten. +- `post-js-footer.js`\ + Emscripten-specific footer for the `--post-js` input. This closes + off the lexical scope opened by `post-js-header.js`. +- `extern-post-js.js`\ + Emscripten-specific header for Emscripten's `--extern-post-js` + flag. This file overwrites the Emscripten-installed + `sqlite3InitModule()` function with one which, after the module is + loaded, also initializes the asynchronous parts of the sqlite3 + module. For example, the OPFS VFS support. diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js new file mode 100644 index 0000000000..84b99b53a6 --- /dev/null +++ b/ext/wasm/api/extern-post-js.js @@ -0,0 +1,103 @@ +/* extern-post-js.js must be appended to the resulting sqlite3.js + file. It gets its name from being used as the value for the + --extern-post-js=... Emscripten flag. Note that this code, unlike + most of the associated JS code, runs outside of the + Emscripten-generated module init scope, in the current + global scope. */ +(function(){ + /** + In order to hide the sqlite3InitModule()'s resulting Emscripten + module from downstream clients (and simplify our documentation by + being able to elide those details), we rewrite + sqlite3InitModule() to return the sqlite3 object. + + Unfortunately, we cannot modify the module-loader/exporter-based + impls which Emscripten installs at some point in the file above + this. + */ + const originalInit = self.sqlite3InitModule; + if(!originalInit){ + throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); + } + /** + We need to add some state which our custom Module.locateFile() + can see, but an Emscripten limitation currently prevents us from + attaching it to the sqlite3InitModule function object: + + https://github.com/emscripten-core/emscripten/issues/18071 + + The only(?) current workaround is to temporarily stash this state + into the global scope and delete it when sqlite3InitModule() + is called. + */ + const initModuleState = self.sqlite3InitModuleState = Object.assign(Object.create(null),{ + moduleScript: self?.document?.currentScript, + isWorker: ('undefined' !== typeof WorkerGlobalScope), + location: self.location, + urlParams: new URL(self.location.href).searchParams + }); + initModuleState.debugModule = + (new URL(self.location.href).searchParams).has('sqlite3.debugModule') + ? (...args)=>console.warn('sqlite3.debugModule:',...args) + : ()=>{}; + + if(initModuleState.urlParams.has('sqlite3.dir')){ + initModuleState.sqlite3Dir = initModuleState.urlParams.get('sqlite3.dir') +'/'; + }else if(initModuleState.moduleScript){ + const li = initModuleState.moduleScript.src.split('/'); + li.pop(); + initModuleState.sqlite3Dir = li.join('/') + '/'; + } + + self.sqlite3InitModule = (...args)=>{ + //console.warn("Using replaced sqlite3InitModule()",self.location); + return originalInit(...args).then((EmscriptenModule)=>{ + if(self.window!==self && + (EmscriptenModule['ENVIRONMENT_IS_PTHREAD'] + || EmscriptenModule['_pthread_self'] + || 'function'===typeof threadAlert + || self.location.pathname.endsWith('.worker.js') + )){ + /** Workaround for wasmfs-generated worker, which calls this + routine from each individual thread and requires that its + argument be returned. All of the criteria above are fragile, + based solely on inspection of the offending code, not public + Emscripten details. */ + return EmscriptenModule; + } + EmscriptenModule.sqlite3.scriptInfo = initModuleState; + //console.warn("sqlite3.scriptInfo =",EmscriptenModule.sqlite3.scriptInfo); + const f = EmscriptenModule.sqlite3.asyncPostInit; + delete EmscriptenModule.sqlite3.asyncPostInit; + return f(); + }).catch((e)=>{ + console.error("Exception loading sqlite3 module:",e); + throw e; + }); + }; + self.sqlite3InitModule.ready = originalInit.ready; + + if(self.sqlite3InitModuleState.moduleScript){ + const sim = self.sqlite3InitModuleState; + let src = sim.moduleScript.src.split('/'); + src.pop(); + sim.scriptDir = src.join('/') + '/'; + } + initModuleState.debugModule('sqlite3InitModuleState =',initModuleState); + if(0){ + console.warn("Replaced sqlite3InitModule()"); + console.warn("self.location.href =",self.location.href); + if('undefined' !== typeof document){ + console.warn("document.currentScript.src =", + document?.currentScript?.src); + } + } + /* Replace the various module exports performed by the Emscripten + glue... */ + if (typeof exports === 'object' && typeof module === 'object') + module.exports = sqlite3InitModule; + else if (typeof exports === 'object') + exports["sqlite3InitModule"] = sqlite3InitModule; + /* AMD modules get injected in a way we cannot override, + so we can't handle those here. */ +})(); diff --git a/ext/wasm/api/extern-pre-js.js b/ext/wasm/api/extern-pre-js.js new file mode 100644 index 0000000000..7d47d33e33 --- /dev/null +++ b/ext/wasm/api/extern-pre-js.js @@ -0,0 +1,7 @@ +/* extern-pre-js.js must be prepended to the resulting sqlite3.js + file. This file is currently only used for holding snippets during + test and development. + + It gets its name from being used as the value for the + --extern-pre-js=... Emscripten flag. +*/ diff --git a/ext/wasm/api/post-js-footer.js b/ext/wasm/api/post-js-footer.js index ee470928ba..58882cbd9c 100644 --- a/ext/wasm/api/post-js-footer.js +++ b/ext/wasm/api/post-js-footer.js @@ -1,3 +1,4 @@ /* The current function scope was opened via post-js-header.js, which - gets prepended to this at build-time. */ + gets prepended to this at build-time. This file closes that + scope. */ })/*postRun.push(...)*/; diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index 1763188a21..82a80e5a17 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -5,22 +5,21 @@ environment must have been set up already but it will not have loaded its WASM when the code in this file is run. The function it installs will be run after the WASM module is loaded, at which - point the sqlite3 WASM API bits will be set up. + point the sqlite3 JS API bits will get set up. */ if(!Module.postRun) Module.postRun = []; Module.postRun.push(function(Module/*the Emscripten-style module object*/){ 'use strict'; - /* This function will contain: + /* This function will contain at least the following: - post-js-header.js (this file) - sqlite3-api-prologue.js => Bootstrapping bits to attach the rest to - - sqlite3-api-whwasmutil.js => Replacements for much of Emscripten's glue - - sqlite3-api-jaccwabyt.js => Jaccwabyt (C/JS struct binding) + - common/whwasmutil.js => Replacements for much of Emscripten's glue + - jaccwaby/jaccwabyt.js => Jaccwabyt (C/JS struct binding) - sqlite3-api-glue.js => glues previous parts together - - sqlite3-api-oo.js => SQLite3 OO API #1. - - sqlite3-api-worker.js => Worker-based API + - sqlite3-api-oo.js => SQLite3 OO API #1 + - sqlite3-api-worker1.js => Worker-based API + - sqlite3-api-opfs.js => OPFS VFS - sqlite3-api-cleanup.js => final API cleanup - post-js-footer.js => closes this postRun() function - - Whew! */ diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.js new file mode 100644 index 0000000000..f31dea1794 --- /dev/null +++ b/ext/wasm/api/pre-js.js @@ -0,0 +1,100 @@ +/** + BEGIN FILE: api/pre-js.js + + This file is intended to be prepended to the sqlite3.js build using + Emscripten's --pre-js=THIS_FILE flag (or equivalent). +*/ + +// See notes in extern-post-js.js +const sqlite3InitModuleState = self.sqlite3InitModuleState || Object.create(null); +delete self.sqlite3InitModuleState; +sqlite3InitModuleState.debugModule('self.location =',self.location); + +/** + This custom locateFile() tries to figure out where to load `path` + from. The intent is to provide a way for foo/bar/X.js loaded from a + Worker constructor or importScripts() to be able to resolve + foo/bar/X.wasm (in the latter case, with some help): + + 1) If URL param named the same as `path` is set, it is returned. + + 2) If sqlite3InitModuleState.sqlite3Dir is set, then (thatName + path) + is returned (note that it's assumed to end with '/'). + + 3) If this code is running in the main UI thread AND it was loaded + from a SCRIPT tag, the directory part of that URL is used + as the prefix. (This form of resolution unfortunately does not + function for scripts loaded via importScripts().) + + 4) If none of the above apply, (prefix+path) is returned. +*/ +Module['locateFile'] = function(path, prefix) { + let theFile; + const up = this.urlParams; + if(up.has(path)){ + theFile = up.get(path); + }else if(this.sqlite3Dir){ + theFile = this.sqlite3Dir + path; + }else if(this.scriptDir){ + theFile = this.scriptDir + path; + }else{ + theFile = prefix + path; + } + sqlite3InitModuleState.debugModule( + "locateFile(",arguments[0], ',', arguments[1],")", + 'sqlite3InitModuleState.scriptDir =',this.scriptDir, + 'up.entries() =',Array.from(up.entries()), + "result =", theFile + ); + return theFile; +}.bind(sqlite3InitModuleState); + +/** + Bug warning: a custom Module.instantiateWasm() does not work + in WASMFS builds: + + https://github.com/emscripten-core/emscripten/issues/17951 + + In such builds we must disable this. +*/ +const xNameOfInstantiateWasm = true + ? 'instantiateWasm' + : 'emscripten-bug-17951'; +Module[xNameOfInstantiateWasm] = function callee(imports,onSuccess){ + imports.env.foo = function(){}; + const uri = Module.locateFile( + callee.uri, ( + ('undefined'===typeof scriptDirectory/*var defined by Emscripten glue*/) + ? '' : scriptDirectory) + ); + sqlite3InitModuleState.debugModule( + "instantiateWasm() uri =", uri + ); + const wfetch = ()=>fetch(uri, {credentials: 'same-origin'}); + const loadWasm = WebAssembly.instantiateStreaming + ? async ()=>{ + return WebAssembly.instantiateStreaming(wfetch(), imports) + .then((arg)=>onSuccess(arg.instance, arg.module)); + } + : async ()=>{ // Safari < v15 + return wfetch() + .then(response => response.arrayBuffer()) + .then(bytes => WebAssembly.instantiate(bytes, imports)) + .then((arg)=>onSuccess(arg.instance, arg.module)); + }; + loadWasm(); + return {}; +}; +/* + It is literally impossible to reliably get the name of _this_ script + at runtime, so impossible to derive X.wasm from script name + X.js. Thus we need, at build-time, to redefine + Module[xNameOfInstantiateWasm].uri by appending it to a build-specific + copy of this file with the name of the wasm file. This is apparently + why Emscripten hard-codes the name of the wasm file into their glue + scripts. +*/ +Module[xNameOfInstantiateWasm].uri = 'sqlite3.wasm'; +/* END FILE: api/pre-js.js, noting that the build process may add a + line after this one to change the above .uri to a build-specific + one. */ diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index a2f921a5d7..bef4d91d70 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -11,34 +11,60 @@ *********************************************************************** This file is the tail end of the sqlite3-api.js constellation, - intended to be appended after all other files so that it can clean - up any global systems temporarily used for setting up the API's - various subsystems. + intended to be appended after all other sqlite3-api-*.js files so + that it can finalize any setup and clean up any global symbols + temporarily used for setting up the API's various subsystems. */ 'use strict'; -self.sqlite3.postInit.forEach( - self.importScripts/*global is a Worker*/ - ? function(f){ - /** We try/catch/report for the sake of failures which happen in - a Worker, as those exceptions can otherwise get completely - swallowed, leading to confusing downstream errors which have - nothing to do with this failure. */ - try{ f(self, self.sqlite3) } - catch(e){ - console.error("Error in postInit() function:",e); - throw e; - } - } - : (f)=>f(self, self.sqlite3) -); -delete self.sqlite3.postInit; -if(self.location && +self.location.port > 1024){ - console.warn("Installing sqlite3 bits as global S for dev-testing purposes."); - self.S = self.sqlite3; +if('undefined' !== typeof Module){ // presumably an Emscripten build + /** + Install a suitable default configuration for sqlite3ApiBootstrap(). + */ + const SABC = Object.assign( + Object.create(null), { + Module: Module /* ==> Currently needs to be exposed here for + test code. NOT part of the public API. */, + exports: Module['asm'], + memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */ + }, + self.sqlite3ApiConfig || Object.create(null) + ); + + /** + For current (2022-08-22) purposes, automatically call + sqlite3ApiBootstrap(). That decision will be revisited at some + point, as we really want client code to be able to call this to + configure certain parts. Clients may modify + self.sqlite3ApiBootstrap.defaultConfig to tweak the default + configuration used by a no-args call to sqlite3ApiBootstrap(), + but must have first loaded their WASM module in order to be + able to provide the necessary configuration state. + */ + //console.warn("self.sqlite3ApiConfig = ",self.sqlite3ApiConfig); + self.sqlite3ApiConfig = SABC; + let sqlite3; + try{ + sqlite3 = self.sqlite3ApiBootstrap(); + }catch(e){ + console.error("sqlite3ApiBootstrap() error:",e); + throw e; + }finally{ + delete self.sqlite3ApiBootstrap; + delete self.sqlite3ApiConfig; + } + + if(self.location && +self.location.port > 1024){ + console.warn("Installing sqlite3 bits as global S for local dev/test purposes."); + self.S = sqlite3; + } + + /* Clean up temporary references to our APIs... */ + delete sqlite3.util /* arguable, but these are (currently) internal-use APIs */; + Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to + pass the sqlite3 object off to the client. */; +}else{ + console.warn("This is not running in an Emscripten module context, so", + "self.sqlite3ApiBootstrap() is _not_ being called due to lack", + "of config info for the WASM environment.", + "It must be called manually."); } -/* Clean up temporary global-scope references to our APIs... */ -self.sqlite3.config.Module.sqlite3 = self.sqlite3 -/* ^^^^ Currently needed by test code and Worker API setup */; -delete self.sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; -delete self.sqlite3 /* clean up our global-scope reference */; -//console.warn("Module.sqlite3 =",Module.sqlite3); diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index e962c93b64..86aa1d1813 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -16,162 +16,14 @@ initializes the main API pieces so that the downstream components (e.g. sqlite3-api-oo1.js) have all that they need. */ -(function(self){ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; - - self.sqlite3 = self.sqlite3ApiBootstrap({ - Module: Module /* ==> Emscripten-style Module object. Currently - needs to be exposed here for test code. NOT part - of the public API. */, - exports: Module['asm'], - memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */, - bigIntEnabled: !!self.BigInt64Array, - allocExportName: 'malloc', - deallocExportName: 'free' - }); - delete self.sqlite3ApiBootstrap; - - const sqlite3 = self.sqlite3; - const capi = sqlite3.capi, wasm = capi.wasm, util = capi.util; - self.WhWasmUtilInstaller(capi.wasm); + const toss3 = sqlite3.SQLite3Error.toss; + const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; + self.WhWasmUtilInstaller(wasm); delete self.WhWasmUtilInstaller; - if(0){ - /* "The problem" is that the following isn't type-safe. - OTOH, nothing about WASM pointers is. */ - /** - Add the `.pointer` xWrap() signature entry to extend - the `pointer` arg handler to check for a `pointer` - property. This can be used to permit, e.g., passing - an SQLite3.DB instance to a C-style sqlite3_xxx function - which takes an `sqlite3*` argument. - */ - const oldP = wasm.xWrap.argAdapter('pointer'); - const adapter = function(v){ - if(v && 'object'===typeof v && v.constructor){ - const x = v.pointer; - if(Number.isInteger(x)) return x; - else toss("Invalid (object) type for pointer-type argument."); - } - return oldP(v); - }; - wasm.xWrap.argAdapter('.pointer', adapter); - } - - // WhWasmUtil.xWrap() bindings... - { - /** - Add some descriptive xWrap() aliases for '*' intended to - (A) initially improve readability/correctness of capi.signatures - and (B) eventually perhaps provide some sort of type-safety - in their conversions. - */ - const aPtr = wasm.xWrap.argAdapter('*'); - wasm.xWrap.argAdapter('sqlite3*', aPtr)('sqlite3_stmt*', aPtr); - - /** - Populate api object with sqlite3_...() by binding the "raw" wasm - exports into type-converting proxies using wasm.xWrap(). - */ - for(const e of wasm.bindingSignatures){ - capi[e[0]] = wasm.xWrap.apply(null, e); - } - - /* For functions which cannot work properly unless - wasm.bigIntEnabled is true, install a bogus impl which - throws if called when bigIntEnabled is false. */ - const fI64Disabled = function(fname){ - return ()=>toss(fname+"() disabled due to lack", - "of BigInt support in this build."); - }; - for(const e of wasm.bindingSignatures.int64){ - capi[e[0]] = wasm.bigIntEnabled - ? wasm.xWrap.apply(null, e) - : fI64Disabled(e[0]); - } - - if(wasm.exports.sqlite3_wasm_db_error){ - util.sqlite3_wasm_db_error = capi.wasm.xWrap( - 'sqlite3_wasm_db_error', 'int', 'sqlite3*', 'int', 'string' - ); - }else{ - util.sqlite3_wasm_db_error = function(pDb,errCode,msg){ - console.warn("sqlite3_wasm_db_error() is not exported.",arguments); - return errCode; - }; - } - - /** - When registering a VFS and its related components it may be - necessary to ensure that JS keeps a reference to them to keep - them from getting garbage collected. Simply pass each such value - to this function and a reference will be held to it for the life - of the app. - */ - capi.sqlite3_vfs_register.addReference = function f(...args){ - if(!f._) f._ = []; - f._.push(...args); - }; - - }/*xWrap() bindings*/; - - /** - Scope-local holder of the two impls of sqlite3_prepare_v2/v3(). - */ - const __prepare = Object.create(null); - /** - This binding expects a JS string as its 2nd argument and - null as its final argument. In order to compile multiple - statements from a single string, the "full" impl (see - below) must be used. - */ - __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', - "int", ["sqlite3*", "string", - "int"/*MUST always be negative*/, - "int", "**", - "**"/*MUST be 0 or null or undefined!*/]); - /** - Impl which requires that the 2nd argument be a pointer - to the SQL string, instead of being converted to a - string. This variant is necessary for cases where we - require a non-NULL value for the final argument - (exec()'ing multiple statements from one input - string). For simpler cases, where only the first - statement in the SQL string is required, the wrapper - named sqlite3_prepare_v2() is sufficient and easier to - use because it doesn't require dealing with pointers. - */ - __prepare.full = wasm.xWrap('sqlite3_prepare_v3', - "int", ["sqlite3*", "*", "int", "int", - "**", "**"]); - - /* Documented in the api object's initializer. */ - capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ - /* 2022-07-08: xWrap() 'string' arg handling may be able do this - special-case handling for us. It needs to be tested. Or maybe - not: we always want to treat pzTail as null when passed a - non-pointer SQL string and the argument adapters don't have - enough state to know that. Maybe they could/should, by passing - the currently-collected args as an array as the 2nd arg to the - argument adapters? Or maybe we collect all args in an array, - pass that to an optional post-args-collected callback, and give - it a chance to manipulate the args before we pass them on? */ - if(util.isSQLableTypedArray(sql)) sql = util.typedArrayToString(sql); - switch(typeof sql){ - case 'string': return __prepare.basic(pDb, sql, -1, prepFlags, ppStmt, null); - case 'number': return __prepare.full(pDb, sql, sqlLen||-1, prepFlags, ppStmt, pzTail); - default: - return util.sqlite3_wasm_db_error( - pDb, capi.SQLITE_MISUSE, - "Invalid SQL argument type for sqlite3_prepare_v2/v3()." - ); - } - }; - - capi.sqlite3_prepare_v2 = - (pDb, sql, sqlLen, ppStmt, pzTail)=>capi.sqlite3_prepare_v3(pDb, sql, sqlLen, 0, ppStmt, pzTail); - /** Install JS<->C struct bindings for the non-opaque struct types we need... */ @@ -185,6 +37,541 @@ }); delete self.Jaccwabyt; + if(0){ + /* "The problem" is that the following isn't even remotely + type-safe. OTOH, nothing about WASM pointers is. */ + const argPointer = wasm.xWrap.argAdapter('*'); + wasm.xWrap.argAdapter('StructType', (v)=>{ + if(v && v.constructor && v instanceof StructBinder.StructType){ + v = v.pointer; + } + return wasm.isPtr(v) + ? argPointer(v) + : toss("Invalid (object) type for StructType-type argument."); + }); + } + + {/* Convert Arrays and certain TypedArrays to strings for + 'flexible-string'-type arguments */ + const xString = wasm.xWrap.argAdapter('string'); + wasm.xWrap.argAdapter( + 'flexible-string', (v)=>xString(util.flexibleString(v)) + ); + } + + if(1){// WhWasmUtil.xWrap() bindings... + /** + Add some descriptive xWrap() aliases for '*' intended to (A) + initially improve readability/correctness of capi.signatures + and (B) eventually perhaps provide automatic conversion from + higher-level representations, e.g. capi.sqlite3_vfs to + `sqlite3_vfs*` via capi.sqlite3_vfs.pointer. + */ + const aPtr = wasm.xWrap.argAdapter('*'); + wasm.xWrap.argAdapter('sqlite3*', aPtr) + ('sqlite3_stmt*', aPtr) + ('sqlite3_context*', aPtr) + ('sqlite3_value*', aPtr) + ('sqlite3_vfs*', aPtr) + ('void*', aPtr); + wasm.xWrap.resultAdapter('sqlite3*', aPtr) + ('sqlite3_context*', aPtr) + ('sqlite3_stmt*', aPtr) + ('sqlite3_vfs*', aPtr) + ('void*', aPtr); + + /** + Populate api object with sqlite3_...() by binding the "raw" wasm + exports into type-converting proxies using wasm.xWrap(). + */ + for(const e of wasm.bindingSignatures){ + capi[e[0]] = wasm.xWrap.apply(null, e); + } + for(const e of wasm.bindingSignatures.wasm){ + wasm[e[0]] = wasm.xWrap.apply(null, e); + } + + /* For C API functions which cannot work properly unless + wasm.bigIntEnabled is true, install a bogus impl which + throws if called when bigIntEnabled is false. */ + const fI64Disabled = function(fname){ + return ()=>toss(fname+"() disabled due to lack", + "of BigInt support in this build."); + }; + for(const e of wasm.bindingSignatures.int64){ + capi[e[0]] = wasm.bigIntEnabled + ? wasm.xWrap.apply(null, e) + : fI64Disabled(e[0]); + } + + /* There's no(?) need to expose bindingSignatures to clients, + implicitly making it part of the public interface. */ + delete wasm.bindingSignatures; + + if(wasm.exports.sqlite3_wasm_db_error){ + util.sqlite3_wasm_db_error = wasm.xWrap( + 'sqlite3_wasm_db_error', 'int', 'sqlite3*', 'int', 'string' + ); + }else{ + util.sqlite3_wasm_db_error = function(pDb,errCode,msg){ + console.warn("sqlite3_wasm_db_error() is not exported.",arguments); + return errCode; + }; + } + + }/*xWrap() bindings*/; + + /** + When registering a VFS and its related components it may be + necessary to ensure that JS keeps a reference to them to keep + them from getting garbage collected. Simply pass each such value + to this function and a reference will be held to it for the life + of the app. + */ + capi.sqlite3_vfs_register.addReference = function f(...args){ + if(!f._) f._ = []; + f._.push(...args); + }; + + /** + Internal helper to assist in validating call argument counts in + the hand-written sqlite3_xyz() wrappers. We do this only for + consistency with non-special-case wrappings. + */ + const __dbArgcMismatch = (pDb,f,n)=>{ + return sqlite3.util.sqlite3_wasm_db_error(pDb, capi.SQLITE_MISUSE, + f+"() requires "+n+" argument"+ + (1===n?"":'s')+"."); + }; + + /** + Helper for flexible-string conversions which require a + byte-length counterpart argument. Passed a value and its + ostensible length, this function returns [V,N], where V + is either v or a transformed copy of v and N is either n, + -1, or the byte length of v (if it's a byte array). + */ + const __flexiString = function(v,n){ + if('string'===typeof v){ + n = -1; + }else if(util.isSQLableTypedArray(v)){ + n = v.byteLength; + v = util.typedArrayToString(v); + }else if(Array.isArray(v)){ + v = v.join(""); + n = -1; + } + return [v, n]; + }; + + if(1){/* Special-case handling of sqlite3_exec() */ + const __exec = wasm.xWrap("sqlite3_exec", "int", + ["sqlite3*", "flexible-string", "*", "*", "**"]); + /* Documented in the api object's initializer. */ + capi.sqlite3_exec = function f(pDb, sql, callback, pVoid, pErrMsg){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(pDb,"sqlite3_exec",f.length); + }else if('function' !== typeof callback){ + return __exec(pDb, sql, callback, pVoid, pErrMsg); + } + /* Wrap the callback in a WASM-bound function and convert the callback's + `(char**)` arguments to arrays of strings... */ + const cbwrap = function(pVoid, nCols, pColVals, pColNames){ + let rc = capi.SQLITE_ERROR; + try { + let aVals = [], aNames = [], i = 0, offset = 0; + for( ; i < nCols; offset += (wasm.ptrSizeof * ++i) ){ + aVals.push( wasm.cstringToJs(wasm.getPtrValue(pColVals + offset)) ); + aNames.push( wasm.cstringToJs(wasm.getPtrValue(pColNames + offset)) ); + } + rc = callback(pVoid, nCols, aVals, aNames) | 0; + /* The first 2 args of the callback are useless for JS but + we want the JS mapping of the C API to be as close to the + C API as possible. */ + }catch(e){ + /* If we set the db error state here, the higher-level exec() call + replaces it with its own, so we have no way of reporting the + exception message except the console. We must not propagate + exceptions through the C API. */ + } + return rc; + }; + let pFunc, rc; + try{ + pFunc = wasm.installFunction("ipipp", cbwrap); + rc = __exec(pDb, sql, pFunc, pVoid, pErrMsg); + }catch(e){ + rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, + "Error running exec(): "+e.message); + }finally{ + if(pFunc) wasm.uninstallFunction(pFunc); + } + return rc; + }; + }/*sqlite3_exec() proxy*/; + + if(1){/* Special-case handling of sqlite3_create_function_v2() + and sqlite3_create_window_function() */ + const sqlite3CreateFunction = wasm.xWrap( + "sqlite3_create_function_v2", "int", + ["sqlite3*", "string"/*funcName*/, "int"/*nArg*/, + "int"/*eTextRep*/, "*"/*pApp*/, + "*"/*xStep*/,"*"/*xFinal*/, "*"/*xValue*/, "*"/*xDestroy*/] + ); + const sqlite3CreateWindowFunction = wasm.xWrap( + "sqlite3_create_window_function", "int", + ["sqlite3*", "string"/*funcName*/, "int"/*nArg*/, + "int"/*eTextRep*/, "*"/*pApp*/, + "*"/*xStep*/,"*"/*xFinal*/, "*"/*xValue*/, + "*"/*xInverse*/, "*"/*xDestroy*/] + ); + + const __udfSetResult = function(pCtx, val){ + //console.warn("udfSetResult",typeof val, val); + switch(typeof val) { + case 'undefined': + /* Assume that the client already called sqlite3_result_xxx(). */ + break; + case 'boolean': + capi.sqlite3_result_int(pCtx, val ? 1 : 0); + break; + case 'bigint': + if(wasm.bigIntEnabled){ + if(util.bigIntFits64(val)) capi.sqlite3_result_int64(pCtx, val); + else toss3("BigInt value",val.toString(),"is too BigInt for int64."); + }else if(util.bigIntFits32(val)){ + capi.sqlite3_result_int(pCtx, Number(val)); + }else if(util.bigIntFitsDouble(val)){ + capi.sqlite3_result_double(pCtx, Number(val)); + }else{ + toss3("BigInt value",val.toString(),"is too BigInt."); + } + break; + case 'number': { + (util.isInt32(val) + ? capi.sqlite3_result_int + : capi.sqlite3_result_double)(pCtx, val); + break; + } + case 'string': + capi.sqlite3_result_text(pCtx, val, -1, capi.SQLITE_TRANSIENT); + break; + case 'object': + if(null===val/*yes, typeof null === 'object'*/) { + capi.sqlite3_result_null(pCtx); + break; + }else if(util.isBindableTypedArray(val)){ + const pBlob = wasm.allocFromTypedArray(val); + capi.sqlite3_result_blob( + pCtx, pBlob, val.byteLength, + wasm.exports[sqlite3.config.deallocExportName] + ); + break; + } + // else fall through + default: + toss3("Don't not how to handle this UDF result value:",(typeof val), val); + }; + }/*__udfSetResult()*/; + + const __udfConvertArgs = function(argc, pArgv){ + let i, pVal, valType, arg; + const tgt = []; + for(i = 0; i < argc; ++i){ + pVal = wasm.getPtrValue(pArgv + (wasm.ptrSizeof * i)); + /** + Curiously: despite ostensibly requiring 8-byte + alignment, the pArgv array is parcelled into chunks of + 4 bytes (1 pointer each). The values those point to + have 8-byte alignment but the individual argv entries + do not. + */ + valType = capi.sqlite3_value_type(pVal); + switch(valType){ + case capi.SQLITE_INTEGER: + if(wasm.bigIntEnabled){ + arg = capi.sqlite3_value_int64(pVal); + if(util.bigIntFitsDouble(arg)) arg = Number(arg); + } + else arg = capi.sqlite3_value_double(pVal)/*yes, double, for larger integers*/; + break; + case capi.SQLITE_FLOAT: + arg = capi.sqlite3_value_double(pVal); + break; + case capi.SQLITE_TEXT: + arg = capi.sqlite3_value_text(pVal); + break; + case capi.SQLITE_BLOB:{ + const n = capi.sqlite3_value_bytes(pVal); + const pBlob = capi.sqlite3_value_blob(pVal); + if(n && !pBlob) sqlite3.WasmAllocError.toss( + "Cannot allocate memory for blob argument of",n,"byte(s)" + ); + arg = n ? wasm.heap8u().slice(pBlob, pBlob + Number(n)) : null; + break; + } + case capi.SQLITE_NULL: + arg = null; break; + default: + toss3("Unhandled sqlite3_value_type()",valType, + "is possibly indicative of incorrect", + "pointer size assumption."); + } + tgt.push(arg); + } + return tgt; + }/*__udfConvertArgs()*/; + + const __udfSetError = (pCtx, e)=>{ + if(e instanceof sqlite3.WasmAllocError){ + capi.sqlite3_result_error_nomem(pCtx); + }else{ + const msg = ('string'===typeof e) ? e : e.message; + capi.sqlite3_result_error(pCtx, msg, -1); + } + }; + + const __xFunc = function(callback){ + return function(pCtx, argc, pArgv){ + try{ __udfSetResult(pCtx, callback(pCtx, ...__udfConvertArgs(argc, pArgv))) } + catch(e){ + //console.error('xFunc() caught:',e); + __udfSetError(pCtx, e); + } + }; + }; + + const __xInverseAndStep = function(callback){ + return function(pCtx, argc, pArgv){ + try{ callback(pCtx, ...__udfConvertArgs(argc, pArgv)) } + catch(e){ __udfSetError(pCtx, e) } + }; + }; + + const __xFinalAndValue = function(callback){ + return function(pCtx){ + try{ __udfSetResult(pCtx, callback(pCtx)) } + catch(e){ __udfSetError(pCtx, e) } + }; + }; + + const __xDestroy = function(callback){ + return function(pVoid){ + try{ callback(pVoid) } + catch(e){ console.error("UDF xDestroy method threw:",e) } + }; + }; + + const __xMap = Object.assign(Object.create(null), { + xFunc: {sig:'v(pip)', f:__xFunc}, + xStep: {sig:'v(pip)', f:__xInverseAndStep}, + xInverse: {sig:'v(pip)', f:__xInverseAndStep}, + xFinal: {sig:'v(p)', f:__xFinalAndValue}, + xValue: {sig:'v(p)', f:__xFinalAndValue}, + xDestroy: {sig:'v(p)', f:__xDestroy} + }); + + const __xWrapFuncs = function(theFuncs, tgtUninst){ + const rc = [] + let k; + for(k in theFuncs){ + let fArg = theFuncs[k]; + if('function'===typeof fArg){ + const w = __xMap[k]; + fArg = wasm.installFunction(w.sig, w.f(fArg)); + tgtUninst.push(fArg); + } + rc.push(fArg); + } + return rc; + }; + + /* Documented in the api object's initializer. */ + capi.sqlite3_create_function_v2 = function f( + pDb, funcName, nArg, eTextRep, pApp, + xFunc, //void (*xFunc)(sqlite3_context*,int,sqlite3_value**) + xStep, //void (*xStep)(sqlite3_context*,int,sqlite3_value**) + xFinal, //void (*xFinal)(sqlite3_context*) + xDestroy //void (*xDestroy)(void*) + ){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(pDb,"sqlite3_create_function_v2",f.length); + } + /* Wrap the callbacks in a WASM-bound functions... */ + const uninstall = [/*funcs to uninstall on error*/]; + let rc; + try{ + const funcArgs = __xWrapFuncs({xFunc, xStep, xFinal, xDestroy}, + uninstall); + rc = sqlite3CreateFunction(pDb, funcName, nArg, eTextRep, + pApp, ...funcArgs); + }catch(e){ + console.error("sqlite3_create_function_v2() setup threw:",e); + for(let v of uninstall){ + wasm.uninstallFunction(v); + } + rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, + "Creation of UDF threw: "+e.message); + } + return rc; + }; + + capi.sqlite3_create_function = function f( + pDb, funcName, nArg, eTextRep, pApp, + xFunc, xStep, xFinal + ){ + return (f.length===arguments.length) + ? capi.sqlite3_create_function_v2(pDb, funcName, nArg, eTextRep, + pApp, xFunc, xStep, xFinal, 0) + : __dbArgcMismatch(pDb,"sqlite3_create_function",f.length); + }; + + /* Documented in the api object's initializer. */ + capi.sqlite3_create_window_function = function f( + pDb, funcName, nArg, eTextRep, pApp, + xStep, //void (*xStep)(sqlite3_context*,int,sqlite3_value**) + xFinal, //void (*xFinal)(sqlite3_context*) + xValue, //void (*xFinal)(sqlite3_context*) + xInverse,//void (*xStep)(sqlite3_context*,int,sqlite3_value**) + xDestroy //void (*xDestroy)(void*) + ){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(pDb,"sqlite3_create_window_function",f.length); + } + /* Wrap the callbacks in a WASM-bound functions... */ + const uninstall = [/*funcs to uninstall on error*/]; + let rc; + try{ + const funcArgs = __xWrapFuncs({xStep, xFinal, xValue, xInverse, xDestroy}, + uninstall); + rc = sqlite3CreateWindowFunction(pDb, funcName, nArg, eTextRep, + pApp, ...funcArgs); + }catch(e){ + console.error("sqlite3_create_window_function() setup threw:",e); + for(let v of uninstall){ + wasm.uninstallFunction(v); + } + rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, + "Creation of UDF threw: "+e.message); + } + return rc; + }; + /** + A helper for UDFs implemented in JS and bound to WASM by the + client. Given a JS value, udfSetResult(pCtx,X) calls one of the + sqlite3_result_xyz(pCtx,...) routines, depending on X's data + type: + + - `null`: sqlite3_result_null() + - `boolean`: sqlite3_result_int() + - `number`: sqlite3_result_int() or sqlite3_result_double() + - `string`: sqlite3_result_text() + - Uint8Array or Int8Array: sqlite3_result_blob() + - `undefined`: indicates that the UDF called one of the + `sqlite3_result_xyz()` routines on its own, making this + function a no-op. Results are _undefined_ if this function is + passed the `undefined` value but did _not_ call one of the + `sqlite3_result_xyz()` routines. + + Anything else triggers sqlite3_result_error(). + */ + capi.sqlite3_create_function_v2.udfSetResult = + capi.sqlite3_create_function.udfSetResult = + capi.sqlite3_create_window_function.udfSetResult = __udfSetResult; + + /** + A helper for UDFs implemented in JS and bound to WASM by the + client. When passed the + (argc,argv) values from the UDF-related functions which receive + them (xFunc, xStep, xInverse), it creates a JS array + representing those arguments, converting each to JS in a manner + appropriate to its data type: numeric, text, blob + (Uint8Array), or null. + + Results are undefined if it's passed anything other than those + two arguments from those specific contexts. + + Thus an argc of 4 will result in a length-4 array containing + the converted values from the corresponding argv. + + The conversion will throw only on allocation error or an internal + error. + */ + capi.sqlite3_create_function_v2.udfConvertArgs = + capi.sqlite3_create_function.udfConvertArgs = + capi.sqlite3_create_window_function.udfConvertArgs = __udfConvertArgs; + + /** + A helper for UDFs implemented in JS and bound to WASM by the + client. It expects to be a passed `(sqlite3_context*, Error)` + (an exception object or message string). And it sets the + current UDF's result to sqlite3_result_error_nomem() or + sqlite3_result_error(), depending on whether the 2nd argument + is a sqlite3.WasmAllocError object or not. + */ + capi.sqlite3_create_function_v2.udfSetError = + capi.sqlite3_create_function.udfSetError = + capi.sqlite3_create_window_function.udfSetError = __udfSetError; + + }/*sqlite3_create_function_v2() and sqlite3_create_window_function() proxies*/; + + if(1){/* Special-case handling of sqlite3_prepare_v2() and + sqlite3_prepare_v3() */ + /** + Scope-local holder of the two impls of sqlite3_prepare_v2/v3(). + */ + const __prepare = Object.create(null); + /** + This binding expects a JS string as its 2nd argument and + null as its final argument. In order to compile multiple + statements from a single string, the "full" impl (see + below) must be used. + */ + __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "string", + "int"/*ignored for this impl!*/, + "int", "**", + "**"/*MUST be 0 or null or undefined!*/]); + /** + Impl which requires that the 2nd argument be a pointer + to the SQL string, instead of being converted to a + string. This variant is necessary for cases where we + require a non-NULL value for the final argument + (exec()'ing multiple statements from one input + string). For simpler cases, where only the first + statement in the SQL string is required, the wrapper + named sqlite3_prepare_v2() is sufficient and easier to + use because it doesn't require dealing with pointers. + */ + __prepare.full = wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "*", "int", "int", + "**", "**"]); + + /* Documented in the api object's initializer. */ + capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(pDb,"sqlite3_prepare_v3",f.length); + } + const [xSql, xSqlLen] = __flexiString(sql, sqlLen); + switch(typeof xSql){ + case 'string': return __prepare.basic(pDb, xSql, xSqlLen, prepFlags, ppStmt, null); + case 'number': return __prepare.full(pDb, xSql, xSqlLen, prepFlags, ppStmt, pzTail); + default: + return util.sqlite3_wasm_db_error( + pDb, capi.SQLITE_MISUSE, + "Invalid SQL argument type for sqlite3_prepare_v2/v3()." + ); + } + }; + + /* Documented in the api object's initializer. */ + capi.sqlite3_prepare_v2 = function f(pDb, sql, sqlLen, ppStmt, pzTail){ + return (f.length===arguments.length) + ? capi.sqlite3_prepare_v3(pDb, sql, sqlLen, 0, ppStmt, pzTail) + : __dbArgcMismatch(pDb,"sqlite3_prepare_v2",f.length); + }; + }/*sqlite3_prepare_v2/v3()*/; + {/* Import C-level constants and structs... */ const cJson = wasm.xCall('sqlite3_wasm_enum_json'); if(!cJson){ @@ -194,18 +581,140 @@ wasm.ctype = JSON.parse(wasm.cstringToJs(cJson)); //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); for(const t of ['access', 'blobFinalizers', 'dataTypes', - 'encodings', 'flock', 'ioCap', + 'encodings', 'fcntl', 'flock', 'ioCap', 'openFlags', 'prepareFlags', 'resultCodes', - 'syncFlags', 'udfFlags', 'version' + 'serialize', 'syncFlags', 'trace', 'udfFlags', + 'version' ]){ - for(const [k,v] of Object.entries(wasm.ctype[t])){ - capi[k] = v; + for(const e of Object.entries(wasm.ctype[t])){ + // ^^^ [k,v] there triggers a buggy code transormation via one + // of the Emscripten-driven optimizers. + capi[e[0]] = e[1]; } } - /* Bind all registered C-side structs... */ - for(const s of wasm.ctype.structs){ - capi[s.name] = sqlite3.StructBinder(s); + const __rcMap = Object.create(null); + for(const t of ['resultCodes']){ + for(const e of Object.entries(wasm.ctype[t])){ + __rcMap[e[1]] = e[0]; + } } - } + /** + For the given integer, returns the SQLITE_xxx result code as a + string, or undefined if no such mapping is found. + */ + capi.sqlite3_js_rc_str = (rc)=>__rcMap[rc]; + /* Bind all registered C-side structs... */ + const notThese = Object.assign(Object.create(null),{ + // Structs NOT to register + WasmTestStruct: true + }); + if(!util.isUIThread()){ + /* We remove the kvvfs VFS from Worker threads below. */ + notThese.sqlite3_kvvfs_methods = true; + } + for(const s of wasm.ctype.structs){ + if(!notThese[s.name]){ + capi[s.name] = sqlite3.StructBinder(s); + } + } + }/*end C constant imports*/ -})(self); + const pKvvfs = capi.sqlite3_vfs_find("kvvfs"); + if( pKvvfs ){/* kvvfs-specific glue */ + if(util.isUIThread()){ + const kvvfsMethods = new capi.sqlite3_kvvfs_methods( + wasm.exports.sqlite3_wasm_kvvfs_methods() + ); + delete capi.sqlite3_kvvfs_methods; + + const kvvfsMakeKey = wasm.exports.sqlite3_wasm_kvvfsMakeKeyOnPstack, + pstack = wasm.pstack, + pAllocRaw = wasm.exports.sqlite3_wasm_pstack_alloc; + + const kvvfsStorage = (zClass)=> + ((115/*=='s'*/===wasm.getMemValue(zClass)) + ? sessionStorage : localStorage); + + /** + Implementations for members of the object referred to by + sqlite3_wasm_kvvfs_methods(). We swap out the native + implementations with these, which use localStorage or + sessionStorage for their backing store. + */ + const kvvfsImpls = { + xRead: (zClass, zKey, zBuf, nBuf)=>{ + const stack = pstack.pointer, + astack = wasm.scopedAllocPush(); + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return -3/*OOM*/; + const jKey = wasm.cstringToJs(zXKey); + const jV = kvvfsStorage(zClass).getItem(jKey); + if(!jV) return -1; + const nV = jV.length /* Note that we are relying 100% on v being + ASCII so that jV.length is equal to the + C-string's byte length. */; + if(nBuf<=0) return nV; + else if(1===nBuf){ + wasm.setMemValue(zBuf, 0); + return nV; + } + const zV = wasm.scopedAllocCString(jV); + if(nBuf > nV + 1) nBuf = nV + 1; + wasm.heap8u().copyWithin(zBuf, zV, zV + nBuf - 1); + wasm.setMemValue(zBuf + nBuf - 1, 0); + return nBuf - 1; + }catch(e){ + console.error("kvstorageRead()",e); + return -2; + }finally{ + pstack.restore(stack); + wasm.scopedAllocPop(astack); + } + }, + xWrite: (zClass, zKey, zData)=>{ + const stack = pstack.pointer; + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return 1/*OOM*/; + const jKey = wasm.cstringToJs(zXKey); + kvvfsStorage(zClass).setItem(jKey, wasm.cstringToJs(zData)); + return 0; + }catch(e){ + console.error("kvstorageWrite()",e); + return capi.SQLITE_IOERR; + }finally{ + pstack.restore(stack); + } + }, + xDelete: (zClass, zKey)=>{ + const stack = pstack.pointer; + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return 1/*OOM*/; + kvvfsStorage(zClass).removeItem(wasm.cstringToJs(zXKey)); + return 0; + }catch(e){ + console.error("kvstorageDelete()",e); + return capi.SQLITE_IOERR; + }finally{ + pstack.restore(stack); + } + } + }/*kvvfsImpls*/; + for(const k of Object.keys(kvvfsImpls)){ + kvvfsMethods[kvvfsMethods.memberKey(k)] = + wasm.installFunction( + kvvfsMethods.memberSignature(k), + kvvfsImpls[k] + ); + } + }else{ + /* Worker thread: unregister kvvfs to avoid it being used + for anything other than local/sessionStorage. It "can" + be used that way but it's not really intended to be. */ + capi.sqlite3_vfs_unregister(pKvvfs); + } + }/*pKvvfs*/ + +}); diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 9e54733966..02ce9c0ced 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -14,11 +14,11 @@ WASM build. It requires that sqlite3-api-glue.js has already run and it installs its deliverable as self.sqlite3.oo1. */ -(function(self){ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = (...args)=>{throw new Error(args.join(' '))}; + const toss3 = (...args)=>{throw new sqlite3.SQLite3Error(...args)}; - const sqlite3 = self.sqlite3 || toss("Missing main sqlite3 object."); - const capi = sqlite3.capi, util = capi.util; + const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; /* What follows is colloquially known as "OO API #1". It is a binding of the sqlite3 API which is designed to be run within the same thread (main or worker) as the one in which the @@ -33,16 +33,10 @@ accessor and store their real values in this map. Keys = DB/Stmt objects, values = pointer values. This also unifies how those are accessed, for potential use downstream via custom - capi.wasm.xWrap() function signatures which know how to extract + wasm.xWrap() function signatures which know how to extract it. */ const __ptrMap = new WeakMap(); - /** - Map of DB instances to objects, each object being a map of UDF - names to wasm function _pointers_ added to that DB handle via - createFunction(). - */ - const __udfMap = new WeakMap(); /** Map of DB instances to objects, each object being a map of Stmt wasm pointers to Stmt objects. @@ -51,22 +45,174 @@ /** If object opts has _its own_ property named p then that property's value is returned, else dflt is returned. */ - const getOwnOption = (opts, p, dflt)=> - opts.hasOwnProperty(p) ? opts[p] : dflt; + const getOwnOption = (opts, p, dflt)=>{ + const d = Object.getOwnPropertyDescriptor(opts,p); + return d ? d.value : dflt; + }; - /** - An Error subclass specifically for reporting DB-level errors and - enabling clients to unambiguously identify such exceptions. - */ - class SQLite3Error extends Error { - constructor(...args){ - super(...args); - this.name = 'SQLite3Error'; + // Documented in DB.checkRc() + const checkSqlite3Rc = function(dbPtr, sqliteResultCode){ + if(sqliteResultCode){ + if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; + toss3( + "sqlite result code",sqliteResultCode+":", + (dbPtr + ? capi.sqlite3_errmsg(dbPtr) + : capi.sqlite3_errstr(sqliteResultCode)) + ); } }; - const toss3 = (...args)=>{throw new SQLite3Error(args)}; - sqlite3.SQLite3Error = SQLite3Error; + /** + sqlite3_trace_v2() callback which gets installed by the DB ctor + if its open-flags contain "t". + */ + const __dbTraceToConsole = + wasm.installFunction('i(ippp)', function(t,c,p,x){ + if(capi.SQLITE_TRACE_STMT===t){ + // x == SQL, p == sqlite3_stmt* + console.log("SQL TRACE #"+(++this.counter), + wasm.cstringToJs(x)); + } + }.bind({counter: 0})); + + /** + A map of sqlite3_vfs pointers to SQL code to run when the DB + constructor opens a database with the given VFS. + */ + const __vfsPostOpenSql = Object.create(null); + + /** + A proxy for DB class constructors. It must be called with the + being-construct DB object as its "this". See the DB constructor + for the argument docs. This is split into a separate function + in order to enable simple creation of special-case DB constructors, + e.g. JsStorageDb and OpfsDb. + + Expects to be passed a configuration object with the following + properties: + + - `.filename`: the db filename. It may be a special name like ":memory:" + or "". + + - `.flags`: as documented in the DB constructor. + + - `.vfs`: as documented in the DB constructor. + + It also accepts those as the first 3 arguments. + */ + const dbCtorHelper = function ctor(...args){ + if(!ctor._name2vfs){ + /** + Map special filenames which we handle here (instead of in C) + to some helpful metadata... + + As of 2022-09-20, the C API supports the names :localStorage: + and :sessionStorage: for kvvfs. However, C code cannot + determine (without embedded JS code, e.g. via Emscripten's + EM_JS()) whether the kvvfs is legal in the current browser + context (namely the main UI thread). In order to help client + code fail early on, instead of it being delayed until they + try to read or write a kvvfs-backed db, we'll check for those + names here and throw if they're not legal in the current + context. + */ + ctor._name2vfs = Object.create(null); + const isWorkerThread = ('function'===typeof importScripts/*===running in worker thread*/) + ? (n)=>toss3("The VFS for",n,"is only available in the main window thread.") + : false; + ctor._name2vfs[':localStorage:'] = { + vfs: 'kvvfs', filename: isWorkerThread || (()=>'local') + }; + ctor._name2vfs[':sessionStorage:'] = { + vfs: 'kvvfs', filename: isWorkerThread || (()=>'session') + }; + } + const opt = ctor.normalizeArgs(...args); + let fn = opt.filename, vfsName = opt.vfs, flagsStr = opt.flags; + if(('string'!==typeof fn && 'number'!==typeof fn) + || 'string'!==typeof flagsStr + || (vfsName && ('string'!==typeof vfsName && 'number'!==typeof vfsName))){ + console.error("Invalid DB ctor args",opt,arguments); + toss3("Invalid arguments for DB constructor."); + } + let fnJs = ('number'===typeof fn) ? wasm.cstringToJs(fn) : fn; + const vfsCheck = ctor._name2vfs[fnJs]; + if(vfsCheck){ + vfsName = vfsCheck.vfs; + fn = fnJs = vfsCheck.filename(fnJs); + } + let pDb, oflags = 0; + if( flagsStr.indexOf('c')>=0 ){ + oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + } + if( flagsStr.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; + if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; + oflags |= capi.SQLITE_OPEN_EXRESCODE; + const stack = wasm.pstack.pointer; + try { + const pPtr = wasm.pstack.allocPtr() /* output (sqlite3**) arg */; + let rc = capi.sqlite3_open_v2(fn, pPtr, oflags, vfsName || 0); + pDb = wasm.getPtrValue(pPtr); + checkSqlite3Rc(pDb, rc); + if(flagsStr.indexOf('t')>=0){ + capi.sqlite3_trace_v2(pDb, capi.SQLITE_TRACE_STMT, + __dbTraceToConsole, 0); + } + // Check for per-VFS post-open SQL... + const pVfs = capi.sqlite3_js_db_vfs(pDb); + //console.warn("Opened db",fn,"with vfs",vfsName,pVfs); + if(!pVfs) toss3("Internal error: cannot get VFS for new db handle."); + const postInitSql = __vfsPostOpenSql[pVfs]; + if(postInitSql){ + rc = capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0); + checkSqlite3Rc(pDb, rc); + } + }catch( e ){ + if( pDb ) capi.sqlite3_close_v2(pDb); + throw e; + }finally{ + wasm.pstack.restore(stack); + } + this.filename = fnJs; + __ptrMap.set(this, pDb); + __stmtMap.set(this, Object.create(null)); + }; + + /** + Sets SQL which should be exec()'d on a DB instance after it is + opened with the given VFS pointer. This is intended only for use + by DB subclasses or sqlite3_vfs implementations. + */ + dbCtorHelper.setVfsPostOpenSql = function(pVfs, sql){ + __vfsPostOpenSql[pVfs] = sql; + }; + + /** + A helper for DB constructors. It accepts either a single + config-style object or up to 3 arguments (filename, dbOpenFlags, + dbVfsName). It returns a new object containing: + + { filename: ..., flags: ..., vfs: ... } + + If passed an object, any additional properties it has are copied + as-is into the new object. + */ + dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){ + const arg = {}; + if(1===arguments.length && 'object'===typeof arguments[0]){ + const x = arguments[0]; + Object.keys(x).forEach((k)=>arg[k] = x[k]); + if(undefined===arg.flags) arg.flags = 'c'; + if(undefined===arg.vfs) arg.vfs = null; + if(undefined===arg.filename) arg.filename = ':memory:'; + }else{ + arg.filename = filename; + arg.flags = flags; + arg.vfs = vfs; + } + return arg; + }; /** The DB class provides a high-level OO wrapper around an sqlite3 db handle. @@ -80,40 +226,62 @@ not resolve to real filenames, but "" uses an on-storage temporary database and requires that the VFS support that. - The db is currently opened with a fixed set of flags: - (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXRESCODE). This API will change in the future - permit the caller to provide those flags via an additional - argument. + The second argument specifies the open/create mode for the + database. It must be string containing a sequence of letters (in + any order, but case sensitive) specifying the mode: + + - "c": create if it does not exist, else fail if it does not + exist. Implies the "w" flag. + + - "w": write. Implies "r": a db cannot be write-only. + + - "r": read-only if neither "w" nor "c" are provided, else it + is ignored. + + - "t": enable tracing of SQL executed on this database handle, + sending it to `console.log()`. To disable it later, call + `sqlite3.capi.sqlite3_trace_v2(thisDb.pointer, 0, 0, 0)`. + + If "w" is not provided, the db is implicitly read-only, noting + that "rc" is meaningless + + Any other letters are currently ignored. The default is + "c". These modes are ignored for the special ":memory:" and "" + names and _may_ be ignored altogether for certain VFSes. + + The final argument is analogous to the final argument of + sqlite3_open_v2(): the name of an sqlite3 VFS. Pass a falsy value, + or none at all, to use the default. If passed a value, it must + be the string name of a VFS. + + The constructor optionally (and preferably) takes its arguments + in the form of a single configuration object with the following + properties: + + - `filename`: database file name + - `flags`: open-mode flags + - `vfs`: the VFS fname + + The `filename` and `vfs` arguments may be either JS strings or + C-strings allocated via WASM. `flags` is required to be a JS + string (because it's specific to this API, which is specific + to JS). For purposes of passing a DB instance to C-style sqlite3 - functions, its read-only `pointer` property holds its `sqlite3*` - pointer value. That property can also be used to check whether - this DB instance is still open. + functions, the DB object's read-only `pointer` property holds its + `sqlite3*` pointer value. That property can also be used to check + whether this DB instance is still open. + + In the main window thread, the filenames `":localStorage:"` and + `":sessionStorage:"` are special: they cause the db to use either + localStorage or sessionStorage for storing the database using + the kvvfs. If one of these names are used, they trump + any vfs name set in the arguments. */ - const DB = function ctor(fn=':memory:'){ - if('string'!==typeof fn){ - toss3("Invalid filename for DB constructor."); - } - const stack = capi.wasm.scopedAllocPush(); - let ptr; - try { - const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; - const rc = capi.sqlite3_open_v2(fn, ppDb, capi.SQLITE_OPEN_READWRITE - | capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_EXRESCODE, null); - ptr = capi.wasm.getMemValue(ppDb, '*'); - ctor.checkRc(ptr, rc); - }catch(e){ - if(ptr) capi.sqlite3_close_v2(ptr); - throw e; - } - finally{capi.wasm.scopedAllocPop(stack);} - this.filename = fn; - __ptrMap.set(this, ptr); - __stmtMap.set(this, Object.create(null)); - __udfMap.set(this, Object.create(null)); + const DB = function(...args){ + dbCtorHelper.apply(this, args); }; + DB.dbCtorHelper = dbCtorHelper; /** Internal-use enum for mapping JS types to DB-bindable types. @@ -129,7 +297,7 @@ blob: 5 }; BindTypes['undefined'] == BindTypes.null; - if(capi.wasm.bigIntEnabled){ + if(wasm.bigIntEnabled){ BindTypes.bigint = BindTypes.number; } @@ -141,6 +309,15 @@ For purposes of passing a Stmt instance to C-style sqlite3 functions, its read-only `pointer` property holds its `sqlite3_stmt*` pointer value. + + Other non-function properties include: + + - `db`: the DB object which created the statement. + + - `columnCount`: the number of result columns in the query, or 0 for + queries which cannot return results. + + - `parameterCount`: the number of bindable paramters in the query. */ const Stmt = function(){ if(BindTypes!==arguments[2]){ @@ -163,7 +340,7 @@ Reminder: this will also fail after the statement is finalized but the resulting error will be about an out-of-bounds column - index. + index rather than a statement-is-finalized error. */ const affirmColIndex = function(stmt,ndx){ if((ndx !== (ndx|0)) || ndx<0 || ndx>=stmt.columnCount){ @@ -173,24 +350,30 @@ }; /** - Expects to be passed (arguments) from DB.exec() and - DB.execMulti(). Does the argument processing/validation, throws - on error, and returns a new object on success: + Expects to be passed the `arguments` object from DB.exec(). Does + the argument processing/validation, throws on error, and returns + a new object on success: { sql: the SQL, opt: optionsObj, cbArg: function} - cbArg is only set if the opt.callback is set, in which case - it's a function which expects to be passed the current Stmt - and returns the callback argument of the type indicated by - the input arguments. + The opt object is a normalized copy of any passed to this + function. The sql will be converted to a string if it is provided + in one of the supported non-string formats. + + cbArg is only set if the opt.callback or opt.resultRows are set, + in which case it's a function which expects to be passed the + current Stmt and returns the callback argument of the type + indicated by the input arguments. */ - const parseExecArgs = function(args){ + const parseExecArgs = function(db, args){ const out = Object.create(null); out.opt = Object.create(null); switch(args.length){ case 1: if('string'===typeof args[0] || util.isSQLableTypedArray(args[0])){ out.sql = args[0]; + }else if(Array.isArray(args[0])){ + out.sql = args[0]; }else if(args[0] && 'object'===typeof args[0]){ out.opt = args[0]; out.sql = out.opt.sql; @@ -202,80 +385,142 @@ break; default: toss3("Invalid argument count for exec()."); }; - if(util.isSQLableTypedArray(out.sql)){ - out.sql = util.typedArrayToString(out.sql); - }else if(Array.isArray(out.sql)){ - out.sql = out.sql.join(''); - }else if('string'!==typeof out.sql){ - toss3("Missing SQL argument."); + out.sql = util.flexibleString(out.sql); + if('string'!==typeof out.sql){ + toss3("Missing SQL argument or unsupported SQL value type."); } - if(out.opt.callback || out.opt.resultRows){ - switch((undefined===out.opt.rowMode) - ? 'stmt' : out.opt.rowMode) { - case 'object': out.cbArg = (stmt)=>stmt.get({}); break; + const opt = out.opt; + switch(opt.returnValue){ + case 'resultRows': + if(!opt.resultRows) opt.resultRows = []; + out.returnVal = ()=>opt.resultRows; + break; + case 'saveSql': + if(!opt.saveSql) opt.saveSql = []; + out.returnVal = ()=>opt.saveSql; + break; + case undefined: + case 'this': + out.returnVal = ()=>db; + break; + default: + toss3("Invalid returnValue value:",opt.returnValue); + } + if(opt.callback || opt.resultRows){ + switch((undefined===opt.rowMode) + ? 'array' : opt.rowMode) { + case 'object': out.cbArg = (stmt)=>stmt.get(Object.create(null)); break; case 'array': out.cbArg = (stmt)=>stmt.get([]); break; case 'stmt': - if(Array.isArray(out.opt.resultRows)){ - toss3("Invalid rowMode for resultRows array: must", + if(Array.isArray(opt.resultRows)){ + toss3("exec(): invalid rowMode for a resultRows array: must", "be one of 'array', 'object',", - "or a result column number."); + "a result column number, or column name reference."); } out.cbArg = (stmt)=>stmt; break; default: - if(util.isInt32(out.opt.rowMode)){ - out.cbArg = (stmt)=>stmt.get(out.opt.rowMode); + if(util.isInt32(opt.rowMode)){ + out.cbArg = (stmt)=>stmt.get(opt.rowMode); break; + }else if('string'===typeof opt.rowMode && opt.rowMode.length>1){ + /* "$X", ":X", and "@X" fetch column named "X" (case-sensitive!) */ + const prefix = opt.rowMode[0]; + if(':'===prefix || '@'===prefix || '$'===prefix){ + out.cbArg = function(stmt){ + const rc = stmt.get(this.obj)[this.colName]; + return (undefined===rc) ? toss3("exec(): unknown result column:",this.colName) : rc; + }.bind({ + obj:Object.create(null), + colName: opt.rowMode.substr(1) + }); + break; + } } - toss3("Invalid rowMode:",out.opt.rowMode); + toss3("Invalid rowMode:",opt.rowMode); } } return out; }; /** - Expects to be given a DB instance or an `sqlite3*` pointer, and an - sqlite3 API result code. If the result code is not falsy, this - function throws an SQLite3Error with an error message from - sqlite3_errmsg(), using dbPtr as the db handle. Note that if it's - passed a non-error code like SQLITE_ROW or SQLITE_DONE, it will - still throw but the error string might be "Not an error." The - various non-0 non-error codes need to be checked for in client - code where they are expected. + Internal impl of the DB.selectArray() and + selectObject() methods. */ - DB.checkRc = function(dbPtr, sqliteResultCode){ - if(sqliteResultCode){ - if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; - throw new SQLite3Error([ - "sqlite result code",sqliteResultCode+":", - capi.sqlite3_errmsg(dbPtr) || "Unknown db error." - ].join(' ')); + const __selectFirstRow = (db, sql, bind, getArg)=>{ + let stmt, rc; + try { + stmt = db.prepare(sql).bind(bind); + if(stmt.step()) rc = stmt.get(getArg); + }finally{ + if(stmt) stmt.finalize(); } + return rc; }; + /** + Expects to be given a DB instance or an `sqlite3*` pointer (may + be null) and an sqlite3 API result code. If the result code is + not falsy, this function throws an SQLite3Error with an error + message from sqlite3_errmsg(), using dbPtr as the db handle, or + sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a + non-error code like SQLITE_ROW or SQLITE_DONE, it will still + throw but the error string might be "Not an error." The various + non-0 non-error codes need to be checked for in + client code where they are expected. + */ + DB.checkRc = checkSqlite3Rc; + DB.prototype = { + /** Returns true if this db handle is open, else false. */ + isOpen: function(){ + return !!this.pointer; + }, + /** Throws if this given DB has been closed, else returns `this`. */ + affirmOpen: function(){ + return affirmDbOpen(this); + }, /** Finalizes all open statements and closes this database connection. This is a no-op if the db has already been closed. After calling close(), `this.pointer` will resolve to `undefined`, so that can be used to check whether the db instance is still opened. + + If this.onclose.before is a function then it is called before + any close-related cleanup. + + If this.onclose.after is a function then it is called after the + db is closed but before auxiliary state like this.filename is + cleared. + + Both onclose handlers are passed this object. If this db is not + opened, neither of the handlers are called. Any exceptions the + handlers throw are ignored because "destructors must not + throw." + + Note that garbage collection of a db handle, if it happens at + all, will never trigger close(), so onclose handlers are not a + reliable way to implement close-time cleanup or maintenance of + a db. */ close: function(){ if(this.pointer){ + if(this.onclose && (this.onclose.before instanceof Function)){ + try{this.onclose.before(this)} + catch(e){/*ignore*/} + } const pDb = this.pointer; - let s; - const that = this; Object.keys(__stmtMap.get(this)).forEach((k,s)=>{ if(s && s.pointer) s.finalize(); }); - Object.values(__udfMap.get(this)).forEach( - capi.wasm.uninstallFunction.bind(capi.wasm) - ); __ptrMap.delete(this); __stmtMap.delete(this); - __udfMap.delete(this); capi.sqlite3_close_v2(pDb); + if(this.onclose && (this.onclose.after instanceof Function)){ + try{this.onclose.after(this)} + catch(e){/*ignore*/} + } delete this.filename; } }, @@ -300,26 +545,13 @@ } }, /** - Similar to this.filename but will return NULL for - special names like ":memory:". Not of much use until - we have filesystem support. Throws if the DB has - been closed. If passed an argument it then it will return - the filename of the ATTACHEd db with that name, else it assumes - a name of `main`. + Similar to the this.filename but returns the + sqlite3_db_filename() value for the given database name, + defaulting to "main". The argument may be either a JS string + or a pointer to a WASM-allocated C-string. */ - fileName: function(dbName){ - return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName||"main"); - }, - /** - Returns true if this db instance has a name which resolves to a - file. If the name is "" or ":memory:", it resolves to false. - Note that it is not aware of the peculiarities of URI-style - names and a URI-style name for a ":memory:" db will fool it. - */ - hasFilename: function(){ - const fn = this.filename; - if(!fn || ':memory'===fn) return false; - return true; + dbFilename: function(dbName='main'){ + return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName); }, /** Returns the name of the given 0-based db number, as documented @@ -328,13 +560,36 @@ dbName: function(dbNumber=0){ return capi.sqlite3_db_name(affirmDbOpen(this).pointer, dbNumber); }, + /** + Returns the name of the sqlite3_vfs used by the given database + of this connection (defaulting to 'main'). The argument may be + either a JS string or a WASM C-string. Returns undefined if the + given db name is invalid. Throws if this object has been + close()d. + */ + dbVfsName: function(dbName=0){ + let rc; + const pVfs = capi.sqlite3_js_db_vfs( + affirmDbOpen(this).pointer, dbName + ); + if(pVfs){ + const v = new capi.sqlite3_vfs(pVfs); + try{ rc = wasm.cstringToJs(v.$zName) } + finally { v.dispose() } + } + return rc; + }, /** Compiles the given SQL and returns a prepared Stmt. This is the only way to create new Stmt objects. Throws on error. - The given SQL must be a string, a Uint8Array holding SQL, or a - WASM pointer to memory holding the NUL-terminated SQL string. - If the SQL contains no statements, an SQLite3Error is thrown. + The given SQL must be a string, a Uint8Array holding SQL, a + WASM pointer to memory holding the NUL-terminated SQL string, + or an array of strings. In the latter case, the array is + concatenated together, with no separators, to form the SQL + string (arrays are often a convenient way to formulate long + statements). If the SQL contains no statements, an + SQLite3Error is thrown. Design note: the C API permits empty SQL, reporting it as a 0 result code and a NULL stmt pointer. Supporting that case here @@ -343,95 +598,33 @@ required to check `stmt.pointer` after calling `prepare()` in order to determine whether the Stmt instance is empty or not. Long-time practice (with other sqlite3 script bindings) - suggests that the empty-prepare case is sufficiently rare (and - useless) that supporting it here would simply hurt overall - usability. + suggests that the empty-prepare case is sufficiently rare that + supporting it here would simply hurt overall usability. */ prepare: function(sql){ affirmDbOpen(this); - const stack = capi.wasm.scopedAllocPush(); + const stack = wasm.pstack.pointer; let ppStmt, pStmt; try{ - ppStmt = capi.wasm.scopedAllocPtr()/* output (sqlite3_stmt**) arg */; + ppStmt = wasm.pstack.alloc(8)/* output (sqlite3_stmt**) arg */; DB.checkRc(this, capi.sqlite3_prepare_v2(this.pointer, sql, -1, ppStmt, null)); - pStmt = capi.wasm.getMemValue(ppStmt, '*'); + pStmt = wasm.getPtrValue(ppStmt); + } + finally { + wasm.pstack.restore(stack); } - finally {capi.wasm.scopedAllocPop(stack)} if(!pStmt) toss3("Cannot prepare empty SQL."); const stmt = new Stmt(this, pStmt, BindTypes); __stmtMap.get(this)[pStmt] = stmt; return stmt; }, - /** - This function works like execMulti(), and takes most of the - same arguments, but is more efficient (performs much less - work) when the input SQL is only a single statement. If - passed a multi-statement SQL, it only processes the first - one. - - This function supports the following additional options not - supported by execMulti(): - - - .multi: if true, this function acts as a proxy for - execMulti() and behaves identically to that function. - - - .columnNames: if this is an array and the query has - result columns, the array is passed to - Stmt.getColumnNames() to append the column names to it - (regardless of whether the query produces any result - rows). If the query has no result columns, this value is - unchanged. - - The following options to execMulti() are _not_ supported by - this method (they are simply ignored): - - - .saveSql - */ - exec: function(/*(sql [,optionsObj]) or (optionsObj)*/){ - affirmDbOpen(this); - const arg = parseExecArgs(arguments); - if(!arg.sql) return this; - else if(arg.opt.multi){ - return this.execMulti(arg, undefined, BindTypes); - } - const opt = arg.opt; - let stmt, rowTarget; - try { - if(Array.isArray(opt.resultRows)){ - rowTarget = opt.resultRows; - } - stmt = this.prepare(arg.sql); - if(stmt.columnCount && Array.isArray(opt.columnNames)){ - stmt.getColumnNames(opt.columnNames); - } - if(opt.bind) stmt.bind(opt.bind); - if(opt.callback || rowTarget){ - while(stmt.step()){ - const row = arg.cbArg(stmt); - if(rowTarget) rowTarget.push(row); - if(opt.callback){ - stmt._isLocked = true; - opt.callback(row, stmt); - stmt._isLocked = false; - } - } - }else{ - stmt.step(); - } - }finally{ - if(stmt){ - delete stmt._isLocked; - stmt.finalize(); - } - } - return this; - }/*exec()*/, /** Executes one or more SQL statements in the form of a single string. Its arguments must be either (sql,optionsObject) or - (optionsObject). In the latter case, optionsObject.sql - must contain the SQL to execute. Returns this - object. Throws on error. + (optionsObject). In the latter case, optionsObject.sql must + contain the SQL to execute. By default it returns this object + but that can be changed via the `returnValue` option as + described below. Throws on error. If no SQL is provided, or a non-string is provided, an exception is triggered. Empty SQL, on the other hand, is @@ -440,92 +633,144 @@ The optional options object may contain any of the following properties: - - .sql = the SQL to run (unless it's provided as the first - argument). This must be of type string, Uint8Array, or an - array of strings (in which case they're concatenated - together as-is, with no separator between elements, - before evaluation). + - `sql` = the SQL to run (unless it's provided as the first + argument). This must be of type string, Uint8Array, or an array + of strings. In the latter case they're concatenated together + as-is, _with no separator_ between elements, before evaluation. + The array form is often simpler for long hand-written queries. - - .bind = a single value valid as an argument for - Stmt.bind(). This is ONLY applied to the FIRST non-empty - statement in the SQL which has any bindable - parameters. (Empty statements are skipped entirely.) + - `bind` = a single value valid as an argument for + Stmt.bind(). This is _only_ applied to the _first_ non-empty + statement in the SQL which has any bindable parameters. (Empty + statements are skipped entirely.) - - .callback = a function which gets called for each row of - the FIRST statement in the SQL which has result - _columns_, but only if that statement has any result - _rows_. The second argument passed to the callback is - always the current Stmt object (so that the caller may - collect column names, or similar). The first argument - passed to the callback defaults to the current Stmt - object but may be changed with ... - - - .rowMode = either a string describing what type of argument - should be passed as the first argument to the callback or an - integer representing a result column index. A `rowMode` of - 'object' causes the results of `stmt.get({})` to be passed to - the `callback` and/or appended to `resultRows`. A value of - 'array' causes the results of `stmt.get([])` to be passed to - passed on. A value of 'stmt' is equivalent to the default, - passing the current Stmt to the callback (noting that it's - always passed as the 2nd argument), but this mode will trigger - an exception if `resultRows` is an array. If `rowMode` is an - integer, only the single value from that result column will be - passed on. Any other value for the option triggers an - exception. - - - .resultRows: if this is an array, it functions similarly to - the `callback` option: each row of the result set (if any) of - the FIRST first statement which has result _columns_ is - appended to the array in the format specified for the `rowMode` - option, with the exception that the only legal values for - `rowMode` in this case are 'array' or 'object', neither of - which is the default. It is legal to use both `resultRows` and - `callback`, but `resultRows` is likely much simpler to use for - small data sets and can be used over a WebWorker-style message - interface. execMulti() throws if `resultRows` is set and - `rowMode` is 'stmt' (which is the default!). - - - saveSql = an optional array. If set, the SQL of each + - `saveSql` = an optional array. If set, the SQL of each executed statement is appended to this array before the - statement is executed (but after it is prepared - we - don't have the string until after that). Empty SQL - statements are elided. + statement is executed (but after it is prepared - we don't have + the string until after that). Empty SQL statements are elided + but can have odd effects in the output. e.g. SQL of: `"select + 1; -- empty\n; select 2"` will result in an array containing + `["select 1;", "--empty \n; select 2"]`. That's simply how + sqlite3 records the SQL for the 2nd statement. - See also the exec() method, which is a close cousin of this - one. + ================================================================== + The following options apply _only_ to the _first_ statement + which has a non-zero result column count, regardless of whether + the statement actually produces any result rows. + ================================================================== - ACHTUNG #1: The callback MUST NOT modify the Stmt - object. Calling any of the Stmt.get() variants, - Stmt.getColumnName(), or similar, is legal, but calling - step() or finalize() is not. Routines which are illegal - in this context will trigger an exception. + - `columnNames`: if this is an array, the column names of the + result set are stored in this array before the callback (if + any) is triggered (regardless of whether the query produces any + result rows). If no statement has result columns, this value is + unchanged. Achtung: an SQL result may have multiple columns + with identical names. + + - `callback` = a function which gets called for each row of + the result set, but only if that statement has any result + _rows_. The callback's "this" is the options object, noting + that this function synthesizes one if the caller does not pass + one to exec(). The second argument passed to the callback is + always the current Stmt object, as it's needed if the caller + wants to fetch the column names or some such (noting that they + could also be fetched via `this.columnNames`, if the client + provides the `columnNames` option). + + ACHTUNG: The callback MUST NOT modify the Stmt object. Calling + any of the Stmt.get() variants, Stmt.getColumnName(), or + similar, is legal, but calling step() or finalize() is + not. Member methods which are illegal in this context will + trigger an exception. + + The first argument passed to the callback defaults to an array of + values from the current result row but may be changed with ... + + - `rowMode` = specifies the type of he callback's first argument. + It may be any of... + + A) A string describing what type of argument should be passed + as the first argument to the callback: + + A.1) `'array'` (the default) causes the results of + `stmt.get([])` to be passed to the `callback` and/or appended + to `resultRows` + + A.2) `'object'` causes the results of + `stmt.get(Object.create(null))` to be passed to the + `callback` and/or appended to `resultRows`. Achtung: an SQL + result may have multiple columns with identical names. In + that case, the right-most column will be the one set in this + object! + + A.3) `'stmt'` causes the current Stmt to be passed to the + callback, but this mode will trigger an exception if + `resultRows` is an array because appending the statement to + the array would be downright unhelpful. + + B) An integer, indicating a zero-based column in the result + row. Only that one single value will be passed on. + + C) A string with a minimum length of 2 and leading character of + ':', '$', or '@' will fetch the row as an object, extract that + one field, and pass that field's value to the callback. Note + that these keys are case-sensitive so must match the case used + in the SQL. e.g. `"select a A from t"` with a `rowMode` of + `'$A'` would work but `'$a'` would not. A reference to a column + not in the result set will trigger an exception on the first + row (as the check is not performed until rows are fetched). + Note also that `$` is a legal identifier character in JS so + need not be quoted. (Design note: those 3 characters were + chosen because they are the characters support for naming bound + parameters.) + + Any other `rowMode` value triggers an exception. + + - `resultRows`: if this is an array, it functions similarly to + the `callback` option: each row of the result set (if any), + with the exception that the `rowMode` 'stmt' is not legal. It + is legal to use both `resultRows` and `callback`, but + `resultRows` is likely much simpler to use for small data sets + and can be used over a WebWorker-style message interface. + exec() throws if `resultRows` is set and `rowMode` is 'stmt'. + + - `returnValue`: is a string specifying what this function + should return: + + A) The default value is `"this"`, meaning that the + DB object itself should be returned. + + B) `"resultRows"` means to return the value of the + `resultRows` option. If `resultRows` is not set, this + function behaves as if it were set to an empty array. + + C) `"saveSql"` means to return the value of the + `saveSql` option. If `saveSql` is not set, this + function behaves as if it were set to an empty array. + + Potential TODOs: + + - `bind`: permit an array of arrays/objects to bind. The first + sub-array would act on the first statement which has bindable + parameters (as it does now). The 2nd would act on the next such + statement, etc. + + - `callback` and `resultRows`: permit an array entries with + semantics similar to those described for `bind` above. - ACHTUNG #2: The semantics of the `bind` and `callback` - options may well change or those options may be removed - altogether for this function (but retained for exec()). - Generally speaking, neither bind parameters nor a callback - are generically useful when executing multi-statement SQL. */ - execMulti: function(/*(sql [,obj]) || (obj)*/){ + exec: function(/*(sql [,obj]) || (obj)*/){ affirmDbOpen(this); - const wasm = capi.wasm; - const arg = (BindTypes===arguments[2] - /* ^^^ Being passed on from exec() */ - ? arguments[0] : parseExecArgs(arguments)); - if(!arg.sql) return this; + const arg = parseExecArgs(this, arguments); + if(!arg.sql){ + return toss3("exec() requires an SQL string."); + } const opt = arg.opt; const callback = opt.callback; - const resultRows = (Array.isArray(opt.resultRows) - ? opt.resultRows : undefined); - if(resultRows && 'stmt'===opt.rowMode){ - toss3("rowMode 'stmt' is not valid in combination", - "with a resultRows array."); - } - let rowMode = (((callback||resultRows) && (undefined!==opt.rowMode)) - ? opt.rowMode : undefined); + const resultRows = + Array.isArray(opt.resultRows) ? opt.resultRows : undefined; let stmt; let bind = opt.bind; + let evalFirstResult = !!(arg.cbArg || opt.columnNames) /* true to evaluate the first result-returning query */; const stack = wasm.scopedAllocPush(); try{ const isTA = util.isSQLableTypedArray(arg.sql) @@ -544,21 +789,21 @@ if(isTA) wasm.heap8().set(arg.sql, pSql); else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); wasm.setMemValue(pSql + sqlByteLen, 0/*NUL terminator*/); - while(wasm.getMemValue(pSql, 'i8') - /* Maintenance reminder: ^^^^ _must_ be i8 or else we + while(pSql && wasm.getMemValue(pSql, 'i8') + /* Maintenance reminder:^^^ _must_ be 'i8' or else we will very likely cause an endless loop. What that's doing is checking for a terminating NUL byte. If we use i32 or similar then we read 4 bytes, read stuff around the NUL terminator, and get stuck in and endless loop at the end of the SQL, endlessly re-preparing an empty statement. */ ){ - wasm.setMemValue(ppStmt, 0, wasm.ptrIR); - wasm.setMemValue(pzTail, 0, wasm.ptrIR); - DB.checkRc(this, capi.sqlite3_prepare_v2( - this.pointer, pSql, sqlByteLen, ppStmt, pzTail + wasm.setPtrValue(ppStmt, 0); + wasm.setPtrValue(pzTail, 0); + DB.checkRc(this, capi.sqlite3_prepare_v3( + this.pointer, pSql, sqlByteLen, 0, ppStmt, pzTail )); - const pStmt = wasm.getMemValue(ppStmt, wasm.ptrIR); - pSql = wasm.getMemValue(pzTail, wasm.ptrIR); + const pStmt = wasm.getPtrValue(ppStmt); + pSql = wasm.getPtrValue(pzTail); sqlByteLen = pSqlEnd - pSql; if(!pStmt) continue; if(Array.isArray(opt.saveSql)){ @@ -569,36 +814,38 @@ stmt.bind(bind); bind = null; } - if(stmt.columnCount && undefined!==rowMode){ + if(evalFirstResult && stmt.columnCount){ /* Only forward SELECT results for the FIRST query in the SQL which potentially has them. */ - while(stmt.step()){ + evalFirstResult = false; + if(Array.isArray(opt.columnNames)){ + stmt.getColumnNames(opt.columnNames); + } + while(!!arg.cbArg && stmt.step()){ stmt._isLocked = true; const row = arg.cbArg(stmt); - if(callback) callback(row, stmt); if(resultRows) resultRows.push(row); + if(callback) callback.call(opt, row, stmt); stmt._isLocked = false; } - rowMode = undefined; }else{ - // Do we need to while(stmt.step()){} here? stmt.step(); } stmt.finalize(); stmt = null; } - }catch(e){ - console.warn("DB.execMulti() is propagating exception",opt,e); + }/*catch(e){ + console.warn("DB.exec() is propagating exception",opt,e); throw e; - }finally{ + }*/finally{ if(stmt){ delete stmt._isLocked; stmt.finalize(); } wasm.scopedAllocPop(stack); } - return this; - }/*execMulti()*/, + return arg.returnVal(); + }/*exec()*/, /** Creates a new scalar UDF (User-Defined Function) which is accessible via SQL code. This function may be called in any @@ -610,178 +857,169 @@ - (optionsObject) In the final two cases, the function must be defined as the - 'callback' property of the options object. In the final + `callback` property of the options object (optionally called + `xFunc` to align with the C API documentation). In the final case, the function's name must be the 'name' property. - This can only be used to create scalar functions, not - aggregate or window functions. UDFs cannot be removed from - a DB handle after they're added. + The first two call forms can only be used for creating scalar + functions. Creating an aggregate or window function requires + the options-object form (see below for details). + + UDFs cannot currently be removed from a DB handle after they're + added. More correctly, they can be removed as documented for + sqlite3_create_function_v2(), but doing so will "leak" the + JS-created WASM binding of those functions. On success, returns this object. Throws on error. - When called from SQL, arguments to the UDF, and its result, - will be converted between JS and SQL with as much fidelity - as is feasible, triggering an exception if a type - conversion cannot be determined. Some freedom is afforded - to numeric conversions due to friction between the JS and C - worlds: integers which are larger than 32 bits will be - treated as doubles, as JS does not support 64-bit integers - and it is (as of this writing) illegal to use WASM - functions which take or return 64-bit integers from JS. + When called from SQL arguments to the UDF, and its result, + will be converted between JS and SQL with as much fidelity as + is feasible, triggering an exception if a type conversion + cannot be determined. The docs for sqlite3_create_function_v2() + describe the conversions in more detail. - The optional options object may contain flags to modify how + The values set in the options object differ for scalar and + aggregate functions: + + - Scalar: set the `xFunc` function-type property to the UDF + function. + + - Aggregate: set the `xStep` and `xFinal` function-type + properties to the "step" and "final" callbacks for the + aggregate. Do not set the `xFunc` property. + + - Window: set the `xStep`, `xFinal`, `xValue`, and `xInverse` + function-type properties. Do not set the `xFunc` property. + + The options object may optionally have an `xDestroy` + function-type property, as per sqlite3_create_function_v2(). + Its argument will be the WASM-pointer-type value of the `pApp` + property, and this function will throw if `pApp` is defined but + is not null, undefined, or a numeric (WASM pointer) + value. i.e. `pApp`, if set, must be value suitable for use as a + WASM pointer argument, noting that `null` or `undefined` will + translate to 0 for that purpose. + + The options object may contain flags to modify how the function is defined: - - .arity: the number of arguments which SQL calls to this - function expect or require. The default value is the - callback's length property (i.e. the number of declared - parameters it has). A value of -1 means that the function - is variadic and may accept any number of arguments, up to - sqlite3's compile-time limits. sqlite3 will enforce the - argument count if is zero or greater. + - `arity`: the number of arguments which SQL calls to this + function expect or require. The default value is `xFunc.length` + or `xStep.length` (i.e. the number of declared parameters it + has) **MINUS 1** (see below for why). As a special case, if the + `length` is 0, its arity is also 0 instead of -1. A negative + arity value means that the function is variadic and may accept + any number of arguments, up to sqlite3's compile-time + limits. sqlite3 will enforce the argument count if is zero or + greater. The callback always receives a pointer to an + `sqlite3_context` object as its first argument. Any arguments + after that are from SQL code. The leading context argument does + _not_ count towards the function's arity. See the docs for + sqlite3.capi.sqlite3_create_function_v2() for why that argument + is needed in the interface. - The following properties correspond to flags documented at: + The following options-object properties correspond to flags + documented at: https://sqlite.org/c3ref/create_function.html - - .deterministic = SQLITE_DETERMINISTIC - - .directOnly = SQLITE_DIRECTONLY - - .innocuous = SQLITE_INNOCUOUS + - `deterministic` = sqlite3.capi.SQLITE_DETERMINISTIC + - `directOnly` = sqlite3.capi.SQLITE_DIRECTONLY + - `innocuous` = sqlite3.capi.SQLITE_INNOCUOUS - Maintenance reminder: the ability to add new - WASM-accessible functions to the runtime requires that the - WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` - flag. + Sidebar: the ability to add new WASM-accessible functions to + the runtime requires that the WASM build is compiled with the + equivalent functionality as that provided by Emscripten's + `-sALLOW_TABLE_GROWTH` flag. */ - createFunction: function f(name, callback,opt){ + createFunction: function f(name, xFunc, opt){ + const isFunc = (f)=>(f instanceof Function); switch(arguments.length){ case 1: /* (optionsObject) */ opt = name; name = opt.name; - callback = opt.callback; + xFunc = opt.xFunc || 0; break; case 2: /* (name, callback|optionsObject) */ - if(!(callback instanceof Function)){ - opt = callback; - callback = opt.callback; + if(!isFunc(xFunc)){ + opt = xFunc; + xFunc = opt.xFunc || 0; } break; + case 3: /* name, xFunc, opt */ + break; default: break; } if(!opt) opt = {}; - if(!(callback instanceof Function)){ - toss3("Invalid arguments: expecting a callback function."); - }else if('string' !== typeof name){ + if('string' !== typeof name){ toss3("Invalid arguments: missing function name."); } - if(!f._extractArgs){ - /* Static init */ - f._extractArgs = function(argc, pArgv){ - let i, pVal, valType, arg; - const tgt = []; - for(i = 0; i < argc; ++i){ - pVal = capi.wasm.getMemValue(pArgv + (capi.wasm.ptrSizeof * i), - capi.wasm.ptrIR); - /** - Curiously: despite ostensibly requiring 8-byte - alignment, the pArgv array is parcelled into chunks of - 4 bytes (1 pointer each). The values those point to - have 8-byte alignment but the individual argv entries - do not. - */ - valType = capi.sqlite3_value_type(pVal); - switch(valType){ - case capi.SQLITE_INTEGER: - case capi.SQLITE_FLOAT: - arg = capi.sqlite3_value_double(pVal); - break; - case capi.SQLITE_TEXT: - arg = capi.sqlite3_value_text(pVal); - break; - case capi.SQLITE_BLOB:{ - const n = capi.sqlite3_value_bytes(pVal); - const pBlob = capi.sqlite3_value_blob(pVal); - arg = new Uint8Array(n); - let i; - const heap = n ? capi.wasm.heap8() : false; - for(i = 0; i < n; ++i) arg[i] = heap[pBlob+i]; - break; - } - case capi.SQLITE_NULL: - arg = null; break; - default: - toss3("Unhandled sqlite3_value_type()",valType, - "is possibly indicative of incorrect", - "pointer size assumption."); - } - tgt.push(arg); - } - return tgt; - }/*_extractArgs()*/; - f._setResult = function(pCx, val){ - switch(typeof val) { - case 'boolean': - capi.sqlite3_result_int(pCx, val ? 1 : 0); - break; - case 'number': { - (util.isInt32(val) - ? capi.sqlite3_result_int - : capi.sqlite3_result_double)(pCx, val); - break; - } - case 'string': - capi.sqlite3_result_text(pCx, val, -1, capi.SQLITE_TRANSIENT); - break; - case 'object': - if(null===val) { - capi.sqlite3_result_null(pCx); - break; - }else if(util.isBindableTypedArray(val)){ - const pBlob = capi.wasm.mallocFromTypedArray(val); - capi.sqlite3_result_blob(pCx, pBlob, val.byteLength, - capi.SQLITE_TRANSIENT); - capi.wasm.dealloc(pBlob); - break; - } - // else fall through - default: - toss3("Don't not how to handle this UDF result value:",val); - }; - }/*_setResult()*/; - }/*static init*/ - const wrapper = function(pCx, argc, pArgv){ - try{ - f._setResult(pCx, callback.apply(null, f._extractArgs(argc, pArgv))); - }catch(e){ - if(e instanceof capi.WasmAllocError){ - capi.sqlite3_result_error_nomem(pCx); - }else{ - capi.sqlite3_result_error(pCx, e.message, -1); - } + let xStep = opt.xStep || 0; + let xFinal = opt.xFinal || 0; + const xValue = opt.xValue || 0; + const xInverse = opt.xInverse || 0; + let isWindow = undefined; + if(isFunc(xFunc)){ + isWindow = false; + if(isFunc(xStep) || isFunc(xFinal)){ + toss3("Ambiguous arguments: scalar or aggregate?"); } - }; - const pUdf = capi.wasm.installFunction(wrapper, "v(iii)"); + xStep = xFinal = null; + }else if(isFunc(xStep)){ + if(!isFunc(xFinal)){ + toss3("Missing xFinal() callback for aggregate or window UDF."); + } + xFunc = null; + }else if(isFunc(xFinal)){ + toss3("Missing xStep() callback for aggregate or window UDF."); + }else{ + toss3("Missing function-type properties."); + } + if(false === isWindow){ + if(isFunc(xValue) || isFunc(xInverse)){ + toss3("xValue and xInverse are not permitted for non-window UDFs."); + } + }else if(isFunc(xValue)){ + if(!isFunc(xInverse)){ + toss3("xInverse must be provided if xValue is."); + } + isWindow = true; + }else if(isFunc(xInverse)){ + toss3("xValue must be provided if xInverse is."); + } + const pApp = opt.pApp; + if(undefined!==pApp && + null!==pApp && + (('number'!==typeof pApp) || !util.isInt32(pApp))){ + toss3("Invalid value for pApp property. Must be a legal WASM pointer value."); + } + const xDestroy = opt.xDestroy || 0; + if(xDestroy && !isFunc(xDestroy)){ + toss3("xDestroy property must be a function."); + } let fFlags = 0 /*flags for sqlite3_create_function_v2()*/; if(getOwnOption(opt, 'deterministic')) fFlags |= capi.SQLITE_DETERMINISTIC; if(getOwnOption(opt, 'directOnly')) fFlags |= capi.SQLITE_DIRECTONLY; if(getOwnOption(opt, 'innocuous')) fFlags |= capi.SQLITE_INNOCUOUS; name = name.toLowerCase(); - try { - DB.checkRc(this, capi.sqlite3_create_function_v2( - this.pointer, name, - (opt.hasOwnProperty('arity') ? +opt.arity : callback.length), - capi.SQLITE_UTF8 | fFlags, null/*pApp*/, pUdf, - null/*xStep*/, null/*xFinal*/, null/*xDestroy*/)); - }catch(e){ - capi.wasm.uninstallFunction(pUdf); - throw e; + const xArity = xFunc || xStep; + const arity = getOwnOption(opt, 'arity'); + const arityArg = ('number'===typeof arity + ? arity + : (xArity.length ? xArity.length-1/*for pCtx arg*/ : 0)); + let rc; + if( isWindow ){ + rc = capi.sqlite3_create_window_function( + this.pointer, name, arityArg, + capi.SQLITE_UTF8 | fFlags, pApp || 0, + xStep, xFinal, xValue, xInverse, xDestroy); + }else{ + rc = capi.sqlite3_create_function_v2( + this.pointer, name, arityArg, + capi.SQLITE_UTF8 | fFlags, pApp || 0, + xFunc, xStep, xFinal, xDestroy); } - const udfMap = __udfMap.get(this); - if(udfMap[name]){ - try{capi.wasm.uninstallFunction(udfMap[name])} - catch(e){/*ignore*/} - } - udfMap[name] = pUdf; + DB.checkRc(this, rc); return this; }/*createFunction()*/, /** @@ -798,7 +1036,7 @@ SQLITE_{typename} constants. Passing the undefined value is the same as not passing a value. - Throws on error (e.g. malformedSQL). + Throws on error (e.g. malformed SQL). */ selectValue: function(sql,bind,asType){ let stmt, rc; @@ -810,6 +1048,38 @@ } return rc; }, + /** + Prepares the given SQL, step()s it one time, and returns an + array containing the values of the first result row. If it has + no results, `undefined` is returned. + + If passed a second argument other than `undefined`, it is + treated like an argument to Stmt.bind(), so may be any type + supported by that function. + + Throws on error (e.g. malformed SQL). + */ + selectArray: function(sql,bind){ + return __selectFirstRow(this, sql, bind, []); + }, + + /** + Prepares the given SQL, step()s it one time, and returns an + object containing the key/value pairs of the first result + row. If it has no results, `undefined` is returned. + + Note that the order of returned object's keys is not guaranteed + to be the same as the order of the fields in the query string. + + If passed a second argument other than `undefined`, it is + treated like an argument to Stmt.bind(), so may be any type + supported by that function. + + Throws on error (e.g. malformed SQL). + */ + selectObject: function(sql,bind){ + return __selectFirstRow(this, sql, bind, {}); + }, /** Returns the number of currently-opened Stmt handles for this db @@ -820,48 +1090,46 @@ }, /** - This function currently does nothing and always throws. It - WILL BE REMOVED pending other refactoring, to eliminate a hard - dependency on Emscripten. This feature will be moved into a - higher-level API or a runtime-configurable feature. + Starts a transaction, calls the given callback, and then either + rolls back or commits the savepoint, depending on whether the + callback throws. The callback is passed this db object as its + only argument. On success, returns the result of the + callback. Throws on error. - That said, what its replacement should eventually do is... + Note that transactions may not be nested, so this will throw if + it is called recursively. For nested transactions, use the + savepoint() method or manually manage SAVEPOINTs using exec(). + */ + transaction: function(callback){ + affirmDbOpen(this).exec("BEGIN"); + try { + const rc = callback(this); + this.exec("COMMIT"); + return rc; + }catch(e){ + this.exec("ROLLBACK"); + throw e; + } + }, - Exports a copy of this db's file as a Uint8Array and - returns it. It is technically not legal to call this while - any prepared statement are currently active because, - depending on the platform, it might not be legal to read - the db while a statement is locking it. Throws if this db - is not open or has any opened statements. - - The resulting buffer can be passed to this class's - constructor to restore the DB. - - Maintenance reminder: the corresponding sql.js impl of this - feature closes the current db, finalizing any active - statements and (seemingly unnecessarily) destroys any UDFs, - copies the file, and then re-opens it (without restoring - the UDFs). Those gymnastics are not necessary on the tested - platform but might be necessary on others. Because of that - eventuality, this interface currently enforces that no - statements are active when this is run. It will throw if - any are. + /** + This works similarly to transaction() but uses sqlite3's SAVEPOINT + feature. This function starts a savepoint (with an unspecified name) + and calls the given callback function, passing it this db object. + If the callback returns, the savepoint is released (committed). If + the callback throws, the savepoint is rolled back. If it does not + throw, it returns the result of the callback. */ - exportBinaryImage: function(){ - toss3("exportBinaryImage() is slated for removal for portability reasons."); - /*********************** - The following is currently kept only for reference when - porting to some other layer, noting that we may well not be - able to implement this, at this level, when using the OPFS - VFS because of its exclusive locking policy. - - affirmDbOpen(this); - if(this.openStatementCount()>0){ - toss3("Cannot export with prepared statements active!", - "finalize() all statements and try again."); - } - return MODCFG.FS.readFile(this.filename, {encoding:"binary"}); - ***********************/ + savepoint: function(callback){ + affirmDbOpen(this).exec("SAVEPOINT oo1"); + try { + const rc = callback(this); + this.exec("RELEASE oo1"); + return rc; + }catch(e){ + this.exec("ROLLBACK to SAVEPOINT oo1; RELEASE SAVEPOINT oo1"); + throw e; + } } }/*DB.prototype*/; @@ -886,7 +1154,7 @@ case BindTypes.string: return t; case BindTypes.bigint: - if(capi.wasm.bigIntEnabled) return t; + if(wasm.bigIntEnabled) return t; /* else fall through */ default: //console.log("isSupportedBindType",t,v); @@ -946,10 +1214,9 @@ const bindOne = function f(stmt,ndx,bindType,val){ affirmUnlocked(stmt, 'bind()'); if(!f._){ - if(capi.wasm.bigIntEnabled){ - f._maxInt = BigInt("0x7fffffffffffffff"); - f._minInt = ~f._maxInt; - } + f._tooBigInt = (v)=>toss3( + "BigInt value is too big to store without precision loss:", v + ); /* Reminder: when not in BigInt mode, it's impossible for JS to represent a number out of the range we can bind, so we have no range checking. */ @@ -957,30 +1224,30 @@ string: function(stmt, ndx, val, asBlob){ if(1){ /* _Hypothetically_ more efficient than the impl in the 'else' block. */ - const stack = capi.wasm.scopedAllocPush(); + const stack = wasm.scopedAllocPush(); try{ - const n = capi.wasm.jstrlen(val); - const pStr = capi.wasm.scopedAlloc(n); - capi.wasm.jstrcpy(val, capi.wasm.heap8u(), pStr, n, false); + const n = wasm.jstrlen(val); + const pStr = wasm.scopedAlloc(n); + wasm.jstrcpy(val, wasm.heap8u(), pStr, n, false); const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; return f(stmt.pointer, ndx, pStr, n, capi.SQLITE_TRANSIENT); }finally{ - capi.wasm.scopedAllocPop(stack); + wasm.scopedAllocPop(stack); } }else{ - const bytes = capi.wasm.jstrToUintArray(val,false); - const pStr = capi.wasm.alloc(bytes.length || 1); - capi.wasm.heap8u().set(bytes.length ? bytes : [0], pStr); + const bytes = wasm.jstrToUintArray(val,false); + const pStr = wasm.alloc(bytes.length || 1); + wasm.heap8u().set(bytes.length ? bytes : [0], pStr); try{ const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; return f(stmt.pointer, ndx, pStr, bytes.length, capi.SQLITE_TRANSIENT); }finally{ - capi.wasm.dealloc(pStr); + wasm.dealloc(pStr); } } } }; - } + }/* static init */ affirmSupportedBindType(val); ndx = affirmParamIndex(stmt,ndx); let rc = 0; @@ -994,15 +1261,24 @@ case BindTypes.number: { let m; if(util.isInt32(val)) m = capi.sqlite3_bind_int; - else if(capi.wasm.bigIntEnabled && ('bigint'===typeof val)){ - if(valf._maxInt){ - toss3("BigInt value is out of range for int64: "+val); + else if('bigint'===typeof val){ + if(!util.bigIntFits64(val)){ + f._tooBigInt(val); + }else if(wasm.bigIntEnabled){ + m = capi.sqlite3_bind_int64; + }else if(util.bigIntFitsDouble(val)){ + val = Number(val); + m = capi.sqlite3_bind_double; + }else{ + f._tooBigInt(val); + } + }else{ // !int32, !bigint + val = Number(val); + if(wasm.bigIntEnabled && Number.isInteger(val)){ + m = capi.sqlite3_bind_int64; + }else{ + m = capi.sqlite3_bind_double; } - m = capi.sqlite3_bind_int64; - }else if(Number.isInteger(val)){ - m = capi.sqlite3_bind_int64; - }else{ - m = capi.sqlite3_bind_double; } rc = m(stmt.pointer, ndx, val); break; @@ -1018,22 +1294,22 @@ "that it be a string, Uint8Array, or Int8Array."); }else if(1){ /* _Hypothetically_ more efficient than the impl in the 'else' block. */ - const stack = capi.wasm.scopedAllocPush(); + const stack = wasm.scopedAllocPush(); try{ - const pBlob = capi.wasm.scopedAlloc(val.byteLength || 1); - capi.wasm.heap8().set(val.byteLength ? val : [0], pBlob) + const pBlob = wasm.scopedAlloc(val.byteLength || 1); + wasm.heap8().set(val.byteLength ? val : [0], pBlob) rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, capi.SQLITE_TRANSIENT); }finally{ - capi.wasm.scopedAllocPop(stack); + wasm.scopedAllocPop(stack); } }else{ - const pBlob = capi.wasm.mallocFromTypedArray(val); + const pBlob = wasm.allocFromTypedArray(val); try{ rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, capi.SQLITE_TRANSIENT); }finally{ - capi.wasm.dealloc(pBlob); + wasm.dealloc(pBlob); } } break; @@ -1042,7 +1318,7 @@ console.warn("Unsupported bind() argument type:",val); toss3("Unsupported bind() argument type: "+(typeof val)); } - if(rc) checkDbRc(stmt.db.pointer, rc); + if(rc) DB.checkRc(stmt.db.pointer, rc); return stmt; }; @@ -1059,6 +1335,7 @@ delete __stmtMap.get(this.db)[this.pointer]; capi.sqlite3_finalize(this.pointer); __ptrMap.delete(this); + delete this._mayGet; delete this.columnCount; delete this.parameterCount; delete this.db; @@ -1105,31 +1382,32 @@ - null is bound as NULL. - undefined as a standalone value is a no-op intended to - simplify certain client-side use cases: passing undefined - as a value to this function will not actually bind - anything and this function will skip confirmation that - binding is even legal. (Those semantics simplify certain - client-side uses.) Conversely, a value of undefined as an - array or object property when binding an array/object - (see below) is treated the same as null. + simplify certain client-side use cases: passing undefined as + a value to this function will not actually bind anything and + this function will skip confirmation that binding is even + legal. (Those semantics simplify certain client-side uses.) + Conversely, a value of undefined as an array or object + property when binding an array/object (see below) is treated + the same as null. - - Numbers are bound as either doubles or integers: doubles - if they are larger than 32 bits, else double or int32, - depending on whether they have a fractional part. (It is, - as of this writing, illegal to call (from JS) a WASM - function which either takes or returns an int64.) - Booleans are bound as integer 0 or 1. It is not expected - the distinction of binding doubles which have no - fractional parts is integers is significant for the - majority of clients due to sqlite3's data typing - model. If capi.wasm.bigIntEnabled is true then this - routine will bind BigInt values as 64-bit integers. + - Numbers are bound as either doubles or integers: doubles if + they are larger than 32 bits, else double or int32, depending + on whether they have a fractional part. Booleans are bound as + integer 0 or 1. It is not expected the distinction of binding + doubles which have no fractional parts is integers is + significant for the majority of clients due to sqlite3's data + typing model. If [BigInt] support is enabled then this + routine will bind BigInt values as 64-bit integers if they'll + fit in 64 bits. If that support disabled, it will store the + BigInt as an int32 or a double if it can do so without loss + of precision. If the BigInt is _too BigInt_ then it will + throw. - Strings are bound as strings (use bindAsBlob() to force - blob binding). + blob binding). - Uint8Array and Int8Array instances are bound as blobs. - (TODO: binding the other TypedArray types.) + (TODO: binding the other TypedArray types.) If passed an array, each element of the array is bound at the parameter index equal to the array index plus 1 @@ -1228,9 +1506,10 @@ return this; }, /** - Steps the statement one time. If the result indicates that - a row of data is available, true is returned. If no row of - data is available, false is returned. Throws on error. + Steps the statement one time. If the result indicates that a + row of data is available, a truthy value is returned. + If no row of data is available, a falsy + value is returned. Throws on error. */ step: function(){ affirmUnlocked(this, 'step()'); @@ -1240,10 +1519,53 @@ case capi.SQLITE_ROW: return this._mayGet = true; default: this._mayGet = false; - console.warn("sqlite3_step() rc=",rc,"SQL =", - capi.sqlite3_sql(this.pointer)); - checkDbRc(this.db.pointer, rc); - }; + console.warn("sqlite3_step() rc=",rc, + capi.sqlite3_js_rc_str(rc), + "SQL =", capi.sqlite3_sql(this.pointer)); + DB.checkRc(this.db.pointer, rc); + } + }, + /** + Functions exactly like step() except that... + + 1) On success, it calls this.reset() and returns this object. + 2) On error, it throws and does not call reset(). + + This is intended to simplify constructs like: + + ``` + for(...) { + stmt.bind(...).stepReset(); + } + ``` + + Note that the reset() call makes it illegal to call this.get() + after the step. + */ + stepReset: function(){ + this.step(); + return this.reset(); + }, + /** + Functions like step() except that it finalizes this statement + immediately after stepping unless the step cannot be performed + because the statement is locked. Throws on error, but any error + other than the statement-is-locked case will also trigger + finalization of this statement. + + On success, it returns true if the step indicated that a row of + data was available, else it returns false. + + This is intended to simplify use cases such as: + + ``` + aDb.prepare("insert into foo(a) values(?)").bind(123).stepFinalize(); + ``` + */ + stepFinalize: function(){ + const rc = this.step(); + this.finalize(); + return rc; }, /** Fetches the value from the given 0-based column index of @@ -1301,7 +1623,7 @@ : asType){ case capi.SQLITE_NULL: return null; case capi.SQLITE_INTEGER:{ - if(capi.wasm.bigIntEnabled){ + if(wasm.bigIntEnabled){ const rc = capi.sqlite3_column_int64(this.pointer, ndx); if(rc>=Number.MIN_SAFE_INTEGER && rc<=Number.MAX_SAFE_INTEGER){ /* Coerce "normal" number ranges to normal number values, @@ -1332,8 +1654,8 @@ const n = capi.sqlite3_column_bytes(this.pointer, ndx), ptr = capi.sqlite3_column_blob(this.pointer, ndx), rc = new Uint8Array(n); - //heap = n ? capi.wasm.heap8() : false; - if(n) rc.set(capi.wasm.heap8u().slice(ptr, ptr+n), 0); + //heap = n ? wasm.heap8() : false; + if(n) rc.set(wasm.heap8u().slice(ptr, ptr+n), 0); //for(let i = 0; i < n; ++i) rc[i] = heap[ptr + i]; if(n && this.db._blobXfer instanceof Array){ /* This is an optimization soley for the @@ -1347,7 +1669,7 @@ default: toss3("Don't know how to translate", "type of result column #"+ndx+"."); } - abort("Not reached."); + toss3("Not reached."); }, /** Equivalent to get(ndx) but coerces the result to an integer. */ @@ -1374,7 +1696,7 @@ }, // Design note: the only reason most of these getters have a 'get' // prefix is for consistency with getVALUE_TYPE(). The latter - // arguablly really need that prefix for API readability and the + // arguably really need that prefix for API readability and the // rest arguably don't, but consistency is a powerful thing. /** Returns the result column name of the given index, or @@ -1395,9 +1717,8 @@ cannot have result columns. This object's columnCount member holds the number of columns. */ - getColumnNames: function(tgt){ + getColumnNames: function(tgt=[]){ affirmColIndex(affirmStmtOpen(this),0); - if(!tgt) tgt = []; for(let i = 0; i < this.columnCount; ++i){ tgt.push(capi.sqlite3_column_name(this.pointer, i)); } @@ -1425,7 +1746,7 @@ Object.defineProperty(Stmt.prototype, 'pointer', prop); Object.defineProperty(DB.prototype, 'pointer', prop); } - + /** The OO API's public namespace. */ sqlite3.oo1 = { version: { @@ -1434,5 +1755,46 @@ }, DB, Stmt - }/*SQLite3 object*/; -})(self); + }/*oo1 object*/; + + if(util.isUIThread()){ + /** + Functionally equivalent to DB(storageName,'c','kvvfs') except + that it throws if the given storage name is not one of 'local' + or 'session'. + */ + sqlite3.oo1.JsStorageDb = function(storageName='session'){ + if('session'!==storageName && 'local'!==storageName){ + toss3("JsStorageDb db name must be one of 'session' or 'local'."); + } + dbCtorHelper.call(this, { + filename: storageName, + flags: 'c', + vfs: "kvvfs" + }); + }; + const jdb = sqlite3.oo1.JsStorageDb; + jdb.prototype = Object.create(DB.prototype); + /** Equivalent to sqlite3_js_kvvfs_clear(). */ + jdb.clearStorage = capi.sqlite3_js_kvvfs_clear; + /** + Clears this database instance's storage or throws if this + instance has been closed. Returns the number of + database blocks which were cleaned up. + */ + jdb.prototype.clearStorage = function(){ + return jdb.clearStorage(affirmDbOpen(this).filename); + }; + /** Equivalent to sqlite3_js_kvvfs_size(). */ + jdb.storageSize = capi.sqlite3_js_kvvfs_size; + /** + Returns the _approximate_ number of bytes this database takes + up in its storage or throws if this instance has been closed. + */ + jdb.prototype.storageSize = function(){ + return jdb.storageSize(affirmDbOpen(this).filename); + }; + }/*main-window-only bits*/ + +}); + diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 4acab7770a..86285df1d3 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -1,5 +1,5 @@ /* - 2022-07-22 + 2022-09-18 The author disclaims copyright to this source code. In place of a legal notice, here is a blessing: @@ -10,10 +10,30 @@ *********************************************************************** - This file contains extensions to the sqlite3 WASM API related to the - Origin-Private FileSystem (OPFS). It is intended to be appended to - the main JS deliverable somewhere after sqlite3-api-glue.js and - before sqlite3-api-cleanup.js. + This file holds the synchronous half of an sqlite3_vfs + implementation which proxies, in a synchronous fashion, the + asynchronous Origin-Private FileSystem (OPFS) APIs using a second + Worker, implemented in sqlite3-opfs-async-proxy.js. This file is + intended to be appended to the main sqlite3 JS deliverable somewhere + after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. +*/ +'use strict'; +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +/** + installOpfsVfs() returns a Promise which, on success, installs an + sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs + which accept a VFS. It is intended to be called via + sqlite3ApiBootstrap.initializersAsync or an equivalent mechanism. + + The installed VFS uses the Origin-Private FileSystem API for + all file storage. On error it is rejected with an exception + explaining the problem. Reasons for rejection include, but are + not limited to: + + - The counterpart Worker (see below) could not be loaded. + + - The environment does not support OPFS. That includes when + this function is called from the main window thread. Significant notes and limitations: @@ -21,373 +41,1268 @@ available in bleeding-edge versions of Chrome (v102+, noting that that number will increase as the OPFS API matures). - - The _synchronous_ family of OPFS features (which is what this API - requires) are only available in non-shared Worker threads. This - file tries to detect that case and becomes a no-op if those - features do not seem to be available. + - The OPFS features used here are only available in dedicated Worker + threads. This file tries to detect that case, resulting in a + rejected Promise if those features do not seem to be available. + + - It requires the SharedArrayBuffer and Atomics classes, and the + former is only available if the HTTP server emits the so-called + COOP and COEP response headers. These features are required for + proxying OPFS's synchronous API via the synchronous interface + required by the sqlite3_vfs API. + + - This function may only be called a single time. When called, this + function removes itself from the sqlite3 object. + + All arguments to this function are for internal/development purposes + only. They do not constitute a public API and may change at any + time. + + The argument may optionally be a plain object with the following + configuration options: + + - proxyUri: as described above + + - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables + logging of errors. 2 enables logging of warnings and errors. 3 + additionally enables debugging info. + + - sanityChecks (=false): if true, some basic sanity tests are + run on the OPFS VFS API after it's initialized, before the + returned Promise resolves. + + On success, the Promise resolves to the top-most sqlite3 namespace + object and that object gets a new object installed in its + `opfs` property, containing several OPFS-specific utilities. */ - -// FileSystemHandle -// FileSystemDirectoryHandle -// FileSystemFileHandle -// FileSystemFileHandle.prototype.createSyncAccessHandle -self.sqlite3.postInit.push(function(self, sqlite3){ - const warn = console.warn.bind(console), - error = console.error.bind(console); - if(!self.importScripts || !self.FileSystemFileHandle - || !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ - warn("OPFS not found or its sync API is not available in this environment."); - return; - }else if(!sqlite3.capi.wasm.bigIntEnabled){ - error("OPFS requires BigInt support but sqlite3.capi.wasm.bigIntEnabled is false."); - return; +const installOpfsVfs = function callee(options){ + if(!self.SharedArrayBuffer || + !self.Atomics || + !self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + return Promise.reject( + new Error("This environment does not have OPFS support.") + ); } - //warn('self.FileSystemFileHandle =',self.FileSystemFileHandle); - //warn('self.FileSystemFileHandle.prototype =',self.FileSystemFileHandle.prototype); - const toss = (...args)=>{throw new Error(args.join(' '))}; - const capi = sqlite3.capi, - wasm = capi.wasm; - const sqlite3_vfs = capi.sqlite3_vfs - || toss("Missing sqlite3.capi.sqlite3_vfs object."); - const sqlite3_file = capi.sqlite3_file - || toss("Missing sqlite3.capi.sqlite3_file object."); - const sqlite3_io_methods = capi.sqlite3_io_methods - || toss("Missing sqlite3.capi.sqlite3_io_methods object."); - const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); - const debug = console.debug.bind(console), - log = console.log.bind(console); - warn("UNDER CONSTRUCTION: setting up OPFS VFS..."); - - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. Though we cannot currently handle - that case, the hope is to eventually be able to. */; - const oVfs = new sqlite3_vfs(); - const oIom = new sqlite3_io_methods(); - oVfs.$iVersion = 2/*yes, two*/; - oVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - oVfs.$mxPathname = 1024/*sure, why not?*/; - oVfs.$zName = wasm.allocCString("opfs"); - oVfs.ondispose = [ - '$zName', oVfs.$zName, - 'cleanup dVfs', ()=>(dVfs ? dVfs.dispose() : null) - ]; - if(dVfs){ - oVfs.$xSleep = dVfs.$xSleep; - oVfs.$xRandomness = dVfs.$xRandomness; + if(!options || 'object'!==typeof options){ + options = Object.create(null); + } + const urlParams = new URL(self.location.href).searchParams; + if(undefined===options.verbose){ + options.verbose = urlParams.has('opfs-verbose') ? 3 : 2; + } + if(undefined===options.sanityChecks){ + options.sanityChecks = urlParams.has('opfs-sanity-check'); + } + if(undefined===options.proxyUri){ + options.proxyUri = callee.defaultProxyUri; } - // All C-side memory of oVfs is zeroed out, but just to be explicit: - oVfs.$xDlOpen = oVfs.$xDlError = oVfs.$xDlSym = oVfs.$xDlClose = null; - /** - Pedantic sidebar about oVfs.ondispose: the entries in that array - are items to clean up when oVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the oVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ - - /** - Installs a StructBinder-bound function pointer member of the - given name and function in the given StructType target object. - It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws - on the slightest hint of error (e.g. tgt is-not-a StructType, - name does not map to a struct-bound member, etc.). - - Returns a proxy for this function which is bound to tgt and takes - 2 args (name,func). That function returns the same thing, - permitting calls to be chained. - - If called with only 1 arg, it has no side effects but returns a - func with the same signature as described above. - */ - const installMethod = function callee(tgt, name, func){ - if(!(tgt instanceof StructBinder.StructType)){ - toss("Usage error: target object is-not-a StructType."); - } - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } + const thePromise = new Promise(function(promiseResolve, promiseReject_){ + const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) + }; + const logImpl = (level,...args)=>{ + if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const toss = function(...args){throw new Error(args.join(' '))}; + const capi = sqlite3.capi; + const wasm = sqlite3.wasm; + const sqlite3_vfs = capi.sqlite3_vfs; + const sqlite3_file = capi.sqlite3_file; + const sqlite3_io_methods = capi.sqlite3_io_methods; + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + */ + const opfsUtil = Object.create(null); + /** + Not part of the public API. Solely for internal/development + use. + */ + opfsUtil.metrics = { + dump: function(){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; } - }; - callee.removeFuncList = function(){ - if(this.ondispose.__removeFuncList){ - this.ondispose.__removeFuncList.forEach( - (v,ndx)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - } - /* else it's a descriptive label for the next number in - the list. */ - } - ); - delete this.ondispose.__removeFuncList; + console.log(self.location.href, + "metrics for",self.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); + console.log("Serialization metrics:",metrics.s11n); + W.postMessage({type:'opfs-async-metrics'}); + }, + reset: function(){ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name); - if(sigN.length<2){ - toss("Member",name," is not a function pointer. Signature =",sigN); - } - const memKey = tgt.memberKey(name); - //log("installMethod",tgt, name, sigN); - const fProxy = 1 - // We can remove this proxy middle-man once the VFS is working - ? callee.argcProxy(func, sigN) - : func; - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose) tgt.ondispose = []; - if(!tgt.ondispose.__removeFuncList){ - tgt.ondispose.push('ondispose.__removeFuncList handler', - callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; - } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + } + }/*metrics*/; + const promiseReject = function(err){ + opfsVfs.dispose(); + return promiseReject_(err); + }; + const W = new Worker(options.proxyUri); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. Though we cannot currently handle + that case, the hope is to eventually be able to. */; + const opfsVfs = new sqlite3_vfs(); + const opfsIoMethods = new sqlite3_io_methods(); + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/*sure, why not?*/; + opfsVfs.$zName = wasm.allocCString("opfs"); + // All C-side memory of opfsVfs is zeroed out, but just to be explicit: + opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; + opfsVfs.ondispose = [ + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null), + 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose() + ]; + /** + Pedantic sidebar about opfsVfs.ondispose: the entries in that array + are items to clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the opfsVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + /** + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. - /** - Map of sqlite3_file pointers to OPFS handles. - */ - const __opfsHandles = Object.create(null); + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(''); - }; + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. + */ + const state = Object.create(null); + state.verbose = options.verbose; + state.littleEndian = (()=>{ + const buffer = new ArrayBuffer(2); + new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); + // Int16Array uses the platform's endianness. + return new Int16Array(buffer)[0] === 256; + })(); + /** + Whether the async counterpart should log exceptions to + the serialization channel. That produces a great deal of + noise for seemingly innocuous things like xAccess() checks + for missing files, so this option may have one of 3 values: - //const rootDir = await navigator.storage.getDirectory(); - - //////////////////////////////////////////////////////////////////////// - // Set up OPFS VFS methods... - let inst = installMethod(oVfs); - inst('xOpen', function(pVfs, zName, pFile, flags, pOutFlags){ - const f = new sqlite3_file(pFile); - f.$pMethods = oIom.pointer; - __opfsHandles[pFile] = f; - f.opfsHandle = null /* TODO */; - if(flags & capi.SQLITE_OPEN_DELETEONCLOSE){ - f.deleteOnClose = true; - } - f.filename = zName ? wasm.cstringToJs(zName) : randomFilename(); - error("OPFS sqlite3_vfs::xOpen is not yet full implemented."); - return capi.SQLITE_IOERR; - }) - ('xFullPathname', function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return i{ + if(undefined === (state.sq3Codes[k] = capi[k])){ + toss("Maintenance required: not found:",k); + } }); - } - //////////////////////////////////////////////////////////////////////// - // Set up OPFS sqlite3_io_methods... - inst = installMethod(oIom); - inst('xClose', async function(pFile){ - warn("xClose(",arguments,") uses await"); - const f = __opfsHandles[pFile]; - delete __opfsHandles[pFile]; - if(f.opfsHandle){ - await f.opfsHandle.close(); - if(f.deleteOnClose){ - // TODO + /** + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The + 2nd and subsequent arguments must be the aruguments for the + async op. + */ + const opRun = (op,...args)=>{ + const opNdx = state.opIds[op] || toss("Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, state.opIds.rc, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + /* async thread will take over here */; + const t = performance.now(); + Atomics.wait(state.sabOPView, state.opIds.rc, -1) + /* When this wait() call returns, the async half will have + completed the operation and reported its results. */; + const rc = Atomics.load(state.sabOPView, state.opIds.rc); + metrics[op].wait += performance.now() - t; + if(rc && state.asyncS11nExceptions){ + const err = state.s11n.deserialize(); + if(err) error(op+"() async error:",...err); } - } - f.dispose(); - return 0; - }) - ('xRead', /*i(ppij)*/function(pFile,pDest,n,offset){ - /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ - try { - const f = __opfsHandles[pFile]; - const heap = wasm.heap8u(); - const b = new Uint8Array(heap.buffer, pDest, n); - const nRead = f.opfsHandle.read(b, {at: offset}); - if(nRead SQLITE_DEFAULT_SECTOR_SIZE */; - //}) + return rc; + }; - const rc = capi.sqlite3_vfs_register(oVfs.pointer, 0); - if(rc){ - oVfs.dispose(); - toss("sqlite3_vfs_register(OPFS) failed with rc",rc); + /** + Not part of the public API. Only for test/development use. + */ + opfsUtil.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; + + const initS11n = ()=>{ + /** + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ACHTUNG: this code is 100% duplicated in the other half of + this proxy! The documentation is maintained in the + "synchronous half". + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset]. Only one dataset is recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from from this thread or the + counterpart thread), or null if the serialization buffer is empty. + */ + state.s11n.deserialize = function(){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + return state.s11n; + }/*initS11n()*/; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(''); + }; + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = Object.create(null); + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error (e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc.). + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + */ + const installMethod = function callee(tgt, name, func){ + if(!(tgt instanceof sqlite3.StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + const fProxy = 0 + /** This middle-man proxy is only for use during development, to + confirm that we always pass the proper number of + arguments. We know that the C-level code will always use the + correct argument count. */ + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + const opTimer = Object.create(null); + opTimer.op = undefined; + opTimer.start = undefined; + const mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + ++metrics[op].count; + }; + const mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start + ); + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + /** + As of late 2022, only a single lock can be held on an OPFS + file. We have no way of checking whether any _other_ db + connection has a lock except by trying to obtain and (on + success) release a sync-handle for it, but doing so would + involve an inherent race condition. For the time being, + pending a better solution, we simply report whether the + given pFile instance has a lock. + */ + const f = __openFiles[pFile]; + wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32'); + return 0; + }, + xClose: function(pFile){ + mTimeStart('xClose'); + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + mTimeEnd(); + return rc; + }, + xDeviceCharacteristics: function(pFile){ + //debug("xDeviceCharacteristics(",pFile,")"); + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + mTimeStart('xFileControl'); + const rc = (capi.SQLITE_FCNTL_SYNC===opId) + ? opRun('xSync', pFile, 0) + : capi.SQLITE_NOTFOUND; + mTimeEnd(); + return rc; + }, + xFileSize: function(pFile,pSz64){ + mTimeStart('xFileSize'); + const rc = opRun('xFileSize', pFile); + if(0==rc){ + const sz = state.s11n.deserialize()[0]; + wasm.setMemValue(pSz64, sz, 'i64'); + } + mTimeEnd(); + return rc; + }, + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === f.lockType ) { + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; + }else{ + f.lockType = lockType; + } + mTimeEnd(); + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + mTimeStart('xRead'); + const f = __openFiles[pFile]; + let rc; + try { + rc = opRun('xRead',pFile, n, Number(offset64)); + if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ + /** + Results get written to the SharedArrayBuffer f.sabView. + Because the heap is _not_ a SharedArrayBuffer, we have + to copy the results. TypedArray.set() seems to be the + fastest way to copy this. */ + wasm.heap8u().set(f.sabView.subarray(0, n), pDest); + } + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + mTimeEnd(); + return rc; + }, + xSync: function(pFile,flags){ + ++metrics.xSync.count; + return 0; // impl'd in xFileControl() + }, + xTruncate: function(pFile,sz64){ + mTimeStart('xTruncate'); + const rc = opRun('xTruncate', pFile, Number(sz64)); + mTimeEnd(); + return rc; + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === lockType + && f.lockType ){ + rc = opRun('xUnlock', pFile, lockType); + } + if( 0===rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + mTimeStart('xWrite'); + const f = __openFiles[pFile]; + let rc; + try { + f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n)); + rc = opRun('xWrite', pFile, n, Number(offset64)); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_WRITE; + } + mTimeEnd(); + return rc; + } + }/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + mTimeStart('xAccess'); + const rc = opRun('xAccess', wasm.cstringToJs(zName)); + wasm.setMemValue( pOut, (rc ? 0 : 1), 'i32' ); + mTimeEnd(); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + /* If it turns out that we need to adjust for timezone, see: + https://stackoverflow.com/a/11760121/1458521 */ + wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + // TODO: confirm that this calculation is correct + wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + mTimeStart('xDelete'); + opRun('xDelete', wasm.cstringToJs(zName), doSyncDir, false); + /* We're ignoring errors because we cannot yet differentiate + between harmless and non-harmless failures. */ + mTimeEnd(); + return 0; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return ipMethods is NULL. */ + if(fh.readOnly){ + wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); + } + __openFiles[pFile] = fh; + fh.sabView = state.sabFileBufView; + fh.sq3File = new sqlite3_file(pFile); + fh.sq3File.$pMethods = opfsIoMethods.pointer; + fh.lockType = capi.SQLITE_LOCK_NONE; + } + mTimeEnd(); + return rc; + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + + if(dVfs){ + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + } + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + /* If we can inherit an xSleep() impl from the default VFS then + assume it's sane and use it, otherwise install a JS-based + one. */ + vfsSyncWrappers.xSleep = function(pVfs,ms){ + Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); + return 0; + }; + } + + /* Install the vfs/io_methods into their C-level shared instances... */ + for(let k of Object.keys(ioSyncWrappers)){ + installMethod(opfsIoMethods, k, ioSyncWrappers[k]); + } + for(let k of Object.keys(vfsSyncWrappers)){ + installMethod(opfsVfs, k, vfsSyncWrappers[k]); + } + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg + is true, the result is returned as an array of path elements, + else an absolute path string is returned. + */ + opfsUtil.getResolvedPath = function(filename,splitIt){ + const p = new URL(filename, "file://irrelevant").pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = opfsUtil.rootDirectory; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //console.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string, intended for use as a + temporary file name. Its argument is the length of the string, + defaulting to 16. + */ + opfsUtil.randomFilename = randomFilename; + + /** + Re-registers the OPFS VFS. This is intended only for odd use + cases which have to call sqlite3_shutdown() as part of their + initialization process, which will unregister the VFS + registered by installOpfsVfs(). If passed a truthy value, the + OPFS VFS is registered as the default VFS, else it is not made + the default. Returns the result of the the + sqlite3_vfs_register() call. + + Design note: the problem of having to re-register things after + a shutdown/initialize pair is more general. How to best plug + that in to the library is unclear. In particular, we cannot + hook in to any C-side calls to sqlite3_initialize(), so we + cannot add an after-initialize callback mechanism. + */ + opfsUtil.registerVfs = (asDefault=false)=>{ + return wasm.exports.sqlite3_vfs_register( + opfsVfs.pointer, asDefault ? 1 : 0 + ); + }; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + await doDir(opfsUtil.rootDirectory, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which it will throw + are not documented (so we cannot tell you what they are). + */ + opfsUtil.rmfr = async function(){ + const dir = opfsUtil.rootDirectory, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each one. + The argument may be either a callback function or an options object + with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + + TODO: add an option which specifies whether to traverse + depth-first or breadth-first. We currently do depth-first but + an incremental file browsing widget would benefit more from + breadth-first. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: opfsUtil.rootDirectory + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + //TODO to support fiddle and worker1 db upload: + //opfsUtil.createFile = function(absName, content=undefined){...} + + if(sqlite3.oo1){ + opfsUtil.OpfsDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = opfsVfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( + opfsVfs.pointer, + [ + /* Truncate journal mode is faster than delete or wal for + this vfs, per speedtest1. */ + "pragma journal_mode=truncate;" + /* + This vfs benefits hugely from cache on moderate/large + speedtest1 --size 50 and --size 100 workloads. We currently + rely on setting a non-default cache size when building + sqlite3.wasm. If that policy changes, the cache can + be set here. + */ + //"pragma cache_size=-8388608;" + ].join('') + ); + } + + /** + Potential TODOs: + + - Expose one or both of the Worker objects via opfsUtil and + publish an interface for proxying the higher-level OPFS + features like getting a directory listing. + */ + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.setMemValue(pOut,0,'i64'); + rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.getMemValue(pOut, 'i64')); + rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.setMemValue(readBuf+6,0); + let jRead = wasm.cstringToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //log("Worker.onmessage:",data); + switch(data.type){ + case 'opfs-async-loaded': + /*Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async worker.*/ + W.postMessage({type: 'opfs-async-init',args: state}); + break; + case 'opfs-async-inited':{ + /*Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin...*/ + try { + const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, 0); + if(rc){ + toss("sqlite3_vfs_register(OPFS) failed with rc",rc); + } + if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){ + toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); + } + capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + initS11n(); + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + navigator.storage.getDirectory().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + sqlite3.opfs = opfsUtil; + opfsUtil.rootDirectory = d; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(sqlite3); + }); + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + default: + promiseReject(e); + error("Unexpected message from the async worker:",data); + break; + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; +}/*installOpfsVfs()*/; +installOpfsVfs.defaultProxyUri = + "sqlite3-opfs-async-proxy.js"; +self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + if(sqlite3.scriptInfo && !sqlite3.scriptInfo.isWorker){ + return; + } + try{ + let proxyJs = installOpfsVfs.defaultProxyUri; + if(sqlite3.scriptInfo.sqlite3Dir){ + installOpfsVfs.defaultProxyUri = + sqlite3.scriptInfo.sqlite3Dir + proxyJs; + //console.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); + } + return installOpfsVfs().catch((e)=>{ + console.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); + }); + }catch(e){ + console.error("installOpfsVfs() exception:",e); + throw e; } - capi.sqlite3_vfs_register.addReference(oVfs, oIom); - warn("End of (very incomplete) OPFS setup.", oVfs); - //oVfs.dispose()/*only because we can't yet do anything with it*/; }); +}/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 60ed61477e..cf44f39707 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -43,10 +43,7 @@ - Insofar as possible, support client-side storage using JS filesystem APIs. As of this writing, such things are still very - much TODO. Initial testing with using IndexedDB as backing storage - showed it to work reasonably well, but it's also too easy to - corrupt by using a web page in two browser tabs because IndexedDB - lacks the locking features needed to support that. + much under development. Specific non-goals of this project: @@ -54,8 +51,10 @@ Encodings in that realm, there are no currently plans to support the UTF16-related sqlite3 APIs. They would add a complication to the bindings for no appreciable benefit. Though web-related - implementation details take priority, the lower-level WASM module - "should" work in non-web WASM environments. + implementation details take priority, and the JavaScript + components of the API specifically focus on browser clients, the + lower-level WASM module "should" work in non-web WASM + environments. - Supporting old or niche-market platforms. WASM is built for a modern web and requires modern platforms. @@ -78,24 +77,183 @@ */ /** - This global symbol is is only a temporary measure: the JS-side - post-processing will remove that object from the global scope when - setup is complete. We require it there temporarily in order to glue - disparate parts together during the loading of the API (which spans - several components). + sqlite3ApiBootstrap() is the only global symbol persistently + exposed by this API. It is intended to be called one time at the + end of the API amalgamation process, passed configuration details + for the current environment, and then optionally be removed from + the global object using `delete self.sqlite3ApiBootstrap`. - This function requires a configuration object intended to abstract + This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so - that it can be used without any _direct_ dependency on Emscripten. - (That said, OO API #1 requires, as of this writing, Emscripten's - virtual filesystem API. Baby steps.) -*/ -self.sqlite3ApiBootstrap = function(config){ - 'use strict'; + that it can be used without any _direct_ dependency on + Emscripten. (Note the default values for the config object!) The + config object is only honored the first time this is + called. Subsequent calls ignore the argument and return the same + (configured) object which gets initialized by the first call. - /** Throws a new Error, the message of which is the concatenation - all args with a space between each. */ - const toss = (...args)=>{throw new Error(args.join(' '))}; + The config object properties include: + + - `exports`[^1]: the "exports" object for the current WASM + environment. In an Emscripten build, this should be set to + `Module['asm']`. + + - `memory`[^1]: optional WebAssembly.Memory object, defaulting to + `exports.memory`. In Emscripten environments this should be set + to `Module.wasmMemory` if the build uses `-sIMPORT_MEMORY`, or be + left undefined/falsy to default to `exports.memory` when using + WASM-exported memory. + + - `bigIntEnabled`: true if BigInt support is enabled. Defaults to + true if self.BigInt64Array is available, else false. Some APIs + will throw exceptions if called without BigInt support, as BigInt + is required for marshalling C-side int64 into and out of JS. + + - `allocExportName`: the name of the function, in `exports`, of the + `malloc(3)`-compatible routine for the WASM environment. Defaults + to `"malloc"`. + + - `deallocExportName`: the name of the function, in `exports`, of + the `free(3)`-compatible routine for the WASM + environment. Defaults to `"free"`. + + - `wasmfsOpfsDir`[^1]: if the environment supports persistent storage, this + directory names the "mount point" for that directory. It must be prefixed + by `/` and may currently contain only a single directory-name part. Using + the root directory name is not supported by any current persistent backend. + + + [^1] = This property may optionally be a function, in which case this + function re-assigns it to the value returned from that function, + enabling delayed evaluation. + +*/ +'use strict'; +self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( + apiConfig = (self.sqlite3ApiConfig || sqlite3ApiBootstrap.defaultConfig) +){ + if(sqlite3ApiBootstrap.sqlite3){ /* already initalized */ + console.warn("sqlite3ApiBootstrap() called multiple times.", + "Config and external initializers are ignored on calls after the first."); + return sqlite3ApiBootstrap.sqlite3; + } + const config = Object.assign(Object.create(null),{ + exports: undefined, + memory: undefined, + bigIntEnabled: (()=>{ + if('undefined'!==typeof Module){ + /* Emscripten module will contain HEAPU64 when built with + -sWASM_BIGINT=1, else it will not. */ + return !!Module.HEAPU64; + } + return !!self.BigInt64Array; + })(), + allocExportName: 'malloc', + deallocExportName: 'free', + wasmfsOpfsDir: '/opfs' + }, apiConfig || {}); + + [ + // If any of these config options are functions, replace them with + // the result of calling that function... + 'exports', 'memory', 'wasmfsOpfsDir' + ].forEach((k)=>{ + if('function' === typeof config[k]){ + config[k] = config[k](); + } + }); + + /** + The main sqlite3 binding API gets installed into this object, + mimicking the C API as closely as we can. The numerous members + names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as + possible, identically to the C-native counterparts, as documented at: + + https://www.sqlite.org/c3ref/intro.html + + A very few exceptions require an additional level of proxy + function or may otherwise require special attention in the WASM + environment, and all such cases are documented somewhere below + in this file or in sqlite3-api-glue.js. capi members which are + not documented are installed as 1-to-1 proxies for their + C-side counterparts. + */ + const capi = Object.create(null); + /** + Holds state which are specific to the WASM-related + infrastructure and glue code. It is not expected that client + code will normally need these, but they're exposed here in case + it does. These APIs are _not_ to be considered an + official/stable part of the sqlite3 WASM API. They may change + as the developers' experience suggests appropriate changes. + + Note that a number of members of this object are injected + dynamically after the api object is fully constructed, so + not all are documented in this file. + */ + const wasm = Object.create(null); + + /** Internal helper for SQLite3Error ctor. */ + const __rcStr = (rc)=>{ + return (capi.sqlite3_js_rc_str && capi.sqlite3_js_rc_str(rc)) + || ("Unknown result code #"+rc); + }; + + /** Internal helper for SQLite3Error ctor. */ + const __isInt = (n)=>'number'===typeof n && n===(n | 0); + + /** + An Error subclass specifically for reporting DB-level errors and + enabling clients to unambiguously identify such exceptions. + The C-level APIs never throw, but some of the higher-level + C-style APIs do and the object-oriented APIs use exceptions + exclusively to report errors. + */ + class SQLite3Error extends Error { + /** + Constructs this object with a message depending on its arguments: + + - If it's passed only a single integer argument, it is assumed + to be an sqlite3 C API result code. The message becomes the + result of sqlite3.capi.sqlite3_js_rc_str() or (if that returns + falsy) a synthesized string which contains that integer. + + - If passed 2 arguments and the 2nd is a object, it bevaves + like the Error(string,object) constructor except that the first + argument is subject to the is-integer semantics from the + previous point. + + - Else all arguments are concatenated with a space between each + one, using args.join(' '), to create the error message. + */ + constructor(...args){ + if(1===args.length && __isInt(args[0])){ + super(__rcStr(args[0])); + }else if(2===args.length && 'object'===typeof args){ + if(__isInt(args[0])) super(__rcStr(args[0]), args[1]); + else super(...args); + }else{ + super(args.join(' ')); + } + this.name = 'SQLite3Error'; + } + }; + + /** + Functionally equivalent to the SQLite3Error constructor but may + be used as part of an expression, e.g.: + + ``` + return someFunction(x) || SQLite3Error.toss(...); + ``` + */ + SQLite3Error.toss = (...args)=>{ + throw new SQLite3Error(...args); + }; + const toss3 = SQLite3Error.toss; + + if(config.wasmfsOpfsDir && !/^\/[^/]+$/.test(config.wasmfsOpfsDir)){ + toss3("config.wasmfsOpfsDir must be falsy or in the form '/dir-name'."); + } /** Returns true if n is a 32-bit (signed) integer, else @@ -103,16 +261,71 @@ self.sqlite3ApiBootstrap = function(config){ double-type DB operations for integer values in order to keep more precision. */ - const isInt32 = function(n){ + const isInt32 = (n)=>{ return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/) && !!(n===(n|0) && n<=2147483647 && n>=-2147483648); }; + /** + Returns true if the given BigInt value is small enough to fit + into an int64 value, else false. + */ + const bigIntFits64 = function f(b){ + if(!f._max){ + f._max = BigInt("0x7fffffffffffffff"); + f._min = ~f._max; + } + return b >= f._min && b <= f._max; + }; + + /** + Returns true if the given BigInt value is small enough to fit + into an int32, else false. + */ + const bigIntFits32 = (b)=>(b >= (-0x7fffffffn - 1n) && b <= 0x7fffffffn); + + /** + Returns true if the given BigInt value is small enough to fit + into a double value without loss of precision, else false. + */ + const bigIntFitsDouble = function f(b){ + if(!f._min){ + f._min = Number.MIN_SAFE_INTEGER; + f._max = Number.MAX_SAFE_INTEGER; + } + return b >= f._min && b <= f._max; + }; /** Returns v if v appears to be a TypedArray, else false. */ const isTypedArray = (v)=>{ return (v && v.constructor && isInt32(v.constructor.BYTES_PER_ELEMENT)) ? v : false; }; + + /** Internal helper to use in operations which need to distinguish + between TypedArrays which are backed by a SharedArrayBuffer + from those which are not. */ + const __SAB = ('undefined'===typeof SharedArrayBuffer) + ? function(){} : SharedArrayBuffer; + /** Returns true if the given TypedArray object is backed by a + SharedArrayBuffer, else false. */ + const isSharedTypedArray = (aTypedArray)=>(aTypedArray.buffer instanceof __SAB); + + /** + Returns either aTypedArray.slice(begin,end) (if + aTypedArray.buffer is a SharedArrayBuffer) or + aTypedArray.subarray(begin,end) (if it's not). + + This distinction is important for APIs which don't like to + work on SABs, e.g. TextDecoder, and possibly for our + own APIs which work on memory ranges which "might" be + modified by other threads while they're working. + */ + const typedArrayPart = (aTypedArray, begin, end)=>{ + return isSharedTypedArray(aTypedArray) + ? aTypedArray.slice(begin, end) + : aTypedArray.subarray(begin, end); + }; + /** Returns true if v appears to be one of our bind()-able TypedArray types: Uint8Array or Int8Array. Support for @@ -139,11 +352,35 @@ self.sqlite3ApiBootstrap = function(config){ that v is not a supported TypedArray value. */ const affirmBindableTypedArray = (v)=>{ return isBindableTypedArray(v) - || toss("Value is not of a supported TypedArray type."); + || toss3("Value is not of a supported TypedArray type."); }; const utf8Decoder = new TextDecoder('utf-8'); - const typedArrayToString = (str)=>utf8Decoder.decode(str); + + /** + Uses TextDecoder to decode the given half-open range of the + given TypedArray to a string. This differs from a simple + call to TextDecoder in that it accounts for whether the + first argument is backed by a SharedArrayBuffer or not, + and can work more efficiently if it's not (TextDecoder + refuses to act upon an SAB). + */ + const typedArrayToString = function(typedArray, begin, end){ + return utf8Decoder.decode(typedArrayPart(typedArray, begin,end)); + }; + + /** + If v is-a Array, its join("") result is returned. If + isSQLableTypedArray(v) is true then typedArrayToString(v) is + returned. If it looks like a WASM pointer, wasm.cstringToJs(v) is + returned. Else v is returned as-is. + */ + const flexibleString = function(v){ + if(isSQLableTypedArray(v)) return typedArrayToString(v); + else if(Array.isArray(v)) return v.join(""); + else if(wasm.isPtr(v)) v = wasm.cstringToJs(v); + return v; + }; /** An Error subclass specifically for reporting Wasm-level malloc() @@ -156,27 +393,304 @@ self.sqlite3ApiBootstrap = function(config){ this.name = 'WasmAllocError'; } }; + /** + Functionally equivalent to the WasmAllocError constructor but may + be used as part of an expression, e.g.: - /** - The main sqlite3 binding API gets installed into this object, - mimicking the C API as closely as we can. The numerous members - names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as - possible, identically to the C-native counterparts, as documented at: - - https://www.sqlite.org/c3ref/intro.html - - A very few exceptions require an additional level of proxy - function or may otherwise require special attention in the WASM - environment, and all such cases are document here. Those not - documented here are installed as 1-to-1 proxies for their C-side - counterparts. + ``` + return someAllocatingFunction(x) || WasmAllocError.toss(...); + ``` */ - const capi = { + WasmAllocError.toss = (...args)=>{ + throw new WasmAllocError(...args); + }; + + Object.assign(capi, { /** - An Error subclass which is thrown by this object's alloc() method - on OOM. + sqlite3_create_function_v2() differs from its native + counterpart only in the following ways: + + 1) The fourth argument (`eTextRep`) argument must not specify + any encoding other than sqlite3.SQLITE_UTF8. The JS API does not + currently support any other encoding and likely never + will. This function does not replace that argument on its own + because it may contain other flags. + + 2) Any of the four final arguments may be either WASM pointers + (assumed to be function pointers) or JS Functions. In the + latter case, each gets bound to WASM using + sqlite3.capi.wasm.installFunction() and that wrapper is passed + on to the native implementation. + + The semantics of JS functions are: + + xFunc: is passed `(pCtx, ...values)`. Its return value becomes + the new SQL function's result. + + xStep: is passed `(pCtx, ...values)`. Its return value is + ignored. + + xFinal: is passed `(pCtx)`. Its return value becomes the new + aggregate SQL function's result. + + xDestroy: is passed `(void*)`. Its return value is ignored. The + pointer passed to it is the one from the 5th argument to + sqlite3_create_function_v2(). + + Note that: + + - `pCtx` in the above descriptions is a `sqlite3_context*`. At + least 99 times out of a hundred, that initial argument will + be irrelevant for JS UDF bindings, but it needs to be there + so that the cases where it _is_ relevant, in particular with + window and aggregate functions, have full access to the + lower-level sqlite3 APIs. + + - When wrapping JS functions, the remaining arguments are passd + to them as positional arguments, not as an array of + arguments, because that allows callback definitions to be + more JS-idiomatic than C-like. For example `(pCtx,a,b)=>a+b` + is more intuitive and legible than + `(pCtx,args)=>args[0]+args[1]`. For cases where an array of + arguments would be more convenient, the callbacks simply need + to be declared like `(pCtx,...args)=>{...}`, in which case + `args` will be an array. + + - If a JS wrapper throws, it gets translated to + sqlite3_result_error() or sqlite3_result_error_nomem(), + depending on whether the exception is an + sqlite3.WasmAllocError object or not. + + - When passing on WASM function pointers, arguments are _not_ + converted or reformulated. They are passed on as-is in raw + pointer form using their native C signatures. Only JS + functions passed in to this routine, and thus wrapped by this + routine, get automatic conversions of arguments and result + values. The routines which perform those conversions are + exposed for client-side use as + sqlite3_create_function_v2.convertUdfArgs() and + sqlite3_create_function_v2.setUdfResult(). sqlite3_create_function() + and sqlite3_create_window_function() have those same methods. + + For xFunc(), xStep(), and xFinal(): + + - When called from SQL, arguments to the UDF, and its result, + will be converted between JS and SQL with as much fidelity as + is feasible, triggering an exception if a type conversion + cannot be determined. Some freedom is afforded to numeric + conversions due to friction between the JS and C worlds: + integers which are larger than 32 bits may be treated as + doubles or BigInts. + + If any JS-side bound functions throw, those exceptions are + intercepted and converted to database-side errors with the + exception of xDestroy(): any exception from it is ignored, + possibly generating a console.error() message. Destructors + must not throw. + + Once installed, there is currently no way to uninstall the + automatically-converted WASM-bound JS functions from WASM. They + can be uninstalled from the database as documented in the C + API, but this wrapper currently has no infrastructure in place + to also free the WASM-bound JS wrappers, effectively resulting + in a memory leak if the client uninstalls the UDF. Improving that + is a potential TODO, but removing client-installed UDFs is rare + in practice. If this factor is relevant for a given client, + they can create WASM-bound JS functions themselves, hold on to their + pointers, and pass the pointers in to here. Later on, they can + free those pointers (using `wasm.uninstallFunction()` or + equivalent). + + C reference: https://www.sqlite.org/c3ref/create_function.html + + Maintenance reminder: the ability to add new + WASM-accessible functions to the runtime requires that the + WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` + flag. */ - WasmAllocError: WasmAllocError, + sqlite3_create_function_v2: function( + pDb, funcName, nArg, eTextRep, pApp, + xFunc, xStep, xFinal, xDestroy + ){/*installed later*/}, + /** + Equivalent to passing the same arguments to + sqlite3_create_function_v2(), with 0 as the final argument. + */ + sqlite3_create_function:function( + pDb, funcName, nArg, eTextRep, pApp, + xFunc, xStep, xFinal + ){/*installed later*/}, + /** + The sqlite3_create_window_function() JS wrapper differs from + its native implementation in the exact same way that + sqlite3_create_function_v2() does. The additional function, + xInverse(), is treated identically to xStep() by the wrapping + layer. + */ + sqlite3_create_window_function: function( + pDb, funcName, nArg, eTextRep, pApp, + xStep, xFinal, xValue, xInverse, xDestroy + ){/*installed later*/}, + /** + The sqlite3_prepare_v3() binding handles two different uses + with differing JS/WASM semantics: + + 1) sqlite3_prepare_v3(pDb, sqlString, -1, prepFlags, ppStmt , null) + + 2) sqlite3_prepare_v3(pDb, sqlPointer, sqlByteLen, prepFlags, ppStmt, sqlPointerToPointer) + + Note that the SQL length argument (the 3rd argument) must, for + usage (1), always be negative because it must be a byte length + and that value is expensive to calculate from JS (where only + the character length of strings is readily available). It is + retained in this API's interface for code/documentation + compatibility reasons but is currently _always_ ignored. With + usage (2), the 3rd argument is used as-is but is is still + critical that the C-style input string (2nd argument) be + terminated with a 0 byte. + + In usage (1), the 2nd argument must be of type string, + Uint8Array, or Int8Array (either of which is assumed to + hold SQL). If it is, this function assumes case (1) and + calls the underyling C function with the equivalent of: + + (pDb, sqlAsString, -1, prepFlags, ppStmt, null) + + The `pzTail` argument is ignored in this case because its + result is meaningless when a string-type value is passed + through: the string goes through another level of internal + conversion for WASM's sake and the result pointer would refer + to that transient conversion's memory, not the passed-in + string. + + If the sql argument is not a string, it must be a _pointer_ to + a NUL-terminated string which was allocated in the WASM memory + (e.g. using capi.wasm.alloc() or equivalent). In that case, + the final argument may be 0/null/undefined or must be a pointer + to which the "tail" of the compiled SQL is written, as + documented for the C-side sqlite3_prepare_v3(). In case (2), + the underlying C function is called with the equivalent of: + + (pDb, sqlAsPointer, sqlByteLen, prepFlags, ppStmt, pzTail) + + It returns its result and compiled statement as documented in + the C API. Fetching the output pointers (5th and 6th + parameters) requires using `capi.wasm.getMemValue()` (or + equivalent) and the `pzTail` will point to an address relative to + the `sqlAsPointer` value. + + If passed an invalid 2nd argument type, this function will + return SQLITE_MISUSE and sqlite3_errmsg() will contain a string + describing the problem. + + Side-note: if given an empty string, or one which contains only + comments or an empty SQL expression, 0 is returned but the result + output pointer will be NULL. + */ + sqlite3_prepare_v3: (dbPtr, sql, sqlByteLen, prepFlags, + stmtPtrPtr, strPtrPtr)=>{}/*installed later*/, + + /** + Equivalent to calling sqlite3_prapare_v3() with 0 as its 4th argument. + */ + sqlite3_prepare_v2: (dbPtr, sql, sqlByteLen, + stmtPtrPtr,strPtrPtr)=>{}/*installed later*/, + + /** + This binding enables the callback argument to be a JavaScript. + + If the callback is a function, then for the duration of the + sqlite3_exec() call, it installs a WASM-bound function which + acts as a proxy for the given callback. That proxy will also + perform a conversion of the callback's arguments from + `(char**)` to JS arrays of strings. However, for API + consistency's sake it will still honor the C-level callback + parameter order and will call it like: + + `callback(pVoid, colCount, listOfValues, listOfColNames)` + + If the callback is not a JS function then this binding performs + no translation of the callback, but the sql argument is still + converted to a WASM string for the call using the + "flexible-string" argument converter. + */ + sqlite3_exec: (pDb, sql, callback, pVoid, pErrMsg)=>{}/*installed later*/, + + /** + If passed a single argument which appears to be a byte-oriented + TypedArray (Int8Array or Uint8Array), this function treats that + TypedArray as an output target, fetches `theArray.byteLength` + bytes of randomness, and populates the whole array with it. As + a special case, if the array's length is 0, this function + behaves as if it were passed (0,0). When called this way, it + returns its argument, else it returns the `undefined` value. + + If called with any other arguments, they are passed on as-is + to the C API. Results are undefined if passed any incompatible + values. + */ + sqlite3_randomness: (n, outPtr)=>{/*installed later*/}, + }/*capi*/); + + /** + Various internal-use utilities are added here as needed. They + are bound to an object only so that we have access to them in + the differently-scoped steps of the API bootstrapping + process. At the end of the API setup process, this object gets + removed. These are NOT part of the public API. + */ + const util = { + affirmBindableTypedArray, flexibleString, + bigIntFits32, bigIntFits64, bigIntFitsDouble, + isBindableTypedArray, + isInt32, isSQLableTypedArray, isTypedArray, + typedArrayToString, + isUIThread: ()=>'undefined'===typeof WorkerGlobalScope, + isSharedTypedArray, + typedArrayPart + }; + + Object.assign(wasm, { + /** + Emscripten APIs have a deep-seated assumption that all pointers + are 32 bits. We'll remain optimistic that that won't always be + the case and will use this constant in places where we might + otherwise use a hard-coded 4. + */ + ptrSizeof: config.wasmPtrSizeof || 4, + /** + The WASM IR (Intermediate Representation) value for + pointer-type values. It MUST refer to a value type of the + size described by this.ptrSizeof _or_ it may be any value + which ends in '*', which Emscripten's glue code internally + translates to i32. + */ + ptrIR: config.wasmPtrIR || "i32", + /** + True if BigInt support was enabled via (e.g.) the + Emscripten -sWASM_BIGINT flag, else false. When + enabled, certain 64-bit sqlite3 APIs are enabled which + are not otherwise enabled due to JS/WASM int64 + impedence mismatches. + */ + bigIntEnabled: !!config.bigIntEnabled, + /** + The symbols exported by the WASM environment. + */ + exports: config.exports + || toss3("Missing API config.exports (WASM module exports)."), + + /** + When Emscripten compiles with `-sIMPORT_MEMORY`, it + initalizes the heap and imports it into wasm, as opposed to + the other way around. In this case, the memory is not + available via this.exports.memory. + */ + memory: config.memory || config.exports['memory'] + || toss3("API config object requires a WebAssembly.Memory object", + "in either config.exports.memory (exported)", + "or config.memory (imported)."), + /** The API's one single point of access to the WASM-side memory allocator. Works like malloc(3) (and is likely bound to @@ -201,180 +715,17 @@ self.sqlite3ApiBootstrap = function(config){ deallocator. Works like free(3) (and is likely bound to free()). */ - dealloc: undefined/*installed later*/, - /** - When using sqlite3_open_v2() it is important to keep the following - in mind: + dealloc: undefined/*installed later*/ - https://www.sqlite.org/c3ref/open.html - - - The flags for use with its 3rd argument are installed in this - object using the C-cide names, e.g. SQLITE_OPEN_CREATE. - - - If the combination of flags passed to it are invalid, - behavior is undefined. Thus is is never okay to call this - with fewer than 3 arguments, as JS will default the - missing arguments to `undefined`, which will result in a - flag value of 0. Most of the available SQLITE_OPEN_xxx - flags are meaningless in the WASM build, e.g. the mutext- - and cache-related flags, but they are retained in this - API for consistency's sake. - - - The final argument to this function specifies the VFS to - use, which is largely (but not entirely!) meaningless in - the WASM environment. It should always be null or - undefined, and it is safe to elide that argument when - calling this function. - */ - sqlite3_open_v2: function(filename,dbPtrPtr,flags,vfsStr){}/*installed later*/, - /** - The sqlite3_prepare_v3() binding handles two different uses - with differing JS/WASM semantics: - - 1) sqlite3_prepare_v3(pDb, sqlString, -1, prepFlags, ppStmt [, null]) - - 2) sqlite3_prepare_v3(pDb, sqlPointer, sqlByteLen, prepFlags, ppStmt, sqlPointerToPointer) - - Note that the SQL length argument (the 3rd argument) must, for - usage (1), always be negative because it must be a byte length - and that value is expensive to calculate from JS (where only - the character length of strings is readily available). It is - retained in this API's interface for code/documentation - compatibility reasons but is currently _always_ ignored. With - usage (2), the 3rd argument is used as-is but is is still - critical that the C-style input string (2nd argument) be - terminated with a 0 byte. - - In usage (1), the 2nd argument must be of type string, - Uint8Array, or Int8Array (either of which is assumed to - hold SQL). If it is, this function assumes case (1) and - calls the underyling C function with the equivalent of: - - (pDb, sqlAsString, -1, prepFlags, ppStmt, null) - - The pzTail argument is ignored in this case because its result - is meaningless when a string-type value is passed through - (because the string goes through another level of internal - conversion for WASM's sake and the result pointer would refer - to that transient conversion's memory, not the passed-in - string). - - If the sql argument is not a string, it must be a _pointer_ to - a NUL-terminated string which was allocated in the WASM memory - (e.g. using cwapi.wasm.alloc() or equivalent). In that case, - the final argument may be 0/null/undefined or must be a pointer - to which the "tail" of the compiled SQL is written, as - documented for the C-side sqlite3_prepare_v3(). In case (2), - the underlying C function is called with the equivalent of: - - (pDb, sqlAsPointer, (sqlByteLen||-1), prepFlags, ppStmt, pzTail) - - It returns its result and compiled statement as documented in - the C API. Fetching the output pointers (5th and 6th - parameters) requires using capi.wasm.getMemValue() (or - equivalent) and the pzTail will point to an address relative to - the sqlAsPointer value. - - If passed an invalid 2nd argument type, this function will - return SQLITE_MISUSE but will unfortunately be able to return - any additional error information because we have no way to set - the db's error state such that this function could return a - non-0 integer and the client could call sqlite3_errcode() or - sqlite3_errmsg() to fetch it. See the RFE at: - - https://sqlite.org/forum/forumpost/f9eb79b11aefd4fc81d - - The alternative would be to throw an exception for that case, - but that would be in strong constrast to the rest of the - C-level API and seems likely to cause more confusion. - - Side-note: in the C API the function does not fail if provided - an empty string but its result output pointer will be NULL. - */ - sqlite3_prepare_v3: function(dbPtr, sql, sqlByteLen, prepFlags, - stmtPtrPtr, strPtrPtr){}/*installed later*/, - - /** - Equivalent to calling sqlite3_prapare_v3() with 0 as its 4th argument. - */ - sqlite3_prepare_v2: function(dbPtr, sql, sqlByteLen, stmtPtrPtr, - strPtrPtr){}/*installed later*/, - - /** - Various internal-use utilities are added here as needed. They - are bound to an object only so that we have access to them in - the differently-scoped steps of the API bootstrapping - process. At the end of the API setup process, this object gets - removed. - */ - util:{ - isInt32, isTypedArray, isBindableTypedArray, isSQLableTypedArray, - affirmBindableTypedArray, typedArrayToString - }, - - /** - Holds state which are specific to the WASM-related - infrastructure and glue code. It is not expected that client - code will normally need these, but they're exposed here in case - it does. These APIs are _not_ to be considered an - official/stable part of the sqlite3 WASM API. They may change - as the developers' experience suggests appropriate changes. - - Note that a number of members of this object are injected - dynamically after the api object is fully constructed, so - not all are documented inline here. - */ - wasm: { - //^^^ TODO?: move wasm from sqlite3.capi.wasm to sqlite3.wasm - /** - Emscripten APIs have a deep-seated assumption that all pointers - are 32 bits. We'll remain optimistic that that won't always be - the case and will use this constant in places where we might - otherwise use a hard-coded 4. - */ - ptrSizeof: config.wasmPtrSizeof || 4, - /** - The WASM IR (Intermediate Representation) value for - pointer-type values. It MUST refer to a value type of the - size described by this.ptrSizeof _or_ it may be any value - which ends in '*', which Emscripten's glue code internally - translates to i32. - */ - ptrIR: config.wasmPtrIR || "i32", - /** - True if BigInt support was enabled via (e.g.) the - Emscripten -sWASM_BIGINT flag, else false. When - enabled, certain 64-bit sqlite3 APIs are enabled which - are not otherwise enabled due to JS/WASM int64 - impedence mismatches. - */ - bigIntEnabled: !!config.bigIntEnabled, - /** - The symbols exported by the WASM environment. - */ - exports: config.exports - || toss("Missing API config.exports (WASM module exports)."), - - /** - When Emscripten compiles with `-sIMPORT_MEMORY`, it - initalizes the heap and imports it into wasm, as opposed to - the other way around. In this case, the memory is not - available via this.exports.memory. - */ - memory: config.memory || config.exports['memory'] - || toss("API config object requires a WebAssembly.Memory object", - "in either config.exports.memory (exported)", - "or config.memory (imported)."), - /* Many more wasm-related APIs get installed later on. */ - }/*wasm*/ - }/*capi*/; + /* Many more wasm-related APIs get installed later on. */ + }/*wasm*/); /** - capi.wasm.alloc()'s srcTypedArray.byteLength bytes, + wasm.alloc()'s srcTypedArray.byteLength bytes, populates them with the values from the source TypedArray, and returns the pointer to that memory. The returned pointer must eventually be passed to - capi.wasm.dealloc() to clean it up. + wasm.dealloc() to clean it up. As a special case, to avoid further special cases where this is used, if srcTypedArray.byteLength is 0, it @@ -387,25 +738,27 @@ self.sqlite3ApiBootstrap = function(config){ Int8Array types and will throw if srcTypedArray is of any other type. */ - capi.wasm.mallocFromTypedArray = function(srcTypedArray){ + wasm.allocFromTypedArray = function(srcTypedArray){ affirmBindableTypedArray(srcTypedArray); - const pRet = this.alloc(srcTypedArray.byteLength || 1); - this.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); + const pRet = wasm.alloc(srcTypedArray.byteLength || 1); + wasm.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); return pRet; - }.bind(capi.wasm); + }; const keyAlloc = config.allocExportName || 'malloc', keyDealloc = config.deallocExportName || 'free'; for(const key of [keyAlloc, keyDealloc]){ - const f = capi.wasm.exports[key]; - if(!(f instanceof Function)) toss("Missing required exports[",key,"] function."); + const f = wasm.exports[key]; + if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); } - capi.wasm.alloc = function(n){ - const m = this.exports[keyAlloc](n); + + wasm.alloc = function(n){ + const m = wasm.exports[keyAlloc](n); if(!m) throw new WasmAllocError("Failed to allocate "+n+" bytes."); return m; - }.bind(capi.wasm) - capi.wasm.dealloc = (m)=>capi.wasm.exports[keyDealloc](m); + }; + + wasm.dealloc = (m)=>wasm.exports[keyDealloc](m); /** Reports info about compile-time options using @@ -436,7 +789,7 @@ self.sqlite3ApiBootstrap = function(config){ "SQLITE_" prefix. When it returns an object of all options, the prefix is elided. */ - capi.wasm.compileOptionUsed = function f(optName){ + wasm.compileOptionUsed = function f(optName){ if(!arguments.length){ if(f._result) return f._result; else if(!f._opt){ @@ -472,26 +825,39 @@ self.sqlite3ApiBootstrap = function(config){ ) ? !!capi.sqlite3_compileoption_used(optName) : false; }/*compileOptionUsed()*/; - capi.wasm.bindingSignatures = [ - /** - Signatures for the WASM-exported C-side functions. Each entry - is an array with 2+ elements: + /** + Signatures for the WASM-exported C-side functions. Each entry + is an array with 2+ elements: - ["c-side name", - "result type" (capi.wasm.xWrap() syntax), - [arg types in xWrap() syntax] - // ^^^ this needn't strictly be an array: it can be subsequent - // elements instead: [x,y,z] is equivalent to x,y,z - ] - */ + [ "c-side name", + "result type" (wasm.xWrap() syntax), + [arg types in xWrap() syntax] + // ^^^ this needn't strictly be an array: it can be subsequent + // elements instead: [x,y,z] is equivalent to x,y,z + ] + + Note that support for the API-specific data types in the + result/argument type strings gets plugged in at a later phase in + the API initialization process. + */ + wasm.bindingSignatures = [ // Please keep these sorted by function name! - ["sqlite3_bind_blob","int", "sqlite3_stmt*", "int", "*", "int", "*"], + ["sqlite3_aggregate_context","void*", "sqlite3_context*", "int"], + ["sqlite3_bind_blob","int", "sqlite3_stmt*", "int", "*", "int", "*" + /* TODO: we should arguably write a custom wrapper which knows + how to handle Blob, TypedArrays, and JS strings. */ + ], ["sqlite3_bind_double","int", "sqlite3_stmt*", "int", "f64"], ["sqlite3_bind_int","int", "sqlite3_stmt*", "int", "int"], ["sqlite3_bind_null",undefined, "sqlite3_stmt*", "int"], ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], - ["sqlite3_bind_text","int", "sqlite3_stmt*", "int", "string", "int", "int"], + ["sqlite3_bind_text","int", "sqlite3_stmt*", "int", "string", "int", "int" + /* We should arguably create a hand-written binding of + bind_text() which does more flexible text conversion, along + the lines of sqlite3_prepare_v3(). The slightly problematic + part is the final argument (text destructor). */ + ], ["sqlite3_close_v2", "int", "sqlite3*"], ["sqlite3_changes", "int", "sqlite3*"], ["sqlite3_clear_bindings","int", "sqlite3_stmt*"], @@ -505,33 +871,46 @@ self.sqlite3ApiBootstrap = function(config){ ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], ["sqlite3_compileoption_get", "string", "int"], ["sqlite3_compileoption_used", "int", "string"], - ["sqlite3_create_function_v2", "int", - "sqlite3*", "string", "int", "int", "*", "*", "*", "*", "*"], + /* sqlite3_create_function(), sqlite3_create_function_v2(), and + sqlite3_create_window_function() use hand-written bindings to + simplify handling of their function-type arguments. */ ["sqlite3_data_count", "int", "sqlite3_stmt*"], ["sqlite3_db_filename", "string", "sqlite3*", "string"], + ["sqlite3_db_handle", "sqlite3*", "sqlite3_stmt*"], ["sqlite3_db_name", "string", "sqlite3*", "int"], + ["sqlite3_deserialize", "int", "sqlite3*", "string", "*", "i64", "i64", "int"] + /* Careful! Short version: de/serialize() are problematic because they + might use a different allocator than the user for managing the + deserialized block. de/serialize() are ONLY safe to use with + sqlite3_malloc(), sqlite3_free(), and its 64-bit variants. */, ["sqlite3_errmsg", "string", "sqlite3*"], ["sqlite3_error_offset", "int", "sqlite3*"], ["sqlite3_errstr", "string", "int"], - //["sqlite3_exec", "int", "sqlite3*", "string", "*", "*", "**"], - // ^^^ TODO: we need a wrapper to support passing a function pointer or a function - // for the callback. + /*["sqlite3_exec", "int", "sqlite3*", "string", "*", "*", "**" + Handled seperately to perform translation of the callback + into a WASM-usable one. ],*/ ["sqlite3_expanded_sql", "string", "sqlite3_stmt*"], ["sqlite3_extended_errcode", "int", "sqlite3*"], ["sqlite3_extended_result_codes", "int", "sqlite3*", "int"], + ["sqlite3_file_control", "int", "sqlite3*", "string", "int", "*"], ["sqlite3_finalize", "int", "sqlite3_stmt*"], + ["sqlite3_free", undefined,"*"], ["sqlite3_initialize", undefined], - ["sqlite3_interrupt", undefined, "sqlite3*" - /* ^^^ we cannot actually currently support this because JS is + /*["sqlite3_interrupt", undefined, "sqlite3*" + ^^^ we cannot actually currently support this because JS is single-threaded and we don't have a portable way to access a DB - from 2 SharedWorkers concurrently. */], + from 2 SharedWorkers concurrently. ],*/ ["sqlite3_libversion", "string"], ["sqlite3_libversion_number", "int"], + ["sqlite3_malloc", "*","int"], ["sqlite3_open", "int", "string", "*"], ["sqlite3_open_v2", "int", "string", "*", "int", "string"], /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled separately due to us requiring two different sets of semantics for those, depending on how their SQL argument is provided. */ + /* sqlite3_randomness() uses a hand-written wrapper to extend + the range of supported argument types. */ + ["sqlite3_realloc", "*","*","int"], ["sqlite3_reset", "int", "sqlite3_stmt*"], ["sqlite3_result_blob",undefined, "*", "*", "int", "*"], ["sqlite3_result_double",undefined, "*", "f64"], @@ -542,26 +921,35 @@ self.sqlite3ApiBootstrap = function(config){ ["sqlite3_result_int",undefined, "*", "int"], ["sqlite3_result_null",undefined, "*"], ["sqlite3_result_text",undefined, "*", "string", "int", "*"], + ["sqlite3_serialize","*", "sqlite3*", "string", "*", "int"], + ["sqlite3_shutdown", undefined], ["sqlite3_sourceid", "string"], ["sqlite3_sql", "string", "sqlite3_stmt*"], ["sqlite3_step", "int", "sqlite3_stmt*"], ["sqlite3_strglob", "int", "string","string"], ["sqlite3_strlike", "int", "string","string","int"], + ["sqlite3_trace_v2", "int", "sqlite3*", "int", "*", "*"], ["sqlite3_total_changes", "int", "sqlite3*"], - ["sqlite3_value_blob", "*", "*"], - ["sqlite3_value_bytes","int", "*"], - ["sqlite3_value_double","f64", "*"], - ["sqlite3_value_text", "string", "*"], - ["sqlite3_value_type", "int", "*"], + ["sqlite3_uri_boolean", "int", "string", "string", "int"], + ["sqlite3_uri_key", "string", "string", "int"], + ["sqlite3_uri_parameter", "string", "string", "string"], + ["sqlite3_user_data","void*", "sqlite3_context*"], + ["sqlite3_value_blob", "*", "sqlite3_value*"], + ["sqlite3_value_bytes","int", "sqlite3_value*"], + ["sqlite3_value_double","f64", "sqlite3_value*"], + ["sqlite3_value_int","int", "sqlite3_value*"], + ["sqlite3_value_text", "string", "sqlite3_value*"], + ["sqlite3_value_type", "int", "sqlite3_value*"], ["sqlite3_vfs_find", "*", "string"], - ["sqlite3_vfs_register", "int", "*", "int"] - ]/*capi.wasm.bindingSignatures*/; + ["sqlite3_vfs_register", "int", "sqlite3_vfs*", "int"], + ["sqlite3_vfs_unregister", "int", "sqlite3_vfs*"] + ]/*wasm.bindingSignatures*/; - if(false && capi.wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ + if(false && wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ /* ^^^ "the problem" is that this is an option feature and the build-time function-export list does not currently take optional features into account. */ - capi.wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); + wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); } /** @@ -569,25 +957,607 @@ self.sqlite3ApiBootstrap = function(config){ the others because we need to conditionally bind them or apply dummy impls, depending on the capabilities of the environment. */ - capi.wasm.bindingSignatures.int64 = [ - ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], - ["sqlite3_changes64","i64", ["sqlite3*"]], - ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], - ["sqlite3_total_changes64", "i64", ["sqlite3*"]] + wasm.bindingSignatures.int64 = [ + ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], + ["sqlite3_changes64","i64", ["sqlite3*"]], + ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], + ["sqlite3_malloc64", "*","i64"], + ["sqlite3_msize", "i64", "*"], + ["sqlite3_realloc64", "*","*", "i64"], + ["sqlite3_result_int64",undefined, "*", "i64"], + ["sqlite3_total_changes64", "i64", ["sqlite3*"]], + ["sqlite3_uri_int64", "i64", ["string", "string", "i64"]], + ["sqlite3_value_int64","i64", "sqlite3_value*"], ]; - /* The remainder of the API will be set up in later steps. */ - return { - capi, - postInit: [ - /* some pieces of the API may install functions into this array, - and each such function will be called, passed (self,sqlite3), - at the very end of the API load/init process, where self is - the current global object and sqlite3 is the object returned - from sqlite3ApiBootstrap(). This array will be removed at the - end of the API setup process. */], - /** Config is needed downstream for gluing pieces together. It - will be removed at the end of the API setup process. */ - config + /** + Functions which are intended solely for API-internal use by the + WASM components, not client code. These get installed into + sqlite3.wasm. + */ + wasm.bindingSignatures.wasm = [ + ["sqlite3_wasm_db_reset", "int", "sqlite3*"], + ["sqlite3_wasm_db_vfs", "sqlite3_vfs*", "sqlite3*","string"], + ["sqlite3_wasm_vfs_create_file", "int", + "sqlite3_vfs*","string","*", "int"], + ["sqlite3_wasm_vfs_unlink", "int", "sqlite3_vfs*","string"] + ]; + + + /** + sqlite3.wasm.pstack (pseudo-stack) holds a special-case + stack-style allocator intended only for use with _small_ data of + not more than (in total) a few kb in size, managed as if it were + stack-based. + + It has only a single intended usage: + + ``` + const stackPos = pstack.pointer; + try{ + const ptr = pstack.alloc(8); + // ==> pstack.pointer === ptr + const otherPtr = pstack.alloc(8); + // ==> pstack.pointer === otherPtr + ... + }finally{ + pstack.restore(stackPos); + // ==> pstack.pointer === stackPos + } + ``` + + This allocator is much faster than a general-purpose one but is + limited to usage patterns like the one shown above. + + It operates from a static range of memory which lives outside of + space managed by Emscripten's stack-management, so does not + collide with Emscripten-provided stack allocation APIs. The + memory lives in the WASM heap and can be used with routines such + as wasm.setMemValue() and any wasm.heap8u().slice(). + */ + wasm.pstack = Object.assign(Object.create(null),{ + /** + Sets the current pstack position to the given pointer. Results + are undefined if the passed-in value did not come from + this.pointer. + */ + restore: wasm.exports.sqlite3_wasm_pstack_restore, + /** + Attempts to allocate the given number of bytes from the + pstack. On success, it zeroes out a block of memory of the + given size, adjusts the pstack pointer, and returns a pointer + to the memory. On error, returns throws a WasmAllocError. The + memory must eventually be released using restore(). + + This method always adjusts the given value to be a multiple + of 8 bytes because failing to do so can lead to incorrect + results when reading and writing 64-bit values from/to the WASM + heap. Similarly, the returned address is always 8-byte aligned. + */ + alloc: (n)=>{ + return wasm.exports.sqlite3_wasm_pstack_alloc(n) + || WasmAllocError.toss("Could not allocate",n, + "bytes from the pstack."); + }, + /** + alloc()'s n chunks, each sz bytes, as a single memory block and + returns the addresses as an array of n element, each holding + the address of one chunk. + + Throws a WasmAllocError if allocation fails. + + Example: + + ``` + const [p1, p2, p3] = wasm.pstack.allocChunks(3,4); + ``` + */ + allocChunks: (n,sz)=>{ + const mem = wasm.pstack.alloc(n * sz); + const rc = []; + let i = 0, offset = 0; + for(; i < n; offset = (sz * ++i)){ + rc.push(mem + offset); + } + return rc; + }, + /** + A convenience wrapper for allocChunks() which sizes each chunk + as either 8 bytes (safePtrSize is truthy) or wasm.ptrSizeof (if + safePtrSize is falsy). + + How it returns its result differs depending on its first + argument: if it's 1, it returns a single pointer value. If it's + more than 1, it returns the same as allocChunks(). + + When a returned pointers will refer to a 64-bit value, e.g. a + double or int64, and that value must be written or fetched, + e.g. using wasm.setMemValue() or wasm.getMemValue(), it is + important that the pointer in question be aligned to an 8-byte + boundary or else it will not be fetched or written properly and + will corrupt or read neighboring memory. + + However, when all pointers involved point to "small" data, it + is safe to pass a falsy value to save a tiny bit of memory. + */ + allocPtr: (n=1,safePtrSize=true)=>{ + return 1===n + ? wasm.pstack.alloc(safePtrSize ? 8 : wasm.ptrSizeof) + : wasm.pstack.allocChunks(n, safePtrSize ? 8 : wasm.ptrSizeof); + } + })/*wasm.pstack*/; + Object.defineProperties(wasm.pstack, { + /** + sqlite3.wasm.pstack.pointer resolves to the current pstack + position pointer. This value is intended _only_ to be saved + for passing to restore(). Writing to this memory, without + first reserving it via wasm.pstack.alloc() and friends, leads + to undefined results. + */ + pointer: { + configurable: false, iterable: true, writeable: false, + get: wasm.exports.sqlite3_wasm_pstack_ptr + //Whether or not a setter as an alternative to restore() is + //clearer or would just lead to confusion is unclear. + //set: wasm.exports.sqlite3_wasm_pstack_restore + }, + /** + sqlite3.wasm.pstack.quota to the total number of bytes + available in the pstack, including any space which is currently + allocated. This value is a compile-time constant. + */ + quota: { + configurable: false, iterable: true, writeable: false, + get: wasm.exports.sqlite3_wasm_pstack_quota + }, + /** + sqlite3.wasm.pstack.remaining resolves to the amount of space + remaining in the pstack. + */ + remaining: { + configurable: false, iterable: true, writeable: false, + get: wasm.exports.sqlite3_wasm_pstack_remaining + } + })/*wasm.pstack properties*/; + + capi.sqlite3_randomness = (...args)=>{ + if(1===args.length && util.isTypedArray(args[0]) + && 1===args[0].BYTES_PER_ELEMENT){ + const ta = args[0]; + if(0===ta.byteLength){ + wasm.exports.sqlite3_randomness(0,0); + return ta; + } + const stack = wasm.pstack.pointer; + try { + let n = ta.byteLength, offset = 0; + const r = wasm.exports.sqlite3_randomness; + const heap = wasm.heap8u(); + const nAlloc = n < 512 ? n : 512; + const ptr = wasm.pstack.alloc(nAlloc); + do{ + const j = (n>nAlloc ? nAlloc : n); + r(j, ptr); + ta.set(typedArrayPart(heap, ptr, ptr+j), offset); + n -= j; + offset += j; + } while(n > 0); + }catch(e){ + console.error("Highly unexpected (and ignored!) "+ + "exception in sqlite3_randomness():",e); + }finally{ + wasm.pstack.restore(stack); + } + return ta; + } + wasm.exports.sqlite3_randomness(...args); }; + + /** State for sqlite3_wasmfs_opfs_dir(). */ + let __wasmfsOpfsDir = undefined; + /** + If the wasm environment has a WASMFS/OPFS-backed persistent + storage directory, its path is returned by this function. If it + does not then it returns "" (noting that "" is a falsy value). + + The first time this is called, this function inspects the current + environment to determine whether persistence support is available + and, if it is, enables it (if needed). + + This function currently only recognizes the WASMFS/OPFS storage + combination and its path refers to storage rooted in the + Emscripten-managed virtual filesystem. + */ + capi.sqlite3_wasmfs_opfs_dir = function(){ + if(undefined !== __wasmfsOpfsDir) return __wasmfsOpfsDir; + // If we have no OPFS, there is no persistent dir + const pdir = config.wasmfsOpfsDir; + if(!pdir + || !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return __wasmfsOpfsDir = ""; + } + try{ + if(pdir && 0===wasm.xCallWrapped( + 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir + )){ + return __wasmfsOpfsDir = pdir; + }else{ + return __wasmfsOpfsDir = ""; + } + }catch(e){ + // sqlite3_wasm_init_wasmfs() is not available + return __wasmfsOpfsDir = ""; + } + }; + + /** + Experimental and subject to change or removal. + + Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a + non-empty string and the given name starts with (that string + + '/'), else returns false. + */ + capi.sqlite3_wasmfs_filename_is_persistent = function(name){ + const p = capi.sqlite3_wasmfs_opfs_dir(); + return (p && name) ? name.startsWith(p+'/') : false; + }; + + // This bit is highly arguable and is incompatible with the fiddle shell. + if(false && 0===wasm.exports.sqlite3_vfs_find(0)){ + /* Assume that sqlite3_initialize() has not yet been called. + This will be the case in an SQLITE_OS_KV build. */ + wasm.exports.sqlite3_initialize(); + } + + /** + Given an `sqlite3*`, an sqlite3_vfs name, and an optional db name + (defaulting to "main"), returns a truthy value (see below) if + that db uses that VFS, else returns false. If pDb is falsy then + the 3rd argument is ignored and this function returns a truthy + value if the default VFS name matches that of the 2nd + argument. Results are undefined if pDb is truthy but refers to an + invalid pointer. The 3rd argument specifies the database name of + the given database connection to check, defaulting to the main + db. + + The 2nd and 3rd arguments may either be a JS string or a WASM + C-string. If the 2nd argument is a NULL WASM pointer, the default + VFS is assumed. If the 3rd is a NULL WASM pointer, "main" is + assumed. + + The truthy value it returns is a pointer to the `sqlite3_vfs` + object. + + To permit safe use of this function from APIs which may be called + via the C stack (like SQL UDFs), this function does not throw: if + bad arguments cause a conversion error when passing into + wasm-space, false is returned. + */ + capi.sqlite3_js_db_uses_vfs = function(pDb,vfsName,dbName=0){ + try{ + const pK = capi.sqlite3_vfs_find(vfsName); + if(!pK) return false; + else if(!pDb){ + return pK===capi.sqlite3_vfs_find(0) ? pK : false; + }else{ + return pK===capi.sqlite3_js_db_vfs(pDb,dbName) ? pK : false; + } + }catch(e){ + /* Ignore - probably bad args to a wasm-bound function. */ + return false; + } + }; + + /** + Returns an array of the names of all currently-registered sqlite3 + VFSes. + */ + capi.sqlite3_js_vfs_list = function(){ + const rc = []; + let pVfs = capi.sqlite3_vfs_find(0); + while(pVfs){ + const oVfs = new capi.sqlite3_vfs(pVfs); + rc.push(wasm.cstringToJs(oVfs.$zName)); + pVfs = oVfs.$pNext; + oVfs.dispose(); + } + return rc; + }; + + /** + Serializes the given `sqlite3*` pointer to a Uint8Array, as per + sqlite3_serialize(). On success it returns a Uint8Array. On + error it throws with a description of the problem. + */ + capi.sqlite3_js_db_export = function(pDb){ + if(!pDb) toss3('Invalid sqlite3* argument.'); + if(!wasm.bigIntEnabled) toss3('BigInt64 support is not enabled.'); + const stack = wasm.pstack.pointer; + let pOut; + try{ + const pSize = wasm.pstack.alloc(8/*i64*/ + wasm.ptrSizeof); + const ppOut = pSize + 8; + /** + Maintenance reminder, since this cost a full hour of grief + and confusion: if the order of pSize/ppOut are reversed in + that memory block, fetching the value of pSize after the + export reads a garbage size because it's not on an 8-byte + memory boundary! + */ + let rc = wasm.exports.sqlite3_wasm_db_serialize( + pDb, ppOut, pSize, 0 + ); + if(rc){ + toss3("Database serialization failed with code", + sqlite3.capi.sqlite3_js_rc_str(rc)); + } + pOut = wasm.getPtrValue(ppOut); + const nOut = wasm.getMemValue(pSize, 'i64'); + rc = nOut + ? wasm.heap8u().slice(pOut, pOut + Number(nOut)) + : new Uint8Array(); + return rc; + }finally{ + if(pOut) wasm.exports.sqlite3_free(pOut); + wasm.pstack.restore(stack); + } + }; + + /** + Given a `sqlite3*` and a database name (JS string or WASM + C-string pointer, which may be 0), returns a pointer to the + sqlite3_vfs responsible for it. If the given db name is null/0, + or not provided, then "main" is assumed. + */ + capi.sqlite3_js_db_vfs = + (dbPointer, dbName=0)=>wasm.sqlite3_wasm_db_vfs(dbPointer, dbName); + + /** + A thin wrapper around capi.sqlite3_aggregate_context() which + behaves the same except that it throws a WasmAllocError if that + function returns 0. As a special case, if n is falsy it does + _not_ throw if that function returns 0. That special case is + intended for use with xFinal() implementations. + */ + capi.sqlite3_js_aggregate_context = (pCtx, n)=>{ + return capi.sqlite3_aggregate_context(pCtx, n) + || (n ? WasmAllocError.toss("Cannot allocate",n, + "bytes for sqlite3_aggregate_context()") + : 0); + }; + + if( util.isUIThread() ){ + /* Features specific to the main window thread... */ + + /** + Internal helper for sqlite3_js_kvvfs_clear() and friends. + Its argument should be one of ('local','session',""). + */ + const __kvvfsInfo = function(which){ + const rc = Object.create(null); + rc.prefix = 'kvvfs-'+which; + rc.stores = []; + if('session'===which || ""===which) rc.stores.push(self.sessionStorage); + if('local'===which || ""===which) rc.stores.push(self.localStorage); + return rc; + }; + + /** + Clears all storage used by the kvvfs DB backend, deleting any + DB(s) stored there. Its argument must be either 'session', + 'local', or "". In the first two cases, only sessionStorage + resp. localStorage is cleared. If it's an empty string (the + default) then both are cleared. Only storage keys which match + the pattern used by kvvfs are cleared: any other client-side + data are retained. + + This function is only available in the main window thread. + + Returns the number of entries cleared. + */ + capi.sqlite3_js_kvvfs_clear = function(which=""){ + let rc = 0; + const kvinfo = __kvvfsInfo(which); + kvinfo.stores.forEach((s)=>{ + const toRm = [] /* keys to remove */; + let i; + for( i = 0; i < s.length; ++i ){ + const k = s.key(i); + if(k.startsWith(kvinfo.prefix)) toRm.push(k); + } + toRm.forEach((kk)=>s.removeItem(kk)); + rc += toRm.length; + }); + return rc; + }; + + /** + This routine guesses the approximate amount of + window.localStorage and/or window.sessionStorage in use by the + kvvfs database backend. Its argument must be one of + ('session', 'local', ""). In the first two cases, only + sessionStorage resp. localStorage is counted. If it's an empty + string (the default) then both are counted. Only storage keys + which match the pattern used by kvvfs are counted. The returned + value is the "length" value of every matching key and value, + noting that JavaScript stores each character in 2 bytes. + + Note that the returned size is not authoritative from the + perspective of how much data can fit into localStorage and + sessionStorage, as the precise algorithms for determining + those limits are unspecified and may include per-entry + overhead invisible to clients. + */ + capi.sqlite3_js_kvvfs_size = function(which=""){ + let sz = 0; + const kvinfo = __kvvfsInfo(which); + kvinfo.stores.forEach((s)=>{ + let i; + for(i = 0; i < s.length; ++i){ + const k = s.key(i); + if(k.startsWith(kvinfo.prefix)){ + sz += k.length; + sz += s.getItem(k).length; + } + } + }); + return sz * 2 /* because JS uses 2-byte char encoding */; + }; + + }/* main-window-only bits */ + + + /* The remainder of the API will be set up in later steps. */ + const sqlite3 = { + WasmAllocError: WasmAllocError, + SQLite3Error: SQLite3Error, + capi, + util, + wasm, + config, + /** + Holds the version info of the sqlite3 source tree from which + the generated sqlite3-api.js gets built. Note that its version + may well differ from that reported by sqlite3_libversion(), but + that should be considered a source file mismatch, as the JS and + WASM files are intended to be built and distributed together. + + This object is initially a placeholder which gets replaced by a + build-generated object. + */ + version: Object.create(null), + /** + Performs any optional asynchronous library-level initialization + which might be required. This function returns a Promise which + resolves to the sqlite3 namespace object. Any error in the + async init will be fatal to the init as a whole, but init + routines are themselves welcome to install dummy catch() + handlers which are not fatal if their failure should be + considered non-fatal. If called more than once, the second and + subsequent calls are no-ops which return a pre-resolved + Promise. + + Ideally this function is called as part of the Promise chain + which handles the loading and bootstrapping of the API. If not + then it must be called by client-level code, which must not use + the library until the returned promise resolves. + + Bug: if called while a prior call is still resolving, the 2nd + call will resolve prematurely, before the 1st call has finished + resolving. The current build setup precludes that possibility, + so it's only a hypothetical problem if/when this function + ever needs to be invoked by clients. + + In Emscripten-based builds, this function is called + automatically and deleted from this object. + */ + asyncPostInit: async function(){ + let lip = sqlite3ApiBootstrap.initializersAsync; + delete sqlite3ApiBootstrap.initializersAsync; + if(!lip || !lip.length) return Promise.resolve(sqlite3); + // Is it okay to resolve these in parallel or do we need them + // to resolve in order? We currently only have 1, so it + // makes no difference. + lip = lip.map((f)=>{ + const p = (f instanceof Promise) ? f : f(sqlite3); + return p.catch((e)=>{ + console.error("an async sqlite3 initializer failed:",e); + throw e; + }); + }); + //let p = lip.shift(); + //while(lip.length) p = p.then(lip.shift()); + //return p.then(()=>sqlite3); + return Promise.all(lip).then(()=>sqlite3); + }, + /** + scriptInfo ideally gets injected into this object by the + infrastructure which assembles the JS/WASM module. It contains + state which must be collected before sqlite3ApiBootstrap() can + be declared. It is not necessarily available to any + sqlite3ApiBootstrap.initializers but "should" be in place (if + it's added at all) by the time that + sqlite3ApiBootstrap.initializersAsync is processed. + + This state is not part of the public API, only intended for use + with the sqlite3 API bootstrapping and wasm-loading process. + */ + scriptInfo: undefined + }; + try{ + sqlite3ApiBootstrap.initializers.forEach((f)=>{ + f(sqlite3); + }); + }catch(e){ + /* If we don't report this here, it can get completely swallowed + up and disappear into the abyss of Promises and Workers. */ + console.error("sqlite3 bootstrap initializer threw:",e); + throw e; + } + delete sqlite3ApiBootstrap.initializers; + sqlite3ApiBootstrap.sqlite3 = sqlite3; + return sqlite3; }/*sqlite3ApiBootstrap()*/; +/** + self.sqlite3ApiBootstrap.initializers is an internal detail used by + the various pieces of the sqlite3 API's amalgamation process. It + must not be modified by client code except when plugging such code + into the amalgamation process. + + Each component of the amalgamation is expected to append a function + to this array. When sqlite3ApiBootstrap() is called for the first + time, each such function will be called (in their appended order) + and passed the sqlite3 namespace object, into which they can install + their features (noting that most will also require that certain + features alread have been installed). At the end of that process, + this array is deleted. + + Note that the order of insertion into this array is significant for + some pieces. e.g. sqlite3.capi and sqlite3.wasm cannot be fully + utilized until the whwasmutil.js part is plugged in via + sqlite3-api-glue.js. +*/ +self.sqlite3ApiBootstrap.initializers = []; +/** + self.sqlite3ApiBootstrap.initializersAsync is an internal detail + used by the sqlite3 API's amalgamation process. It must not be + modified by client code except when plugging such code into the + amalgamation process. + + The counterpart of self.sqlite3ApiBootstrap.initializers, + specifically for initializers which are asynchronous. All entries in + this list must be either async functions, non-async functions which + return a Promise, or a Promise. Each function in the list is called + with the sqlite3 ojbect as its only argument. + + The resolved value of any Promise is ignored and rejection will kill + the asyncPostInit() process (at an indeterminate point because all + of them are run asynchronously in parallel). + + This list is not processed until the client calls + sqlite3.asyncPostInit(). This means, for example, that intializers + added to self.sqlite3ApiBootstrap.initializers may push entries to + this list. +*/ +self.sqlite3ApiBootstrap.initializersAsync = []; +/** + Client code may assign sqlite3ApiBootstrap.defaultConfig an + object-type value before calling sqlite3ApiBootstrap() (without + arguments) in order to tell that call to use this object as its + default config value. The intention of this is to provide + downstream clients with a reasonably flexible approach for plugging in + an environment-suitable configuration without having to define a new + global-scope symbol. +*/ +self.sqlite3ApiBootstrap.defaultConfig = Object.create(null); +/** + Placeholder: gets installed by the first call to + self.sqlite3ApiBootstrap(). However, it is recommended that the + caller of sqlite3ApiBootstrap() capture its return value and delete + self.sqlite3ApiBootstrap after calling it. It returns the same + value which will be stored here. +*/ +self.sqlite3ApiBootstrap.sqlite3 = undefined; + diff --git a/ext/wasm/api/sqlite3-api-worker.js b/ext/wasm/api/sqlite3-api-worker.js deleted file mode 100644 index 95b27b21ec..0000000000 --- a/ext/wasm/api/sqlite3-api-worker.js +++ /dev/null @@ -1,420 +0,0 @@ -/* - 2022-07-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file implements a Worker-based wrapper around SQLite3 OO API - #1. - - In order to permit this API to be loaded in worker threads without - automatically registering onmessage handlers, initializing the - worker API requires calling initWorkerAPI(). If this function - is called from a non-worker thread then it throws an exception. - - When initialized, it installs message listeners to receive messages - from the main thread and then it posts a message in the form: - - ``` - {type:'sqlite3-api',data:'worker-ready'} - ``` - - This file requires that the core C-style sqlite3 API and OO API #1 - have been loaded and that self.sqlite3 contains both, - as documented for those APIs. -*/ -self.sqlite3.initWorkerAPI = function(){ - 'use strict'; - /** - UNDER CONSTRUCTION - - We need an API which can proxy the DB API via a Worker message - interface. The primary quirky factor in such an API is that we - cannot pass callback functions between the window thread and a - worker thread, so we have to receive all db results via - asynchronous message-passing. That requires an asychronous API - with a distinctly different shape that the main OO API. - - Certain important considerations here include: - - - Support only one db connection or multiple? The former is far - easier, but there's always going to be a user out there who wants - to juggle six database handles at once. Do we add that complexity - or tell such users to write their own code using the provided - lower-level APIs? - - - Fetching multiple results: do we pass them on as a series of - messages, with start/end messages on either end, or do we collect - all results and bundle them back in a single message? The former - is, generically speaking, more memory-efficient but the latter - far easier to implement in this environment. The latter is - untennable for large data sets. Despite a web page hypothetically - being a relatively limited environment, there will always be - those users who feel that they should/need to be able to work - with multi-hundred-meg (or larger) blobs, and passing around - arrays of those may quickly exhaust the JS engine's memory. - - TODOs include, but are not limited to: - - - The ability to manage multiple DB handles. This can - potentially be done via a simple mapping of DB.filename or - DB.pointer (`sqlite3*` handle) to DB objects. The open() - interface would need to provide an ID (probably DB.pointer) back - to the user which can optionally be passed as an argument to - the other APIs (they'd default to the first-opened DB, for - ease of use). Client-side usability of this feature would - benefit from making another wrapper class (or a singleton) - available to the main thread, with that object proxying all(?) - communication with the worker. - - - Revisit how virtual files are managed. We currently delete DBs - from the virtual filesystem when we close them, for the sake of - saving memory (the VFS lives in RAM). Supporting multiple DBs may - require that we give up that habit. Similarly, fully supporting - ATTACH, where a user can upload multiple DBs and ATTACH them, - also requires the that we manage the VFS entries better. - */ - const toss = (...args)=>{throw new Error(args.join(' '))}; - if('function' !== typeof importScripts){ - toss("Cannot initalize the sqlite3 worker API in the main thread."); - } - const self = this.self; - const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); - const SQLite3 = sqlite3.oo1 || toss("Missing this.sqlite3.oo1 OO API."); - const DB = SQLite3.DB; - - /** - Returns the app-wide unique ID for the given db, creating one if - needed. - */ - const getDbId = function(db){ - let id = wState.idMap.get(db); - if(id) return id; - id = 'db#'+(++wState.idSeq)+'@'+db.pointer; - /** ^^^ can't simply use db.pointer b/c closing/opening may re-use - the same address, which could map pending messages to a wrong - instance. */ - wState.idMap.set(db, id); - return id; - }; - - /** - Helper for managing Worker-level state. - */ - const wState = { - defaultDb: undefined, - idSeq: 0, - idMap: new WeakMap, - open: function(arg){ - // TODO: if arg is a filename, look for a db in this.dbs with the - // same filename and close/reopen it (or just pass it back as is?). - if(!arg && this.defaultDb) return this.defaultDb; - //???if(this.defaultDb) this.defaultDb.close(); - let db; - db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); - this.dbs[getDbId(db)] = db; - if(!this.defaultDb) this.defaultDb = db; - return db; - }, - close: function(db,alsoUnlink){ - if(db){ - delete this.dbs[getDbId(db)]; - db.close(alsoUnlink); - if(db===this.defaultDb) this.defaultDb = undefined; - } - }, - post: function(type,data,xferList){ - if(xferList){ - self.postMessage({type, data},xferList); - xferList.length = 0; - }else{ - self.postMessage({type, data}); - } - }, - /** Map of DB IDs to DBs. */ - dbs: Object.create(null), - getDb: function(id,require=true){ - return this.dbs[id] - || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); - } - }; - - /** Throws if the given db is falsy or not opened. */ - const affirmDbOpen = function(db = wState.defaultDb){ - return (db && db.pointer) ? db : toss("DB is not opened."); - }; - - /** Extract dbId from the given message payload. */ - const getMsgDb = function(msgData,affirmExists=true){ - const db = wState.getDb(msgData.dbId,false) || wState.defaultDb; - return affirmExists ? affirmDbOpen(db) : db; - }; - - const getDefaultDbId = function(){ - return wState.defaultDb && getDbId(wState.defaultDb); - }; - - /** - A level of "organizational abstraction" for the Worker - API. Each method in this object must map directly to a Worker - message type key. The onmessage() dispatcher attempts to - dispatch all inbound messages to a method of this object, - passing it the event.data part of the inbound event object. All - methods must return a plain Object containing any response - state, which the dispatcher may amend. All methods must throw - on error. - */ - const wMsgHandler = { - xfer: [/*Temp holder for "transferable" postMessage() state.*/], - /** - Proxy for DB.exec() which expects a single argument of type - string (SQL to execute) or an options object in the form - expected by exec(). The notable differences from exec() - include: - - - The default value for options.rowMode is 'array' because - the normal default cannot cross the window/Worker boundary. - - - A function-type options.callback property cannot cross - the window/Worker boundary, so is not useful here. If - options.callback is a string then it is assumed to be a - message type key, in which case a callback function will be - applied which posts each row result via: - - postMessage({type: thatKeyType, data: theRow}) - - And, at the end of the result set (whether or not any - result rows were produced), it will post an identical - message with data:null to alert the caller than the result - set is completed. - - The callback proxy must not recurse into this interface, or - results are undefined. (It hypothetically cannot recurse - because an exec() call will be tying up the Worker thread, - causing any recursion attempt to wait until the first - exec() is completed.) - - The response is the input options object (or a synthesized - one if passed only a string), noting that - options.resultRows and options.columnNames may be populated - by the call to exec(). - - This opens/creates the Worker's db if needed. - */ - exec: function(ev){ - const opt = ( - 'string'===typeof ev.data - ) ? {sql: ev.data} : (ev.data || Object.create(null)); - if(undefined===opt.rowMode){ - /* Since the default rowMode of 'stmt' is not useful - for the Worker interface, we'll default to - something else. */ - opt.rowMode = 'array'; - }else if('stmt'===opt.rowMode){ - toss("Invalid rowMode for exec(): stmt mode", - "does not work in the Worker API."); - } - const db = getMsgDb(ev); - if(opt.callback || Array.isArray(opt.resultRows)){ - // Part of a copy-avoidance optimization for blobs - db._blobXfer = this.xfer; - } - const callbackMsgType = opt.callback; - if('string' === typeof callbackMsgType){ - /* Treat this as a worker message type and post each - row as a message of that type. */ - const that = this; - opt.callback = - (row)=>wState.post(callbackMsgType,row,this.xfer); - } - try { - db.exec(opt); - if(opt.callback instanceof Function){ - opt.callback = callbackMsgType; - wState.post(callbackMsgType, null); - } - }/*catch(e){ - console.warn("Worker is propagating:",e);throw e; - }*/finally{ - delete db._blobXfer; - if(opt.callback){ - opt.callback = callbackMsgType; - } - } - return opt; - }/*exec()*/, - /** - TO(re)DO, once we can abstract away access to the - JS environment's virtual filesystem. Currently this - always throws. - - Response is (should be) an object: - - { - buffer: Uint8Array (db file contents), - filename: the current db filename, - mimetype: 'application/x-sqlite3' - } - - TODO is to determine how/whether this feature can support - exports of ":memory:" and "" (temp file) DBs. The latter is - ostensibly easy because the file is (potentially) on disk, but - the former does not have a structure which maps directly to a - db file image. - */ - export: function(ev){ - toss("export() requires reimplementing for portability reasons."); - /**const db = getMsgDb(ev); - const response = { - buffer: db.exportBinaryImage(), - filename: db.filename, - mimetype: 'application/x-sqlite3' - }; - this.xfer.push(response.buffer.buffer); - return response;**/ - }/*export()*/, - /** - Proxy for the DB constructor. Expects to be passed a single - object or a falsy value to use defaults. The object may - have a filename property to name the db file (see the DB - constructor for peculiarities and transformations) and/or a - buffer property (a Uint8Array holding a complete database - file's contents). The response is an object: - - { - filename: db filename (possibly differing from the input), - - id: an opaque ID value intended for future distinction - between multiple db handles. Messages including a specific - ID will use the DB for that ID. - - } - - If the Worker's db is currently opened, this call closes it - before proceeding. - */ - open: function(ev){ - wState.close(/*true???*/); - const args = [], data = (ev.data || {}); - if(data.simulateError){ - toss("Throwing because of open.simulateError flag."); - } - if(data.filename) args.push(data.filename); - if(data.buffer){ - args.push(data.buffer); - this.xfer.push(data.buffer.buffer); - } - const db = wState.open(args); - return { - filename: db.filename, - dbId: getDbId(db) - }; - }, - /** - Proxy for DB.close(). If ev.data may either be a boolean or - an object with an `unlink` property. If that value is - truthy then the db file (if the db is currently open) will - be unlinked from the virtual filesystem, else it will be - kept intact. The response object is: - - { - filename: db filename _if_ the db is opened when this - is called, else the undefined value - } - */ - close: function(ev){ - const db = getMsgDb(ev,false); - const response = { - filename: db && db.filename - }; - if(db){ - wState.close(db, !!((ev.data && 'object'===typeof ev.data) - ? ev.data.unlink : ev.data)); - } - return response; - }, - toss: function(ev){ - toss("Testing worker exception"); - } - }/*wMsgHandler*/; - - /** - UNDER CONSTRUCTION! - - A subset of the DB API is accessible via Worker messages in the - form: - - { type: apiCommand, - dbId: optional DB ID value (else uses a default db handle) - data: apiArguments - } - - As a rule, these commands respond with a postMessage() of their - own in the same form, but will, if needed, transform the `data` - member to an object and may add state to it. The responses - always have an object-format `data` part. If the inbound `data` - is an object which has a `messageId` property, that property is - always mirrored in the result object, for use in client-side - dispatching of these asynchronous results. Exceptions thrown - during processing result in an `error`-type event with a - payload in the form: - - { - message: error string, - errorClass: class name of the error type, - dbId: DB handle ID, - input: ev.data, - [messageId: if set in the inbound message] - } - - The individual APIs are documented in the wMsgHandler object. - */ - self.onmessage = function(ev){ - ev = ev.data; - let response, dbId = ev.dbId, evType = ev.type; - const arrivalTime = performance.now(); - try { - if(wMsgHandler.hasOwnProperty(evType) && - wMsgHandler[evType] instanceof Function){ - response = wMsgHandler[evType](ev); - }else{ - toss("Unknown db worker message type:",ev.type); - } - }catch(err){ - evType = 'error'; - response = { - message: err.message, - errorClass: err.name, - input: ev - }; - if(err.stack){ - response.stack = ('string'===typeof err.stack) - ? err.stack.split('\n') : err.stack; - } - if(0) console.warn("Worker is propagating an exception to main thread.", - "Reporting it _here_ for the stack trace:",err,response); - } - if(!response.messageId && ev.data - && 'object'===typeof ev.data && ev.data.messageId){ - response.messageId = ev.data.messageId; - } - if(!dbId){ - dbId = response.dbId/*from 'open' cmd*/ - || getDefaultDbId(); - } - if(!response.dbId) response.dbId = dbId; - // Timing info is primarily for use in testing this API. It's not part of - // the public API. arrivalTime = when the worker got the message. - response.workerReceivedTime = arrivalTime; - response.workerRespondTime = performance.now(); - response.departureTime = ev.departureTime; - wState.post(evType, response, wMsgHandler.xfer); - }; - setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'worker-ready'}), 0); -}.bind({self, sqlite3: self.sqlite3}); diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.js new file mode 100644 index 0000000000..62e2bb9bdf --- /dev/null +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -0,0 +1,654 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements the initializer for the sqlite3 "Worker API + #1", a very basic DB access API intended to be scripted from a main + window thread via Worker-style messages. Because of limitations in + that type of communication, this API is minimalistic and only + capable of serving relatively basic DB requests (e.g. it cannot + process nested query loops concurrently). + + This file requires that the core C-style sqlite3 API and OO API #1 + have been loaded. +*/ + +/** + sqlite3.initWorker1API() implements a Worker-based wrapper around + SQLite3 OO API #1, colloquially known as "Worker API #1". + + In order to permit this API to be loaded in worker threads without + automatically registering onmessage handlers, initializing the + worker API requires calling initWorker1API(). If this function is + called from a non-worker thread then it throws an exception. It + must only be called once per Worker. + + When initialized, it installs message listeners to receive Worker + messages and then it posts a message in the form: + + ``` + {type:'sqlite3-api', result:'worker1-ready'} + ``` + + to let the client know that it has been initialized. Clients may + optionally depend on this function not returning until + initialization is complete, as the initialization is synchronous. + In some contexts, however, listening for the above message is + a better fit. + + Note that the worker-based interface can be slightly quirky because + of its async nature. In particular, any number of messages may be posted + to the worker before it starts handling any of them. If, e.g., an + "open" operation fails, any subsequent messages will fail. The + Promise-based wrapper for this API (`sqlite3-worker1-promiser.js`) + is more comfortable to use in that regard. + + The documentation for the input and output worker messages for + this API follows... + + ==================================================================== + Common message format... + + Each message posted to the worker has an operation-independent + envelope and operation-dependent arguments: + + ``` + { + type: string, // one of: 'open', 'close', 'exec', 'config-get' + + messageId: OPTIONAL arbitrary value. The worker will copy it as-is + into response messages to assist in client-side dispatching. + + dbId: a db identifier string (returned by 'open') which tells the + operation which database instance to work on. If not provided, the + first-opened db is used. This is an "opaque" value, with no + inherently useful syntax or information. Its value is subject to + change with any given build of this API and cannot be used as a + basis for anything useful beyond its one intended purpose. + + args: ...operation-dependent arguments... + + // the framework may add other properties for testing or debugging + // purposes. + + } + ``` + + Response messages, posted back to the main thread, look like: + + ``` + { + type: string. Same as above except for error responses, which have the type + 'error', + + messageId: same value, if any, provided by the inbound message + + dbId: the id of the db which was operated on, if any, as returned + by the corresponding 'open' operation. + + result: ...operation-dependent result... + + } + ``` + + ==================================================================== + Error responses + + Errors are reported messages in an operation-independent format: + + ``` + { + type: "error", + + messageId: ...as above..., + + dbId: ...as above... + + result: { + + operation: type of the triggering operation: 'open', 'close', ... + + message: ...error message text... + + errorClass: string. The ErrorClass.name property from the thrown exception. + + input: the message object which triggered the error. + + stack: _if available_, a stack trace array. + + } + + } + ``` + + + ==================================================================== + "config-get" + + This operation fetches the serializable parts of the sqlite3 API + configuration. + + Message format: + + ``` + { + type: "config-get", + messageId: ...as above..., + args: currently ignored and may be elided. + } + ``` + + Response: + + ``` + { + type: "config-get", + messageId: ...as above..., + result: { + + version: sqlite3.version object + + bigIntEnabled: bool. True if BigInt support is enabled. + + wasmfsOpfsDir: path prefix, if any, _intended_ for use with + WASMFS OPFS persistent storage. + + wasmfsOpfsEnabled: true if persistent storage is enabled in the + current environment. Only files stored under wasmfsOpfsDir + will persist using that mechanism, however. It is legal to use + the non-WASMFS OPFS VFS to open a database via a URI-style + db filename. + + vfsList: result of sqlite3.capi.sqlite3_js_vfs_list() + } + } + ``` + + + ==================================================================== + "open" a database + + Message format: + + ``` + { + type: "open", + messageId: ...as above..., + args:{ + + filename [=":memory:" or "" (unspecified)]: the db filename. + See the sqlite3.oo1.DB constructor for peculiarities and + transformations, + + vfs: sqlite3_vfs name. Ignored if filename is ":memory:" or "". + This may change how the given filename is resolved. + } + } + ``` + + Response: + + ``` + { + type: "open", + messageId: ...as above..., + result: { + filename: db filename, possibly differing from the input. + + dbId: an opaque ID value which must be passed in the message + envelope to other calls in this API to tell them which db to + use. If it is not provided to future calls, they will default to + operating on the least-recently-opened db. This property is, for + API consistency's sake, also part of the containing message + envelope. Only the `open` operation includes it in the `result` + property. + + persistent: true if the given filename resides in the + known-persistent storage, else false. + + vfs: name of the VFS the "main" db is using. + } + } + ``` + + ==================================================================== + "close" a database + + Message format: + + ``` + { + type: "close", + messageId: ...as above... + dbId: ...as above... + args: OPTIONAL {unlink: boolean} + } + ``` + + If the `dbId` does not refer to an opened ID, this is a no-op. If + the `args` object contains a truthy `unlink` value then the database + will be unlinked (deleted) after closing it. The inability to close a + db (because it's not opened) or delete its file does not trigger an + error. + + Response: + + ``` + { + type: "close", + messageId: ...as above..., + result: { + + filename: filename of closed db, or undefined if no db was closed + + } + } + ``` + + ==================================================================== + "exec" SQL + + All SQL execution is processed through the exec operation. It offers + most of the features of the oo1.DB.exec() method, with a few limitations + imposed by the state having to cross thread boundaries. + + Message format: + + ``` + { + type: "exec", + messageId: ...as above... + dbId: ...as above... + args: string (SQL) or {... see below ...} + } + ``` + + Response: + + ``` + { + type: "exec", + messageId: ...as above..., + dbId: ...as above... + result: { + input arguments, possibly modified. See below. + } + } + ``` + + The arguments are in the same form accepted by oo1.DB.exec(), with + the exceptions noted below. + + A function-type args.callback property cannot cross + the window/Worker boundary, so is not useful here. If + args.callback is a string then it is assumed to be a + message type key, in which case a callback function will be + applied which posts each row result via: + + postMessage({type: thatKeyType, + rowNumber: 1-based-#, + row: theRow, + columnNames: anArray + }) + + And, at the end of the result set (whether or not any result rows + were produced), it will post an identical message with + (row=undefined, rowNumber=null) to alert the caller than the result + set is completed. Note that a row value of `null` is a legal row + result for certain arg.rowMode values. + + (Design note: we don't use (row=undefined, rowNumber=undefined) to + indicate end-of-results because fetching those would be + indistinguishable from fetching from an empty object unless the + client used hasOwnProperty() (or similar) to distinguish "missing + property" from "property with the undefined value". Similarly, + `null` is a legal value for `row` in some case , whereas the db + layer won't emit a result value of `undefined`.) + + The callback proxy must not recurse into this interface. An exec() + call will tie up the Worker thread, causing any recursion attempt + to wait until the first exec() is completed. + + The response is the input options object (or a synthesized one if + passed only a string), noting that options.resultRows and + options.columnNames may be populated by the call to db.exec(). + +*/ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +sqlite3.initWorker1API = function(){ + 'use strict'; + const toss = (...args)=>{throw new Error(args.join(' '))}; + if('function' !== typeof importScripts){ + toss("initWorker1API() must be run from a Worker thread."); + } + const self = this.self; + const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); + const DB = sqlite3.oo1.DB; + + /** + Returns the app-wide unique ID for the given db, creating one if + needed. + */ + const getDbId = function(db){ + let id = wState.idMap.get(db); + if(id) return id; + id = 'db#'+(++wState.idSeq)+'@'+db.pointer; + /** ^^^ can't simply use db.pointer b/c closing/opening may re-use + the same address, which could map pending messages to a wrong + instance. */ + wState.idMap.set(db, id); + return id; + }; + + /** + Internal helper for managing Worker-level state. + */ + const wState = { + /** + Each opened DB is added to this.dbList, and the first entry in + that list is the default db. As each db is closed, its entry is + removed from the list. + */ + dbList: [], + /** Sequence number of dbId generation. */ + idSeq: 0, + /** Map of DB instances to dbId. */ + idMap: new WeakMap, + /** Temp holder for "transferable" postMessage() state. */ + xfer: [], + open: function(opt){ + const db = new DB(opt); + this.dbs[getDbId(db)] = db; + if(this.dbList.indexOf(db)<0) this.dbList.push(db); + return db; + }, + close: function(db,alsoUnlink){ + if(db){ + delete this.dbs[getDbId(db)]; + const filename = db.filename; + const pVfs = sqlite3.wasm.sqlite3_wasm_db_vfs(db.pointer, 0); + db.close(); + const ddNdx = this.dbList.indexOf(db); + if(ddNdx>=0) this.dbList.splice(ddNdx, 1); + if(alsoUnlink && filename && pVfs){ + sqlite3.wasm.sqlite3_wasm_vfs_unlink(pVfs, filename); + } + } + }, + /** + Posts the given worker message value. If xferList is provided, + it must be an array, in which case a copy of it passed as + postMessage()'s second argument and xferList.length is set to + 0. + */ + post: function(msg,xferList){ + if(xferList && xferList.length){ + self.postMessage( msg, Array.from(xferList) ); + xferList.length = 0; + }else{ + self.postMessage(msg); + } + }, + /** Map of DB IDs to DBs. */ + dbs: Object.create(null), + /** Fetch the DB for the given id. Throw if require=true and the + id is not valid, else return the db or undefined. */ + getDb: function(id,require=true){ + return this.dbs[id] + || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); + } + }; + + /** Throws if the given db is falsy or not opened, else returns its + argument. */ + const affirmDbOpen = function(db = wState.dbList[0]){ + return (db && db.pointer) ? db : toss("DB is not opened."); + }; + + /** Extract dbId from the given message payload. */ + const getMsgDb = function(msgData,affirmExists=true){ + const db = wState.getDb(msgData.dbId,false) || wState.dbList[0]; + return affirmExists ? affirmDbOpen(db) : db; + }; + + const getDefaultDbId = function(){ + return wState.dbList[0] && getDbId(wState.dbList[0]); + }; + + const guessVfs = function(filename){ + const m = /^file:.+(vfs=(\w+))/.exec(filename); + return sqlite3.capi.sqlite3_vfs_find(m ? m[2] : 0); + }; + + const isSpecialDbFilename = (n)=>{ + return ""===n || ':'===n[0]; + }; + + /** + A level of "organizational abstraction" for the Worker1 + API. Each method in this object must map directly to a Worker1 + message type key. The onmessage() dispatcher attempts to + dispatch all inbound messages to a method of this object, + passing it the event.data part of the inbound event object. All + methods must return a plain Object containing any result + state, which the dispatcher may amend. All methods must throw + on error. + */ + const wMsgHandler = { + open: function(ev){ + const oargs = Object.create(null), args = (ev.args || Object.create(null)); + if(args.simulateError){ // undocumented internal testing option + toss("Throwing because of simulateError flag."); + } + const rc = Object.create(null); + const pDir = sqlite3.capi.sqlite3_wasmfs_opfs_dir(); + let byteArray, pVfs; + oargs.vfs = args.vfs; + if(isSpecialDbFilename(args.filename)){ + oargs.filename = args.filename || ""; + }else{ + oargs.filename = args.filename; + byteArray = args.byteArray; + if(byteArray) pVfs = guessVfs(args.filename); + } + if(pVfs){ + /* 2022-11-02: this feature is as-yet untested except that + sqlite3_wasm_vfs_create_file() has been tested from the + browser dev console. */ + let pMem; + try{ + pMem = sqlite3.wasm.allocFromTypedArray(byteArray); + const rc = sqlite3.wasm.sqlite3_wasm_vfs_create_file( + pVfs, oargs.filename, pMem, byteArray.byteLength + ); + if(rc) sqlite3.SQLite3Error.toss(rc); + }catch(e){ + throw new sqlite3.SQLite3Error( + e.name+' creating '+args.filename+": "+e.message, { + cause: e + } + ); + }finally{ + if(pMem) sqlite3.wasm.dealloc(pMem); + } + } + const db = wState.open(oargs); + rc.filename = db.filename; + rc.persistent = (!!pDir && db.filename.startsWith(pDir+'/')) + || !!sqlite3.capi.sqlite3_js_db_uses_vfs(db.pointer, "opfs"); + rc.dbId = getDbId(db); + rc.vfs = db.dbVfsName(); + return rc; + }, + + close: function(ev){ + const db = getMsgDb(ev,false); + const response = { + filename: db && db.filename + }; + if(db){ + const doUnlink = ((ev.args && 'object'===typeof ev.args) + ? !!ev.args.unlink : false); + wState.close(db, doUnlink); + } + return response; + }, + + exec: function(ev){ + const rc = ( + 'string'===typeof ev.args + ) ? {sql: ev.args} : (ev.args || Object.create(null)); + if('stmt'===rc.rowMode){ + toss("Invalid rowMode for 'exec': stmt mode", + "does not work in the Worker API."); + }else if(!rc.sql){ + toss("'exec' requires input SQL."); + } + const db = getMsgDb(ev); + if(rc.callback || Array.isArray(rc.resultRows)){ + // Part of a copy-avoidance optimization for blobs + db._blobXfer = wState.xfer; + } + const theCallback = rc.callback; + let rowNumber = 0; + const hadColNames = !!rc.columnNames; + if('string' === typeof theCallback){ + if(!hadColNames) rc.columnNames = []; + /* Treat this as a worker message type and post each + row as a message of that type. */ + rc.callback = function(row,stmt){ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: ++rowNumber, + row: row + }, wState.xfer); + } + } + try { + db.exec(rc); + if(rc.callback instanceof Function){ + rc.callback = theCallback; + /* Post a sentinel message to tell the client that the end + of the result set has been reached (possibly with zero + rows). */ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: null /*null to distinguish from "property not set"*/, + row: undefined /*undefined because null is a legal row value + for some rowType values, but undefined is not*/ + }); + } + }finally{ + delete db._blobXfer; + if(rc.callback) rc.callback = theCallback; + } + return rc; + }/*exec()*/, + + 'config-get': function(){ + const rc = Object.create(null), src = sqlite3.config; + [ + 'wasmfsOpfsDir', 'bigIntEnabled' + ].forEach(function(k){ + if(Object.getOwnPropertyDescriptor(src, k)) rc[k] = src[k]; + }); + rc.wasmfsOpfsEnabled = !!sqlite3.capi.sqlite3_wasmfs_opfs_dir(); + rc.version = sqlite3.version; + rc.vfsList = sqlite3.capi.sqlite3_js_vfs_list(); + rc.opfsEnabled = !!sqlite3.opfs; + return rc; + }, + + /** + Exports the database to a byte array, as per + sqlite3_serialize(). Response is an object: + + { + byteArray: Uint8Array (db file contents), + filename: the current db filename, + mimetype: 'application/x-sqlite3' + } + */ + export: function(ev){ + const db = getMsgDb(ev); + const response = { + byteArray: sqlite3.capi.sqlite3_js_db_export(db.pointer), + filename: db.filename, + mimetype: 'application/x-sqlite3' + }; + wState.xfer.push(response.byteArray.buffer); + return response; + }/*export()*/, + + toss: function(ev){ + toss("Testing worker exception"); + }, + + 'opfs-tree': async function(ev){ + if(!sqlite3.opfs) toss("OPFS support is unavailable."); + const response = await sqlite3.opfs.treeList(); + return response; + } + }/*wMsgHandler*/; + + self.onmessage = async function(ev){ + ev = ev.data; + let result, dbId = ev.dbId, evType = ev.type; + const arrivalTime = performance.now(); + try { + if(wMsgHandler.hasOwnProperty(evType) && + wMsgHandler[evType] instanceof Function){ + result = await wMsgHandler[evType](ev); + }else{ + toss("Unknown db worker message type:",ev.type); + } + }catch(err){ + evType = 'error'; + result = { + operation: ev.type, + message: err.message, + errorClass: err.name, + input: ev + }; + if(err.stack){ + result.stack = ('string'===typeof err.stack) + ? err.stack.split(/\n\s*/) : err.stack; + } + if(0) console.warn("Worker is propagating an exception to main thread.", + "Reporting it _here_ for the stack trace:",err,result); + } + if(!dbId){ + dbId = result.dbId/*from 'open' cmd*/ + || getDefaultDbId(); + } + // Timing info is primarily for use in testing this API. It's not part of + // the public API. arrivalTime = when the worker got the message. + wState.post({ + type: evType, + dbId: dbId, + messageId: ev.messageId, + workerReceivedTime: arrivalTime, + workerRespondTime: performance.now(), + departureTime: ev.departureTime, + // TODO: move the timing bits into... + //timing:{ + // departure: ev.departureTime, + // workerReceived: arrivalTime, + // workerResponse: performance.now(); + //}, + result: result + }, wState.xfer); + }; + self.postMessage({type:'sqlite3-api',result:'worker1-ready'}); +}.bind({self, sqlite3}); +}); diff --git a/ext/wasm/api/sqlite3-license-version-header.js b/ext/wasm/api/sqlite3-license-version-header.js new file mode 100644 index 0000000000..f8b3eddc5f --- /dev/null +++ b/ext/wasm/api/sqlite3-license-version-header.js @@ -0,0 +1,25 @@ +/* +** LICENSE for the sqlite3 WebAssembly/JavaScript APIs. +** +** This bundle (typically released as sqlite3.js or sqlite3-wasmfs.js) +** is an amalgamation of JavaScript source code from two projects: +** +** 1) https://emscripten.org: the Emscripten "glue code" is covered by +** the terms of the MIT license and University of Illinois/NCSA +** Open Source License, as described at: +** +** https://emscripten.org/docs/introducing_emscripten/emscripten_license.html +** +** 2) https://sqlite.org: all code and documentation labeled as being +** from this source are released under the same terms as the sqlite3 +** C library: +** +** 2022-10-16 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +*/ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js new file mode 100644 index 0000000000..09c56ff1df --- /dev/null +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -0,0 +1,774 @@ +/* + 2022-09-16 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A Worker which manages asynchronous OPFS handles on behalf of a + synchronous API which controls it via a combination of Worker + messages, SharedArrayBuffer, and Atomics. It is the asynchronous + counterpart of the API defined in sqlite3-api-opfs.js. + + Highly indebted to: + + https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js + + for demonstrating how to use the OPFS APIs. + + This file is to be loaded as a Worker. It does not have any direct + access to the sqlite3 JS/WASM bits, so any bits which it needs (most + notably SQLITE_xxx integer codes) have to be imported into it via an + initialization process. + + This file represents an implementation detail of a larger piece of + code, and not a public interface. Its details may change at any time + and are not intended to be used by any client-level code. +*/ +"use strict"; +const toss = function(...args){throw new Error(args.join(' '))}; +if(self.window === self){ + toss("This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); +}else if(!navigator.storage.getDirectory){ + toss("This API requires navigator.storage.getDirectory."); +} + +/** + Will hold state copied to this object from the syncronous side of + this API. +*/ +const state = Object.create(null); +/** + verbose: + + 0 = no logging output + 1 = only errors + 2 = warnings and errors + 3 = debug, warnings, and errors +*/ +state.verbose = 2; + +const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) +}; +const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); +}; +const log = (...args)=>logImpl(2, ...args); +const warn = (...args)=>logImpl(1, ...args); +const error = (...args)=>logImpl(0, ...args); +const metrics = Object.create(null); +metrics.reset = ()=>{ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); + } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; +}; +metrics.dump = ()=>{ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + } + console.log(self.location.href, + "metrics for",self.location.href,":\n", + metrics, + "\nTotal of",n,"op(s) for",t,"ms", + "approx",w,"ms spent waiting on OPFS APIs."); + console.log("Serialization metrics:",metrics.s11n); +}; + +/** + Map of sqlite3_file pointers (integers) to metadata related to a + given OPFS file handles. The pointers are, in this side of the + interface, opaque file handle IDs provided by the synchronous + part of this constellation. Each value is an object with a structure + demonstrated in the xOpen() impl. +*/ +const __openFiles = Object.create(null); + +/** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg is + true, the result is returned as an array of path elements, else an + absolute path string is returned. +*/ +const getResolvedPath = function(filename,splitIt){ + const p = new URL( + filename, 'file://irrelevant' + ).pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; +}; + +/** + Takes the absolute path to a filesystem element. Returns an array + of [handleOfContainingDir, filename]. If the 2nd argument is truthy + then each directory element leading to the file is created along + the way. Throws if any creation or resolution fails. +*/ +const getDirForFilename = async function f(absFilename, createDirs = false){ + const path = getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = state.rootDir; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; +}; + +/** + An error class specifically for use with getSyncHandle(), the goal + of which is to eventually be able to distinguish unambiguously + between locking-related failures and other types, noting that we + cannot currently do so because createSyncAccessHandle() does not + define its exceptions in the required level of detail. +*/ +class GetSyncHandleError extends Error { + constructor(errorObject, ...msg){ + super(); + this.error = errorObject; + this.message = [ + ...msg, ': Original exception ['+errorObject.name+']:', + errorObject.message + ].join(' '); + this.name = 'GetSyncHandleError'; + } +}; + +/** + Returns the sync access handle associated with the given file + handle object (which must be a valid handle object, as created by + xOpen()), lazily opening it if needed. + + In order to help alleviate cross-tab contention for a dabase, + if an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to 3 times. If acquisition + still fails at that point it will give up and propagate the + exception. +*/ +const getSyncHandle = async (fh)=>{ + if(!fh.syncHandle){ + const t = performance.now(); + log("Acquiring sync handle for",fh.filenameAbs); + const maxTries = 4, msBase = 300; + let i = 1, ms = msBase; + for(; true; ms = msBase * ++i){ + try { + //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); + //TODO? A config option which tells it to throw here + //randomly every now and then, for testing purposes. + fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); + break; + }catch(e){ + if(i === maxTries){ + throw new GetSyncHandleError( + e, "Error getting sync handle.",maxTries, + "attempts failed.",fh.filenameAbs + ); + } + warn("Error getting sync handle. Waiting",ms, + "ms and trying again.",fh.filenameAbs,e); + Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); + } + } + log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + } + return fh.syncHandle; +}; + +/** + If the given file-holding object has a sync handle attached to it, + that handle is remove and asynchronously closed. Though it may + sound sensible to continue work as soon as the close() returns + (noting that it's asynchronous), doing so can cause operations + performed soon afterwards, e.g. a call to getSyncHandle() to fail + because they may happen out of order from the close(). OPFS does + not guaranty that the actual order of operations is retained in + such cases. i.e. always "await" on the result of this function. +*/ +const closeSyncHandle = async (fh)=>{ + if(fh.syncHandle){ + log("Closing sync handle for",fh.filenameAbs); + const h = fh.syncHandle; + delete fh.syncHandle; + return h.close(); + } +}; + +/** + Stores the given value at state.sabOPView[state.opIds.rc] and then + Atomics.notify()'s it. +*/ +const storeAndNotify = (opName, value)=>{ + log(opName+"() => notify(",value,")"); + Atomics.store(state.sabOPView, state.opIds.rc, value); + Atomics.notify(state.sabOPView, state.opIds.rc); +}; + +/** + Throws if fh is a file-holding object which is flagged as read-only. +*/ +const affirmNotRO = function(opName,fh){ + if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); +}; +const affirmLocked = function(opName,fh){ + //if(!fh.syncHandle) toss(opName+"(): File does not have a lock: "+fh.filenameAbs); + /** + Currently a no-op, as speedtest1 triggers xRead() without a + lock (that seems like a bug but it's currently uninvestigated). + This means, however, that some OPFS VFS routines may trigger + acquisition of a lock but never let it go until xUnlock() is + called (which it likely won't be if xLock() was not called). + */ +}; + +/** + We track 2 different timers: the "metrics" timer records how much + time we spend performing work. The "wait" timer records how much + time we spend waiting on the underlying OPFS timer. See the calls + to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() + throughout this file to see how they're used. +*/ +const __mTimer = Object.create(null); +__mTimer.op = undefined; +__mTimer.start = undefined; +const mTimeStart = (op)=>{ + __mTimer.start = performance.now(); + __mTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + ++metrics[op].count; +}; +const mTimeEnd = ()=>( + metrics[__mTimer.op].time += performance.now() - __mTimer.start +); +const __wTimer = Object.create(null); +__wTimer.op = undefined; +__wTimer.start = undefined; +const wTimeStart = (op)=>{ + __wTimer.start = performance.now(); + __wTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); +}; +const wTimeEnd = ()=>( + metrics[__wTimer.op].wait += performance.now() - __wTimer.start +); + +/** + Gets set to true by the 'opfs-async-shutdown' command to quit the + wait loop. This is only intended for debugging purposes: we cannot + inspect this file's state while the tight waitLoop() is running and + need a way to stop that loop for introspection purposes. +*/ +let flagAsyncShutdown = false; + + +/** + Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods + methods, as well as helpers like mkdir(). Maintenance reminder: + members are in alphabetical order to simplify finding them. +*/ +const vfsAsyncImpls = { + 'opfs-async-metrics': async ()=>{ + mTimeStart('opfs-async-metrics'); + metrics.dump(); + storeAndNotify('opfs-async-metrics', 0); + mTimeEnd(); + }, + 'opfs-async-shutdown': async ()=>{ + flagAsyncShutdown = true; + storeAndNotify('opfs-async-shutdown', 0); + }, + mkdir: async (dirname)=>{ + mTimeStart('mkdir'); + let rc = 0; + wTimeStart('mkdir'); + try { + await getDirForFilename(dirname+"/filepart", true); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('mkdir', rc); + mTimeEnd(); + }, + xAccess: async (filename)=>{ + mTimeStart('xAccess'); + /* OPFS cannot support the full range of xAccess() queries sqlite3 + calls for. We can essentially just tell if the file is + accessible, but if it is it's automatically writable (unless + it's locked, which we cannot(?) know without trying to open + it). OPFS does not have the notion of read-only. + + The return semantics of this function differ from sqlite3's + xAccess semantics because we are limited in what we can + communicate back to our synchronous communication partner: 0 = + accessible, non-0 means not accessible. + */ + let rc = 0; + wTimeStart('xAccess'); + try{ + const [dh, fn] = await getDirForFilename(filename); + await dh.getFileHandle(fn); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('xAccess', rc); + mTimeEnd(); + }, + xClose: async function(fid/*sqlite3_file pointer*/){ + const opName = 'xClose'; + mTimeStart(opName); + const fh = __openFiles[fid]; + let rc = 0; + wTimeStart('xClose'); + if(fh){ + delete __openFiles[fid]; + await closeSyncHandle(fh); + if(fh.deleteOnClose){ + try{ await fh.dirHandle.removeEntry(fh.filenamePart) } + catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + } + }else{ + state.s11n.serialize(); + rc = state.sq3Codes.SQLITE_NOTFOUND; + } + wTimeEnd(); + storeAndNotify(opName, rc); + mTimeEnd(); + }, + xDelete: async function(...args){ + mTimeStart('xDelete'); + const rc = await vfsAsyncImpls.xDeleteNoWait(...args); + storeAndNotify('xDelete', rc); + mTimeEnd(); + }, + xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ + /* The syncDir flag is, for purposes of the VFS API's semantics, + ignored here. However, if it has the value 0x1234 then: after + deleting the given file, recursively try to delete any empty + directories left behind in its wake (ignoring any errors and + stopping at the first failure). + + That said: we don't know for sure that removeEntry() fails if + the dir is not empty because the API is not documented. It has, + however, a "recursive" flag which defaults to false, so + presumably it will fail if the dir is not empty and that flag + is false. + */ + let rc = 0; + wTimeStart('xDelete'); + try { + while(filename){ + const [hDir, filenamePart] = await getDirForFilename(filename, false); + if(!filenamePart) break; + await hDir.removeEntry(filenamePart, {recursive}); + if(0x1234 !== syncDir) break; + recursive = false; + filename = getResolvedPath(filename, true); + filename.pop(); + filename = filename.join('/'); + } + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_DELETE; + } + wTimeEnd(); + return rc; + }, + xFileSize: async function(fid/*sqlite3_file pointer*/){ + mTimeStart('xFileSize'); + const fh = __openFiles[fid]; + let rc; + wTimeStart('xFileSize'); + try{ + affirmLocked('xFileSize',fh); + rc = await (await getSyncHandle(fh)).getSize(); + state.s11n.serialize(Number(rc)); + rc = 0; + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + } + wTimeEnd(); + storeAndNotify('xFileSize', rc); + mTimeEnd(); + }, + xLock: async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + mTimeStart('xLock'); + const fh = __openFiles[fid]; + let rc = 0; + if( !fh.syncHandle ){ + wTimeStart('xLock'); + try { await getSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_LOCK; + } + wTimeEnd(); + } + storeAndNotify('xLock',rc); + mTimeEnd(); + }, + xOpen: async function(fid/*sqlite3_file pointer*/, filename, + flags/*SQLITE_OPEN_...*/){ + const opName = 'xOpen'; + mTimeStart(opName); + const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); + const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); + wTimeStart('xOpen'); + try{ + let hDir, filenamePart; + try { + [hDir, filenamePart] = await getDirForFilename(filename, !!create); + }catch(e){ + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); + mTimeEnd(); + wTimeEnd(); + return; + } + const hFile = await hDir.getFileHandle(filenamePart, {create}); + /** + wa-sqlite, at this point, grabs a SyncAccessHandle and + assigns it to the syncHandle prop of the file state + object, but only for certain cases and it's unclear why it + places that limitation on it. + */ + wTimeEnd(); + __openFiles[fid] = Object.assign(Object.create(null),{ + filenameAbs: filename, + filenamePart: filenamePart, + dirHandle: hDir, + fileHandle: hFile, + sabView: state.sabFileBufView, + readOnly: create + ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), + deleteOnClose: deleteOnClose + }); + storeAndNotify(opName, 0); + }catch(e){ + wTimeEnd(); + error(opName,e); + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); + } + mTimeEnd(); + }, + xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xRead'); + let rc = 0, nRead; + const fh = __openFiles[fid]; + try{ + affirmLocked('xRead',fh); + wTimeStart('xRead'); + nRead = (await getSyncHandle(fh)).read( + fh.sabView.subarray(0, n), + {at: Number(offset64)} + ); + wTimeEnd(); + if(nRead < n){/* Zero-fill remaining bytes */ + fh.sabView.fill(0, nRead, n); + rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; + } + }catch(e){ + if(undefined===nRead) wTimeEnd(); + error("xRead() failed",e,fh); + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_READ; + } + storeAndNotify('xRead',rc); + mTimeEnd(); + }, + xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ + mTimeStart('xSync'); + const fh = __openFiles[fid]; + let rc = 0; + if(!fh.readOnly && fh.syncHandle){ + try { + wTimeStart('xSync'); + await fh.syncHandle.flush(); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_FSYNC; + } + wTimeEnd(); + } + storeAndNotify('xSync',rc); + mTimeEnd(); + }, + xTruncate: async function(fid/*sqlite3_file pointer*/,size){ + mTimeStart('xTruncate'); + let rc = 0; + const fh = __openFiles[fid]; + wTimeStart('xTruncate'); + try{ + affirmLocked('xTruncate',fh); + affirmNotRO('xTruncate', fh); + await (await getSyncHandle(fh)).truncate(size); + }catch(e){ + error("xTruncate():",e,fh); + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; + } + wTimeEnd(); + storeAndNotify('xTruncate',rc); + mTimeEnd(); + }, + xUnlock: async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + mTimeStart('xUnlock'); + let rc = 0; + const fh = __openFiles[fid]; + if( state.sq3Codes.SQLITE_LOCK_NONE===lockType + && fh.syncHandle ){ + wTimeStart('xUnlock'); + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + } + wTimeEnd(); + } + storeAndNotify('xUnlock',rc); + mTimeEnd(); + }, + xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xWrite'); + let rc; + const fh = __openFiles[fid]; + wTimeStart('xWrite'); + try{ + affirmLocked('xWrite',fh); + affirmNotRO('xWrite', fh); + rc = ( + n === (await getSyncHandle(fh)) + .write(fh.sabView.subarray(0, n), + {at: Number(offset64)}) + ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + }catch(e){ + error("xWrite():",e,fh); + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_WRITE; + } + wTimeEnd(); + storeAndNotify('xWrite',rc); + mTimeEnd(); + } +}/*vfsAsyncImpls*/; + +const initS11n = ()=>{ + /** + ACHTUNG: this code is 100% duplicated in the other half of this + proxy! The documentation is maintained in the "synchronous half". + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + state.s11n.deserialize = function(){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; + + return state.s11n; +}/*initS11n()*/; + +const waitLoop = async function f(){ + const opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } + /** + waitTime is how long (ms) to wait for each Atomics.wait(). + We need to wake up periodically to give the thread a chance + to do other things. + */ + const waitTime = 1000; + while(!flagAsyncShutdown){ + try { + if('timed-out'===Atomics.wait( + state.sabOPView, state.opIds.whichOp, 0, waitTime + )){ + continue; + } + const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); + Atomics.store(state.sabOPView, state.opIds.whichOp, 0); + const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + const args = state.s11n.deserialize() || []; + state.s11n.serialize(/* clear s11n to keep the caller from + confusing this with an exception string + written by the upcoming operation */); + //warn("waitLoop() whichOp =",opId, hnd, args); + if(hnd.f) await hnd.f(...args); + else error("Missing callback for opId",opId); + }catch(e){ + error('in waitLoop():',e); + } + } +}; + +navigator.storage.getDirectory().then(function(d){ + const wMsg = (type)=>postMessage({type}); + state.rootDir = d; + self.onmessage = function({data}){ + switch(data.type){ + case 'opfs-async-init':{ + /* Receive shared state from synchronous partner */ + const opt = data.args; + state.littleEndian = opt.littleEndian; + state.asyncS11nExceptions = opt.asyncS11nExceptions; + state.verbose = opt.verbose ?? 2; + state.fileBufferSize = opt.fileBufferSize; + state.sabS11nOffset = opt.sabS11nOffset; + state.sabS11nSize = opt.sabS11nSize; + state.sabOP = opt.sabOP; + state.sabOPView = new Int32Array(state.sabOP); + state.sabIO = opt.sabIO; + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.opIds = opt.opIds; + state.sq3Codes = opt.sq3Codes; + Object.keys(vfsAsyncImpls).forEach((k)=>{ + if(!Number.isFinite(state.opIds[k])){ + toss("Maintenance required: missing state.opIds[",k,"]"); + } + }); + initS11n(); + metrics.reset(); + log("init state",state); + wMsg('opfs-async-inited'); + waitLoop(); + break; + } + case 'opfs-async-restart': + if(flagAsyncShutdown){ + warn("Restarting after opfs-async-shutdown. Might or might not work."); + flagAsyncShutdown = false; + waitLoop(); + } + break; + case 'opfs-async-metrics': + metrics.dump(); + break; + } + }; + wMsg('opfs-async-loaded'); +}).catch((e)=>error("error initializing OPFS asyncer:",e)); diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 6a81da3e5f..9d04ad1291 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1,4 +1,299 @@ -#include "sqlite3.c" +/* +** This file requires access to sqlite3.c static state in order to +** implement certain WASM-specific features, and thus directly +** includes that file. Unlike the rest of sqlite3.c, this file +** requires compiling with -std=c99 (or equivalent, or a later C +** version) because it makes use of features not available in C89. +** +** At its simplest, to build sqlite3.wasm either place this file +** in the same directory as sqlite3.c/h before compilation or use the +** -I/path flag to tell the compiler where to find both of those +** files, then compile this file. For example: +** +** emcc -o sqlite3.wasm ... -I/path/to/sqlite3-c-and-h sqlite3-wasm.c +*/ +#define SQLITE_WASM +#ifdef SQLITE_WASM_ENABLE_C_TESTS +/* +** Code blocked off by SQLITE_WASM_TESTS is intended solely for use in +** unit/regression testing. They may be safely omitted from +** client-side builds. The main unit test script, tester1.js, will +** skip related tests if it doesn't find the corresponding functions +** in the WASM exports. +*/ +# define SQLITE_WASM_TESTS 1 +#else +# define SQLITE_WASM_TESTS 0 +#endif + +/* +** Threading and file locking: JS is single-threaded. Each Worker +** thread is a separate instance of the JS engine so can never access +** the same db handle as another thread, thus multi-threading support +** is unnecessary in the library. Because the filesystems are virtual +** and local to a given wasm runtime instance, two Workers can never +** access the same db file at once, with the exception of OPFS. As of +** this writing (2022-09-30), OPFS exclusively locks a file when +** opening it, so two Workers can never open the same OPFS-backed file +** at once. That situation will change if and when lower-level locking +** features are added to OPFS (as is currently planned, per folks +** involved with its development). +** +** Summary: except for the case of future OPFS, which supports +** locking, and any similar future filesystems, threading and file +** locking support are unnecessary in the wasm build. +*/ + +/* +** Undefine any SQLITE_... config flags which we specifically do not +** want undefined. Please keep these alphabetized. +*/ +#undef SQLITE_OMIT_DESERIALIZE +#undef SQLITE_OMIT_MEMORYDB + +/* +** Define any SQLITE_... config defaults we want if they aren't +** overridden by the builder. Please keep these alphabetized. +*/ + +/**********************************************************************/ +/* SQLITE_D... */ +#ifndef SQLITE_DEFAULT_CACHE_SIZE +/* +** The OPFS impls benefit tremendously from an increased cache size +** when working on large workloads, e.g. speedtest1 --size 50 or +** higher. On smaller workloads, e.g. speedtest1 --size 25, they +** clearly benefit from having 4mb of cache, but not as much as a +** larger cache benefits the larger workloads. Speed differences +** between 2x and nearly 3x have been measured with ample page cache. +*/ +# define SQLITE_DEFAULT_CACHE_SIZE -16384 +#endif +#if 0 && !defined(SQLITE_DEFAULT_PAGE_SIZE) +/* TODO: experiment with this. */ +# define SQLITE_DEFAULT_PAGE_SIZE 8192 /*4096*/ +#endif +#ifndef SQLITE_DEFAULT_UNIX_VFS +# define SQLITE_DEFAULT_UNIX_VFS "unix-none" +#endif +#undef SQLITE_DQS +#define SQLITE_DQS 0 + +/**********************************************************************/ +/* SQLITE_ENABLE_... */ +#ifndef SQLITE_ENABLE_BYTECODE_VTAB +# define SQLITE_ENABLE_BYTECODE_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_DBPAGE_VTAB +# define SQLITE_ENABLE_DBPAGE_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_DBSTAT_VTAB +# define SQLITE_ENABLE_DBSTAT_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_EXPLAIN_COMMENTS +# define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 +#endif +#ifndef SQLITE_ENABLE_FTS4 +# define SQLITE_ENABLE_FTS4 1 +#endif +#ifndef SQLITE_ENABLE_OFFSET_SQL_FUNC +# define SQLITE_ENABLE_OFFSET_SQL_FUNC 1 +#endif +#ifndef SQLITE_ENABLE_RTREE +# define SQLITE_ENABLE_RTREE 1 +#endif +#ifndef SQLITE_ENABLE_STMTVTAB +# define SQLITE_ENABLE_STMTVTAB 1 +#endif +#ifndef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +# define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +#endif + +/**********************************************************************/ +/* SQLITE_O... */ +#ifndef SQLITE_OMIT_DEPRECATED +# define SQLITE_OMIT_DEPRECATED 1 +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif +#ifndef SQLITE_OMIT_SHARED_CACHE +# define SQLITE_OMIT_SHARED_CACHE 1 +#endif +#ifndef SQLITE_OMIT_UTF16 +# define SQLITE_OMIT_UTF16 1 +#endif +#ifndef SQLITE_OMIT_WAL +# define SQLITE_OMIT_WAL 1 +#endif +#ifndef SQLITE_OS_KV_OPTIONAL +# define SQLITE_OS_KV_OPTIONAL 1 +#endif + +/**********************************************************************/ +/* SQLITE_T... */ +#ifndef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 3 +#endif +#ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 0 +#endif + +/**********************************************************************/ +/* SQLITE_USE_... */ +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 1 +#endif + +#include +#include "sqlite3.c" /* yes, .c instead of .h. */ + +#if defined(__EMSCRIPTEN__) +# include +#endif + +/* +** SQLITE_WASM_KEEP is functionally identical to EMSCRIPTEN_KEEPALIVE +** but is not Emscripten-specific. It explicitly marks functions for +** export into the target wasm file without requiring explicit listing +** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list +** (or equivalent in other build platforms). Any function with neither +** this attribute nor which is listed as an explicit export will not +** be exported from the wasm file (but may still be used internally +** within the wasm file). +** +** The functions in this file (sqlite3-wasm.c) which require exporting +** are marked with this flag. They may also be added to any explicit +** build-time export list but need not be. All of these APIs are +** intended for use only within the project's own JS/WASM code, and +** not by client code, so an argument can be made for reducing their +** visibility by not including them in any build-time export lists. +** +** 2022-09-11: it's not yet _proven_ that this approach works in +** non-Emscripten builds. If not, such builds will need to export +** those using the --export=... wasm-ld flag (or equivalent). As of +** this writing we are tied to Emscripten for various reasons +** and cannot test the library with other build environments. +*/ +#define SQLITE_WASM_KEEP __attribute__((used,visibility("default"))) +// See also: +//__attribute__((export_name("theExportedName"), used, visibility("default"))) + + +#if 0 +/* +** An EXPERIMENT in implementing a stack-based allocator analog to +** Emscripten's stackSave(), stackAlloc(), stackRestore(). +** Unfortunately, this cannot work together with Emscripten because +** Emscripten defines its own native one and we'd stomp on each +** other's memory. Other than that complication, basic tests show it +** to work just fine. +** +** Another option is to malloc() a chunk of our own and call that our +** "stack". +*/ +SQLITE_WASM_KEEP void * sqlite3_wasm_stack_end(void){ + extern void __heap_base + /* see https://stackoverflow.com/questions/10038964 */; + return &__heap_base; +} +SQLITE_WASM_KEEP void * sqlite3_wasm_stack_begin(void){ + extern void __data_end; + return &__data_end; +} +static void * pWasmStackPtr = 0; +SQLITE_WASM_KEEP void * sqlite3_wasm_stack_ptr(void){ + if(!pWasmStackPtr) pWasmStackPtr = sqlite3_wasm_stack_end(); + return pWasmStackPtr; +} +SQLITE_WASM_KEEP void sqlite3_wasm_stack_restore(void * p){ + pWasmStackPtr = p; +} +SQLITE_WASM_KEEP void * sqlite3_wasm_stack_alloc(int n){ + if(n<=0) return 0; + n = (n + 7) & ~7 /* align to 8-byte boundary */; + unsigned char * const p = (unsigned char *)sqlite3_wasm_stack_ptr(); + unsigned const char * const b = (unsigned const char *)sqlite3_wasm_stack_begin(); + if(b + n >= p || b + n < b/*overflow*/) return 0; + return pWasmStackPtr = p - n; +} +#endif /* stack allocator experiment */ + +/* +** State for the "pseudo-stack" allocator implemented in +** sqlite3_wasm_pstack_xyz(). In order to avoid colliding with +** Emscripten-controled stack space, it carves out a bit of stack +** memory to use for that purpose. This memory ends up in the +** WASM-managed memory, such that routines which manipulate the wasm +** heap can also be used to manipulate this memory. +** +** This particular allocator is intended for small allocations such as +** storage for output pointers. We cannot reasonably size it large +** enough for general-purpose string conversions because some of our +** tests use input files (strings) of 16MB+. +*/ +static unsigned char PStack_mem[512 * 8] = {0}; +static struct { + unsigned const char * const pBegin;/* Start (inclusive) of memory */ + unsigned const char * const pEnd; /* One-after-the-end of memory */ + unsigned char * pPos; /* Current stack pointer */ +} PStack = { + &PStack_mem[0], + &PStack_mem[0] + sizeof(PStack_mem), + &PStack_mem[0] + sizeof(PStack_mem) +}; +/* +** Returns the current pstack position. +*/ +SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_ptr(void){ + return PStack.pPos; +} +/* +** Sets the pstack position poitner to p. Results are undefined if the +** given value did not come from sqlite3_wasm_pstack_ptr(). +*/ +SQLITE_WASM_KEEP void sqlite3_wasm_pstack_restore(unsigned char * p){ + assert(p>=PStack.pBegin && p<=PStack.pEnd && p>=PStack.pPos); + assert(0==(p & 0x7)); + if(p>=PStack.pBegin && p<=PStack.pEnd /*&& p>=PStack.pPos*/){ + PStack.pPos = p; + } +} +/* +** Allocate and zero out n bytes from the pstack. Returns a pointer to +** the memory on success, 0 on error (including a negative n value). n +** is always adjusted to be a multiple of 8 and returned memory is +** always zeroed out before returning (because this keeps the client +** JS code from having to do so, and most uses of the pstack will +** call for doing so). +*/ +SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_alloc(int n){ + if( n<=0 ) return 0; + //if( n & 0x7 ) n += 8 - (n & 0x7) /* align to 8-byte boundary */; + n = (n + 7) & ~7 /* align to 8-byte boundary */; + if( PStack.pBegin + n > PStack.pPos /*not enough space left*/ + || PStack.pBegin + n <= PStack.pBegin /*overflow*/ ) return 0; + memset((PStack.pPos = PStack.pPos - n), 0, (unsigned int)n); + return PStack.pPos; +} +/* +** Return the number of bytes left which can be +** sqlite3_wasm_pstack_alloc()'d. +*/ +SQLITE_WASM_KEEP int sqlite3_wasm_pstack_remaining(void){ + assert(PStack.pPos >= PStack.pBegin); + assert(PStack.pPos <= PStack.pEnd); + return (int)(PStack.pPos - PStack.pBegin); +} + +/* +** Return the total number of bytes available in the pstack, including +** any space which is currently allocated. This value is a +** compile-time constant. +*/ +SQLITE_WASM_KEEP int sqlite3_wasm_pstack_quota(void){ + return (int)(PStack.pEnd - PStack.pBegin); +} /* ** This function is NOT part of the sqlite3 public API. It is strictly @@ -14,9 +309,9 @@ ** ** Returns err_code. */ -int sqlite3_wasm_db_error(sqlite3*db, int err_code, - const char *zMsg){ - if(0!=zMsg){ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ + if( 0!=zMsg ){ const int nMsg = sqlite3Strlen30(zMsg); sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); }else{ @@ -25,6 +320,28 @@ int sqlite3_wasm_db_error(sqlite3*db, int err_code, return err_code; } +#if SQLITE_WASM_TESTS +struct WasmTestStruct { + int v4; + void * ppV; + const char * cstr; + int64_t v8; + void (*xFunc)(void*); +}; +typedef struct WasmTestStruct WasmTestStruct; +SQLITE_WASM_KEEP +void sqlite3_wasm_test_struct(WasmTestStruct * s){ + if(s){ + s->v4 *= 2; + s->v8 = s->v4 * 2; + s->ppV = s; + s->cstr = __FILE__; + if(s->xFunc) s->xFunc(s); + } + return; +} +#endif /* SQLITE_WASM_TESTS */ + /* ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. Unlike the @@ -32,54 +349,182 @@ int sqlite3_wasm_db_error(sqlite3*db, int err_code, ** variadic macros. ** ** Returns a string containing a JSON-format "enum" of C-level -** constants intended to be imported into the JS environment. The JSON -** is initialized the first time this function is called and that -** result is reused for all future calls. +** constants and struct-related metadata intended to be imported into +** the JS environment. The JSON is initialized the first time this +** function is called and that result is reused for all future calls. ** ** If this function returns NULL then it means that the internal -** buffer is not large enough for the generated JSON. In debug builds -** that will trigger an assert(). +** buffer is not large enough for the generated JSON and needs to be +** increased. In debug builds that will trigger an assert(). */ +SQLITE_WASM_KEEP const char * sqlite3_wasm_enum_json(void){ - static char strBuf[1024 * 8] = {0} /* where the JSON goes */; - int n = 0, childCount = 0, structCount = 0 + static char aBuffer[1024 * 12] = {0} /* where the JSON goes */; + int n = 0, nChildren = 0, nStruct = 0 /* output counters for figuring out where commas go */; - char * pos = &strBuf[1] /* skip first byte for now to help protect + char * zPos = &aBuffer[1] /* skip first byte for now to help protect ** against a small race condition */; - char const * const zEnd = pos + sizeof(strBuf) /* one-past-the-end */; - if(strBuf[0]) return strBuf; - /* Leave strBuf[0] at 0 until the end to help guard against a tiny + char const * const zEnd = &aBuffer[0] + sizeof(aBuffer) /* one-past-the-end */; + if(aBuffer[0]) return aBuffer; + /* Leave aBuffer[0] at 0 until the end to help guard against a tiny ** race condition. If this is called twice concurrently, they might - ** end up both writing to strBuf, but they'll both write the same + ** end up both writing to aBuffer, but they'll both write the same ** thing, so that's okay. If we set byte 0 up front then the 2nd ** instance might return and use the string before the 1st instance ** is done filling it. */ /* Core output macros... */ -#define lenCheck assert(pos < zEnd - 128 \ +#define lenCheck assert(zPos < zEnd - 128 \ && "sqlite3_wasm_enum_json() buffer is too small."); \ - if(pos >= zEnd - 128) return 0 + if( zPos >= zEnd - 128 ) return 0 #define outf(format,...) \ - pos += snprintf(pos, ((size_t)(zEnd - pos)), format, __VA_ARGS__); \ + zPos += snprintf(zPos, ((size_t)(zEnd - zPos)), format, __VA_ARGS__); \ lenCheck #define out(TXT) outf("%s",TXT) #define CloseBrace(LEVEL) \ - assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck + assert(LEVEL<5); memset(zPos, '}', LEVEL); zPos+=LEVEL; lenCheck /* Macros for emitting maps of integer- and string-type macros to ** their values. */ #define DefGroup(KEY) n = 0; \ - outf("%s\"" #KEY "\": {",(childCount++ ? "," : "")); + outf("%s\"" #KEY "\": {",(nChildren++ ? "," : "")); #define DefInt(KEY) \ outf("%s\"%s\": %d", (n++ ? ", " : ""), #KEY, (int)KEY) #define DefStr(KEY) \ outf("%s\"%s\": \"%s\"", (n++ ? ", " : ""), #KEY, KEY) #define _DefGroup CloseBrace(1) - DefGroup(version) { - DefInt(SQLITE_VERSION_NUMBER); - DefStr(SQLITE_VERSION); - DefStr(SQLITE_SOURCE_ID); + /* The following groups are sorted alphabetic by group name. */ + DefGroup(access){ + DefInt(SQLITE_ACCESS_EXISTS); + DefInt(SQLITE_ACCESS_READWRITE); + DefInt(SQLITE_ACCESS_READ)/*docs say this is unused*/; + } _DefGroup; + + DefGroup(blobFinalizers) { + /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as + ** integers to avoid casting-related warnings. */ + out("\"SQLITE_STATIC\":0, \"SQLITE_TRANSIENT\":-1"); + } _DefGroup; + + DefGroup(dataTypes) { + DefInt(SQLITE_INTEGER); + DefInt(SQLITE_FLOAT); + DefInt(SQLITE_TEXT); + DefInt(SQLITE_BLOB); + DefInt(SQLITE_NULL); + } _DefGroup; + + DefGroup(encodings) { + /* Noting that the wasm binding only aims to support UTF-8. */ + DefInt(SQLITE_UTF8); + DefInt(SQLITE_UTF16LE); + DefInt(SQLITE_UTF16BE); + DefInt(SQLITE_UTF16); + /*deprecated DefInt(SQLITE_ANY); */ + DefInt(SQLITE_UTF16_ALIGNED); + } _DefGroup; + + DefGroup(fcntl) { + DefInt(SQLITE_FCNTL_LOCKSTATE); + DefInt(SQLITE_FCNTL_GET_LOCKPROXYFILE); + DefInt(SQLITE_FCNTL_SET_LOCKPROXYFILE); + DefInt(SQLITE_FCNTL_LAST_ERRNO); + DefInt(SQLITE_FCNTL_SIZE_HINT); + DefInt(SQLITE_FCNTL_CHUNK_SIZE); + DefInt(SQLITE_FCNTL_FILE_POINTER); + DefInt(SQLITE_FCNTL_SYNC_OMITTED); + DefInt(SQLITE_FCNTL_WIN32_AV_RETRY); + DefInt(SQLITE_FCNTL_PERSIST_WAL); + DefInt(SQLITE_FCNTL_OVERWRITE); + DefInt(SQLITE_FCNTL_VFSNAME); + DefInt(SQLITE_FCNTL_POWERSAFE_OVERWRITE); + DefInt(SQLITE_FCNTL_PRAGMA); + DefInt(SQLITE_FCNTL_BUSYHANDLER); + DefInt(SQLITE_FCNTL_TEMPFILENAME); + DefInt(SQLITE_FCNTL_MMAP_SIZE); + DefInt(SQLITE_FCNTL_TRACE); + DefInt(SQLITE_FCNTL_HAS_MOVED); + DefInt(SQLITE_FCNTL_SYNC); + DefInt(SQLITE_FCNTL_COMMIT_PHASETWO); + DefInt(SQLITE_FCNTL_WIN32_SET_HANDLE); + DefInt(SQLITE_FCNTL_WAL_BLOCK); + DefInt(SQLITE_FCNTL_ZIPVFS); + DefInt(SQLITE_FCNTL_RBU); + DefInt(SQLITE_FCNTL_VFS_POINTER); + DefInt(SQLITE_FCNTL_JOURNAL_POINTER); + DefInt(SQLITE_FCNTL_WIN32_GET_HANDLE); + DefInt(SQLITE_FCNTL_PDB); + DefInt(SQLITE_FCNTL_BEGIN_ATOMIC_WRITE); + DefInt(SQLITE_FCNTL_COMMIT_ATOMIC_WRITE); + DefInt(SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE); + DefInt(SQLITE_FCNTL_LOCK_TIMEOUT); + DefInt(SQLITE_FCNTL_DATA_VERSION); + DefInt(SQLITE_FCNTL_SIZE_LIMIT); + DefInt(SQLITE_FCNTL_CKPT_DONE); + DefInt(SQLITE_FCNTL_RESERVE_BYTES); + DefInt(SQLITE_FCNTL_CKPT_START); + DefInt(SQLITE_FCNTL_EXTERNAL_READER); + DefInt(SQLITE_FCNTL_CKSM_FILE); + } _DefGroup; + + DefGroup(flock) { + DefInt(SQLITE_LOCK_NONE); + DefInt(SQLITE_LOCK_SHARED); + DefInt(SQLITE_LOCK_RESERVED); + DefInt(SQLITE_LOCK_PENDING); + DefInt(SQLITE_LOCK_EXCLUSIVE); + } _DefGroup; + + DefGroup(ioCap) { + DefInt(SQLITE_IOCAP_ATOMIC); + DefInt(SQLITE_IOCAP_ATOMIC512); + DefInt(SQLITE_IOCAP_ATOMIC1K); + DefInt(SQLITE_IOCAP_ATOMIC2K); + DefInt(SQLITE_IOCAP_ATOMIC4K); + DefInt(SQLITE_IOCAP_ATOMIC8K); + DefInt(SQLITE_IOCAP_ATOMIC16K); + DefInt(SQLITE_IOCAP_ATOMIC32K); + DefInt(SQLITE_IOCAP_ATOMIC64K); + DefInt(SQLITE_IOCAP_SAFE_APPEND); + DefInt(SQLITE_IOCAP_SEQUENTIAL); + DefInt(SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN); + DefInt(SQLITE_IOCAP_POWERSAFE_OVERWRITE); + DefInt(SQLITE_IOCAP_IMMUTABLE); + DefInt(SQLITE_IOCAP_BATCH_ATOMIC); + } _DefGroup; + + DefGroup(openFlags) { + /* Noting that not all of these will have any effect in + ** WASM-space. */ + DefInt(SQLITE_OPEN_READONLY); + DefInt(SQLITE_OPEN_READWRITE); + DefInt(SQLITE_OPEN_CREATE); + DefInt(SQLITE_OPEN_URI); + DefInt(SQLITE_OPEN_MEMORY); + DefInt(SQLITE_OPEN_NOMUTEX); + DefInt(SQLITE_OPEN_FULLMUTEX); + DefInt(SQLITE_OPEN_SHAREDCACHE); + DefInt(SQLITE_OPEN_PRIVATECACHE); + DefInt(SQLITE_OPEN_EXRESCODE); + DefInt(SQLITE_OPEN_NOFOLLOW); + /* OPEN flags for use with VFSes... */ + DefInt(SQLITE_OPEN_MAIN_DB); + DefInt(SQLITE_OPEN_MAIN_JOURNAL); + DefInt(SQLITE_OPEN_TEMP_DB); + DefInt(SQLITE_OPEN_TEMP_JOURNAL); + DefInt(SQLITE_OPEN_TRANSIENT_DB); + DefInt(SQLITE_OPEN_SUBJOURNAL); + DefInt(SQLITE_OPEN_SUPER_JOURNAL); + DefInt(SQLITE_OPEN_WAL); + DefInt(SQLITE_OPEN_DELETEONCLOSE); + DefInt(SQLITE_OPEN_EXCLUSIVE); + } _DefGroup; + + DefGroup(prepareFlags) { + DefInt(SQLITE_PREPARE_PERSISTENT); + DefInt(SQLITE_PREPARE_NORMALIZE); + DefInt(SQLITE_PREPARE_NO_VTAB); } _DefGroup; DefGroup(resultCodes) { @@ -114,7 +559,6 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_WARNING); DefInt(SQLITE_ROW); DefInt(SQLITE_DONE); - // Extended Result Codes DefInt(SQLITE_ERROR_MISSING_COLLSEQ); DefInt(SQLITE_ERROR_RETRY); @@ -193,60 +637,11 @@ const char * sqlite3_wasm_enum_json(void){ //DefInt(SQLITE_OK_SYMLINK) /* internal use only */; } _DefGroup; - DefGroup(dataTypes) { - DefInt(SQLITE_INTEGER); - DefInt(SQLITE_FLOAT); - DefInt(SQLITE_TEXT); - DefInt(SQLITE_BLOB); - DefInt(SQLITE_NULL); - } _DefGroup; - - DefGroup(encodings) { - /* Noting that the wasm binding only aims to support UTF-8. */ - DefInt(SQLITE_UTF8); - DefInt(SQLITE_UTF16LE); - DefInt(SQLITE_UTF16BE); - DefInt(SQLITE_UTF16); - /*deprecated DefInt(SQLITE_ANY); */ - DefInt(SQLITE_UTF16_ALIGNED); - } _DefGroup; - - DefGroup(blobFinalizers) { - /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as - ** integers to avoid casting-related warnings. */ - out("\"SQLITE_STATIC\":0, \"SQLITE_TRANSIENT\":-1"); - } _DefGroup; - - DefGroup(udfFlags) { - DefInt(SQLITE_DETERMINISTIC); - DefInt(SQLITE_DIRECTONLY); - DefInt(SQLITE_INNOCUOUS); - } _DefGroup; - - DefGroup(openFlags) { - /* Noting that not all of these will have any effect in WASM-space. */ - DefInt(SQLITE_OPEN_READONLY); - DefInt(SQLITE_OPEN_READWRITE); - DefInt(SQLITE_OPEN_CREATE); - DefInt(SQLITE_OPEN_URI); - DefInt(SQLITE_OPEN_MEMORY); - DefInt(SQLITE_OPEN_NOMUTEX); - DefInt(SQLITE_OPEN_FULLMUTEX); - DefInt(SQLITE_OPEN_SHAREDCACHE); - DefInt(SQLITE_OPEN_PRIVATECACHE); - DefInt(SQLITE_OPEN_EXRESCODE); - DefInt(SQLITE_OPEN_NOFOLLOW); - /* OPEN flags for use with VFSes... */ - DefInt(SQLITE_OPEN_MAIN_DB); - DefInt(SQLITE_OPEN_MAIN_JOURNAL); - DefInt(SQLITE_OPEN_TEMP_DB); - DefInt(SQLITE_OPEN_TEMP_JOURNAL); - DefInt(SQLITE_OPEN_TRANSIENT_DB); - DefInt(SQLITE_OPEN_SUBJOURNAL); - DefInt(SQLITE_OPEN_SUPER_JOURNAL); - DefInt(SQLITE_OPEN_WAL); - DefInt(SQLITE_OPEN_DELETEONCLOSE); - DefInt(SQLITE_OPEN_EXCLUSIVE); + DefGroup(serialize){ + DefInt(SQLITE_SERIALIZE_NOCOPY); + DefInt(SQLITE_DESERIALIZE_FREEONCLOSE); + DefInt(SQLITE_DESERIALIZE_READONLY); + DefInt(SQLITE_DESERIALIZE_RESIZEABLE); } _DefGroup; DefGroup(syncFlags) { @@ -255,42 +650,23 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_SYNC_DATAONLY); } _DefGroup; - DefGroup(prepareFlags) { - DefInt(SQLITE_PREPARE_PERSISTENT); - DefInt(SQLITE_PREPARE_NORMALIZE); - DefInt(SQLITE_PREPARE_NO_VTAB); + DefGroup(trace) { + DefInt(SQLITE_TRACE_STMT); + DefInt(SQLITE_TRACE_PROFILE); + DefInt(SQLITE_TRACE_ROW); + DefInt(SQLITE_TRACE_CLOSE); } _DefGroup; - DefGroup(flock) { - DefInt(SQLITE_LOCK_NONE); - DefInt(SQLITE_LOCK_SHARED); - DefInt(SQLITE_LOCK_RESERVED); - DefInt(SQLITE_LOCK_PENDING); - DefInt(SQLITE_LOCK_EXCLUSIVE); + DefGroup(udfFlags) { + DefInt(SQLITE_DETERMINISTIC); + DefInt(SQLITE_DIRECTONLY); + DefInt(SQLITE_INNOCUOUS); } _DefGroup; - DefGroup(ioCap) { - DefInt(SQLITE_IOCAP_ATOMIC); - DefInt(SQLITE_IOCAP_ATOMIC512); - DefInt(SQLITE_IOCAP_ATOMIC1K); - DefInt(SQLITE_IOCAP_ATOMIC2K); - DefInt(SQLITE_IOCAP_ATOMIC4K); - DefInt(SQLITE_IOCAP_ATOMIC8K); - DefInt(SQLITE_IOCAP_ATOMIC16K); - DefInt(SQLITE_IOCAP_ATOMIC32K); - DefInt(SQLITE_IOCAP_ATOMIC64K); - DefInt(SQLITE_IOCAP_SAFE_APPEND); - DefInt(SQLITE_IOCAP_SEQUENTIAL); - DefInt(SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN); - DefInt(SQLITE_IOCAP_POWERSAFE_OVERWRITE); - DefInt(SQLITE_IOCAP_IMMUTABLE); - DefInt(SQLITE_IOCAP_BATCH_ATOMIC); - } _DefGroup; - - DefGroup(access){ - DefInt(SQLITE_ACCESS_EXISTS); - DefInt(SQLITE_ACCESS_READWRITE); - DefInt(SQLITE_ACCESS_READ)/*docs say this is unused*/; + DefGroup(version) { + DefInt(SQLITE_VERSION_NUMBER); + DefStr(SQLITE_VERSION); + DefStr(SQLITE_SOURCE_ID); } _DefGroup; #undef DefGroup @@ -319,7 +695,7 @@ const char * sqlite3_wasm_enum_json(void){ /** Macros for emitting StructBinder description. */ #define StructBinder__(TYPE) \ n = 0; \ - outf("%s{", (structCount++ ? ", " : "")); \ + outf("%s{", (nStruct++ ? ", " : "")); \ out("\"name\": \"" # TYPE "\","); \ outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ out(",\"members\": {"); @@ -335,7 +711,7 @@ const char * sqlite3_wasm_enum_json(void){ (int)sizeof(((CurrentStruct*)0)->MEMBER), \ SIG) - structCount = 0; + nStruct = 0; out(", \"structs\": ["); { #define CurrentStruct sqlite3_vfs @@ -391,16 +767,37 @@ const char * sqlite3_wasm_enum_json(void){ #define CurrentStruct sqlite3_file StructBinder { - M(pMethods,"P"); + M(pMethods,"p"); } _StructBinder; #undef CurrentStruct +#define CurrentStruct sqlite3_kvvfs_methods + StructBinder { + M(xRead,"i(sspi)"); + M(xWrite,"i(sss)"); + M(xDelete,"i(ss)"); + M(nKeySize,"i"); + } _StructBinder; +#undef CurrentStruct + +#if SQLITE_WASM_TESTS +#define CurrentStruct WasmTestStruct + StructBinder { + M(v4,"i"); + M(cstr,"s"); + M(ppV,"p"); + M(v8,"j"); + M(xFunc,"v(p)"); + } _StructBinder; +#undef CurrentStruct +#endif + } out( "]"/*structs*/); out("}"/*top-level object*/); - *pos = 0; - strBuf[0] = '{'/*end of the race-condition workaround*/; - return strBuf; + *zPos = 0; + aBuffer[0] = '{'/*end of the race-condition workaround*/; + return aBuffer; #undef StructBinder #undef StructBinder_ #undef StructBinder__ @@ -411,3 +808,377 @@ const char * sqlite3_wasm_enum_json(void){ #undef outf #undef lenCheck } + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This function invokes the xDelete method of the given VFS (or the +** default VFS if pVfs is NULL), passing on the given filename. If +** zName is NULL, no default VFS is found, or it has no xDelete +** method, SQLITE_MISUSE is returned, else the result of the xDelete() +** call is returned. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char * zName){ + int rc = SQLITE_MISUSE /* ??? */; + if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); + if( zName && pVfs && pVfs->xDelete ){ + rc = pVfs->xDelete(pVfs, zName, 1); + } + return rc; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Returns a pointer to the given DB's VFS for the given DB name, +** defaulting to "main" if zDbName is 0. Returns 0 if no db with the +** given name is open. +*/ +SQLITE_WASM_KEEP +sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ + sqlite3_vfs * pVfs = 0; + sqlite3_file_control(pDb, zDbName ? zDbName : "main", + SQLITE_FCNTL_VFS_POINTER, &pVfs); + return pVfs; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This function resets the given db pointer's database as described at +** +** https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase +** +** Returns 0 on success, an SQLITE_xxx code on error. Returns +** SQLITE_MISUSE if pDb is NULL. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_reset(sqlite3*pDb){ + int rc = SQLITE_MISUSE; + if( pDb ){ + rc = sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) rc = sqlite3_exec(pDb, "VACUUM", 0, 0, 0); + sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + } + return rc; +} + +/* +** Uses the given database's VFS xRead to stream the db file's +** contents out to the given callback. The callback gets a single +** chunk of size n (its 2nd argument) on each call and must return 0 +** on success, non-0 on error. This function returns 0 on success, +** SQLITE_NOTFOUND if no db is open, or propagates any other non-0 +** code from the callback. Note that this is not thread-friendly: it +** expects that it will be the only thread reading the db file and +** takes no measures to ensure that is the case. +** +** This implementation appears to work fine, but +** sqlite3_wasm_db_serialize() is arguably the better way to achieve +** this. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_export_chunked( sqlite3* pDb, + int (*xCallback)(unsigned const char *zOut, int n) ){ + sqlite3_int64 nSize = 0; + sqlite3_int64 nPos = 0; + sqlite3_file * pFile = 0; + unsigned char buf[1024 * 8]; + int nBuf = (int)sizeof(buf); + int rc = pDb + ? sqlite3_file_control(pDb, "main", + SQLITE_FCNTL_FILE_POINTER, &pFile) + : SQLITE_NOTFOUND; + if( rc ) return rc; + rc = pFile->pMethods->xFileSize(pFile, &nSize); + if( rc ) return rc; + if(nSize % nBuf){ + /* DB size is not an even multiple of the buffer size. Reduce + ** buffer size so that we do not unduly inflate the db size + ** with zero-padding when exporting. */ + if(0 == nSize % 4096) nBuf = 4096; + else if(0 == nSize % 2048) nBuf = 2048; + else if(0 == nSize % 1024) nBuf = 1024; + else nBuf = 512; + } + for( ; 0==rc && nPospMethods->xRead(pFile, buf, nBuf, nPos); + if(SQLITE_IOERR_SHORT_READ == rc){ + rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/; + } + if( 0==rc ) rc = xCallback(buf, nBuf); + } + return rc; +} + +/* +** A proxy for sqlite3_serialize() which serializes the "main" schema +** of pDb, placing the serialized output in pOut and nOut. nOut may be +** NULL. If pDb or pOut are NULL then SQLITE_MISUSE is returned. If +** allocation of the serialized copy fails, SQLITE_NOMEM is returned. +** On success, 0 is returned and `*pOut` will contain a pointer to the +** memory unless mFlags includes SQLITE_SERIALIZE_NOCOPY and the +** database has no contiguous memory representation, in which case +** `*pOut` will be NULL but 0 will be returned. +** +** If `*pOut` is not NULL, the caller is responsible for passing it to +** sqlite3_free() to free it. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_serialize( sqlite3 *pDb, unsigned char **pOut, + sqlite3_int64 *nOut, unsigned int mFlags ){ + unsigned char * z; + if( !pDb || !pOut ) return SQLITE_MISUSE; + if(nOut) *nOut = 0; + z = sqlite3_serialize(pDb, "main", nOut, mFlags); + if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){ + *pOut = z; + return 0; + }else{ + return SQLITE_NOMEM; + } +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Creates a new file using the I/O API of the given VFS, containing +** the given number of bytes of the given data. If the file exists, +** it is truncated to the given length and populated with the given +** data. +** +** This function exists so that we can implement the equivalent of +** Emscripten's FS.createDataFile() in a VFS-agnostic way. This +** functionality is intended for use in uploading database files. +** +** If pVfs is NULL, sqlite3_vfs_find(0) is used. +** +** If zFile is NULL, pVfs is NULL (and sqlite3_vfs_find(0) returns +** NULL), or nData is negative, SQLITE_MISUSE are returned. +** +** On success, it creates a new file with the given name, populated +** with the fist nData bytes of pData. If pData is NULL, the file is +** created and/or truncated to nData bytes. +** +** Whether or not directory components of zFilename are created +** automatically or not is unspecified: that detail is left to the +** VFS. The "opfs" VFS, for example, create them. +** +** Not all VFSes support this functionality, e.g. the "kvvfs" does +** not. +** +** If an error happens while populating or truncating the file, the +** target file will be deleted (if needed) if this function created +** it. If this function did not create it, it is not deleted but may +** be left in an undefined state. +** +** Returns 0 on success. On error, it returns a code described above +** or propagates a code from one of the I/O methods. +** +** Design note: nData is an integer, instead of int64, for WASM +** portability, so that the API can still work in builds where BigInt +** support is disabled or unavailable. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, + const char *zFilename, + const unsigned char * pData, + int nData ){ + int rc; + sqlite3_file *pFile = 0; + sqlite3_io_methods const *pIo; + const int openFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + int flagsOut = 0; + int fileExisted = 0; + int doUnlock = 0; + const unsigned char *pPos = pData; + const int blockSize = 512 + /* Because we are using pFile->pMethods->xWrite() for writing, and + ** it may have a buffer limit related to sqlite3's pager size, we + ** conservatively write in 512-byte blocks (smallest page + ** size). */; + + if( !pVfs ) pVfs = sqlite3_vfs_find(0); + if( !pVfs || !zFilename || nData<0 ) return SQLITE_MISUSE; + pVfs->xAccess(pVfs, zFilename, SQLITE_ACCESS_EXISTS, &fileExisted); + rc = sqlite3OsOpenMalloc(pVfs, zFilename, &pFile, openFlags, &flagsOut); + if(rc) return rc; + pIo = pFile->pMethods; + if( pIo->xLock ) { + /* We need xLock() in order to accommodate the OPFS VFS, as it + ** obtains a writeable handle via the lock operation and releases + ** it in xUnlock(). If we don't do those here, we have to add code + ** to the VFS to account check whether it was locked before + ** xFileSize(), xTruncate(), and the like, and release the lock + ** only if it was unlocked when the op was started. */ + rc = pIo->xLock(pFile, SQLITE_LOCK_EXCLUSIVE); + doUnlock = 0==rc; + } + if( 0==rc) rc = pIo->xTruncate(pFile, nData); + if( 0==rc && 0!=pData && nData>0 ){ + while( 0==rc && nData>0 ){ + const int n = nData>=blockSize ? blockSize : nData; + rc = pIo->xWrite(pFile, pPos, n, (sqlite3_int64)(pPos - pData)); + nData -= n; + pPos += n; + } + if( 0==rc && nData>0 ){ + assert( nDataxWrite(pFile, pPos, nData, (sqlite3_int64)(pPos - pData)); + } + } + if( pIo->xUnlock && doUnlock!=0 ) pIo->xUnlock(pFile, SQLITE_LOCK_NONE); + pIo->xClose(pFile); + if( rc!=0 && 0==fileExisted ){ + pVfs->xDelete(pVfs, zFilename, 1); + } + return rc; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Allocates sqlite3KvvfsMethods.nKeySize bytes from +** sqlite3_wasm_pstack_alloc() and returns 0 if that allocation fails, +** else it passes that string to kvstorageMakeKey() and returns a +** NUL-terminated pointer to that string. It is up to the caller to +** use sqlite3_wasm_pstack_restore() to free the returned pointer. +*/ +SQLITE_WASM_KEEP +char * sqlite3_wasm_kvvfsMakeKeyOnPstack(const char *zClass, + const char *zKeyIn){ + assert(sqlite3KvvfsMethods.nKeySize>24); + char *zKeyOut = + (char *)sqlite3_wasm_pstack_alloc(sqlite3KvvfsMethods.nKeySize); + if(zKeyOut){ + kvstorageMakeKey(zClass, zKeyIn, zKeyOut); + } + return zKeyOut; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Returns the pointer to the singleton object which holds the kvvfs +** I/O methods and associated state. +*/ +SQLITE_WASM_KEEP +sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ + return &sqlite3KvvfsMethods; +} + +#if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) +#include + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings, specifically +** only when building with Emscripten's WASMFS support. +** +** This function should only be called if the JS side detects the +** existence of the Origin-Private FileSystem (OPFS) APIs in the +** client. The first time it is called, this function instantiates a +** WASMFS backend impl for OPFS. On success, subsequent calls are +** no-ops. +** +** This function may be passed a "mount point" name, which must have a +** leading "/" and is currently restricted to a single path component, +** e.g. "/foo" is legal but "/foo/" and "/foo/bar" are not. If it is +** NULL or empty, it defaults to "/opfs". +** +** Returns 0 on success, SQLITE_NOMEM if instantiation of the backend +** object fails, SQLITE_IOERR if mkdir() of the zMountPoint dir in +** the virtual FS fails. In builds compiled without SQLITE_ENABLE_WASMFS +** defined, SQLITE_NOTFOUND is returned without side effects. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ + static backend_t pOpfs = 0; + if( !zMountPoint || !*zMountPoint ) zMountPoint = "/opfs"; + if( !pOpfs ){ + pOpfs = wasmfs_create_opfs_backend(); + } + /** It's not enough to instantiate the backend. We have to create a + mountpoint in the VFS and attach the backend to it. */ + if( pOpfs && 0!=access(zMountPoint, F_OK) ){ + /* mkdir() simply hangs when called from fiddle app. Cause is + not yet determined but the hypothesis is an init-order + issue. */ + /* Note that this check and is not robust but it will + hypothetically suffice for the transient wasm-based virtual + filesystem we're currently running in. */ + const int rc = wasmfs_create_directory(zMountPoint, 0777, pOpfs); + /*emscripten_console_logf("OPFS mkdir(%s) rc=%d", zMountPoint, rc);*/ + if(rc) return SQLITE_IOERR; + } + return pOpfs ? 0 : SQLITE_NOMEM; +} +#else +SQLITE_WASM_KEEP +int sqlite3_wasm_init_wasmfs(const char *zUnused){ + //emscripten_console_warn("WASMFS OPFS is not compiled in."); + if(zUnused){/*unused*/} + return SQLITE_NOTFOUND; +} +#endif /* __EMSCRIPTEN__ && SQLITE_ENABLE_WASMFS */ + +#if SQLITE_WASM_TESTS + +SQLITE_WASM_KEEP +int sqlite3_wasm_test_intptr(int * p){ + return *p = *p * 2; +} + +SQLITE_WASM_KEEP +int64_t sqlite3_wasm_test_int64_max(void){ + return (int64_t)0x7fffffffffffffff; +} + +SQLITE_WASM_KEEP +int64_t sqlite3_wasm_test_int64_min(void){ + return ~sqlite3_wasm_test_int64_max(); +} + +SQLITE_WASM_KEEP +int64_t sqlite3_wasm_test_int64_times2(int64_t x){ + return x * 2; +} + +SQLITE_WASM_KEEP +void sqlite3_wasm_test_int64_minmax(int64_t * min, int64_t *max){ + *max = sqlite3_wasm_test_int64_max(); + *min = sqlite3_wasm_test_int64_min(); + /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ +} + +SQLITE_WASM_KEEP +int64_t sqlite3_wasm_test_int64ptr(int64_t * p){ + /*printf("sqlite3_wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ + return *p = *p * 2; +} + +SQLITE_WASM_KEEP +void sqlite3_wasm_test_stack_overflow(int recurse){ + if(recurse) sqlite3_wasm_test_stack_overflow(recurse); +} + +/* For testing the 'string-free' whwasmutil.xWrap() conversion. */ +SQLITE_WASM_KEEP +char * sqlite3_wasm_test_str_hello(int fail){ + char * s = fail ? 0 : (char *)malloc(6); + if(s){ + memcpy(s, "hello", 5); + s[5] = 0; + } + return s; +} +#endif /* SQLITE_WASM_TESTS */ + +#undef SQLITE_WASM_KEEP diff --git a/ext/wasm/api/sqlite3-worker1-promiser.js b/ext/wasm/api/sqlite3-worker1-promiser.js new file mode 100644 index 0000000000..7360512d49 --- /dev/null +++ b/ext/wasm/api/sqlite3-worker1-promiser.js @@ -0,0 +1,259 @@ +/* + 2022-08-24 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements a Promise-based proxy for the sqlite3 Worker + API #1. It is intended to be included either from the main thread or + a Worker, but only if (A) the environment supports nested Workers + and (B) it's _not_ a Worker which loads the sqlite3 WASM/JS + module. This file's features will load that module and provide a + slightly simpler client-side interface than the slightly-lower-level + Worker API does. + + This script necessarily exposes one global symbol, but clients may + freely `delete` that symbol after calling it. +*/ +'use strict'; +/** + Configures an sqlite3 Worker API #1 Worker such that it can be + manipulated via a Promise-based interface and returns a factory + function which returns Promises for communicating with the worker. + This proxy has an _almost_ identical interface to the normal + worker API, with any exceptions documented below. + + It requires a configuration object with the following properties: + + - `worker` (required): a Worker instance which loads + `sqlite3-worker1.js` or a functional equivalent. Note that the + promiser factory replaces the worker.onmessage property. This + config option may alternately be a function, in which case this + function re-assigns this property with the result of calling that + function, enabling delayed instantiation of a Worker. + + - `onready` (optional, but...): this callback is called with no + arguments when the worker fires its initial + 'sqlite3-api'/'worker1-ready' message, which it does when + sqlite3.initWorker1API() completes its initialization. This is + the simplest way to tell the worker to kick off work at the + earliest opportunity. + + - `onunhandled` (optional): a callback which gets passed the + message event object for any worker.onmessage() events which + are not handled by this proxy. Ideally that "should" never + happen, as this proxy aims to handle all known message types. + + - `generateMessageId` (optional): a function which, when passed an + about-to-be-posted message object, generates a _unique_ message ID + for the message, which this API then assigns as the messageId + property of the message. It _must_ generate unique IDs on each call + so that dispatching can work. If not defined, a default generator + is used (which should be sufficient for most or all cases). + + - `debug` (optional): a console.debug()-style function for logging + information about messages. + + This function returns a stateful factory function with the + following interfaces: + + - Promise function(messageType, messageArgs) + - Promise function({message object}) + + The first form expects the "type" and "args" values for a Worker + message. The second expects an object in the form {type:..., + args:...} plus any other properties the client cares to set. This + function will always set the `messageId` property on the object, + even if it's already set, and will set the `dbId` property to the + current database ID if it is _not_ set in the message object. + + The function throws on error. + + The function installs a temporary message listener, posts a + message to the configured Worker, and handles the message's + response via the temporary message listener. The then() callback + of the returned Promise is passed the `message.data` property from + the resulting message, i.e. the payload from the worker, stripped + of the lower-level event state which the onmessage() handler + receives. + + Example usage: + + ``` + const config = {...}; + const sq3Promiser = sqlite3Worker1Promiser(config); + sq3Promiser('open', {filename:"/foo.db"}).then(function(msg){ + console.log("open response",msg); // => {type:'open', result: {filename:'/foo.db'}, ...} + }); + sq3Promiser({type:'close'}).then((msg)=>{ + console.log("close response",msg); // => {type:'close', result: {filename:'/foo.db'}, ...} + }); + ``` + + Differences from Worker API #1: + + - exec's {callback: STRING} option does not work via this + interface (it triggers an exception), but {callback: function} + does and works exactly like the STRING form does in the Worker: + the callback is called one time for each row of the result set, + passed the same worker message format as the worker API emits: + + {type:typeString, + row:VALUE, + rowNumber:1-based-#, + columnNames: array} + + Where `typeString` is an internally-synthesized message type string + used temporarily for worker message dispatching. It can be ignored + by all client code except that which tests this API. The `row` + property contains the row result in the form implied by the + `rowMode` option (defaulting to `'array'`). The `rowNumber` is a + 1-based integer value incremented by 1 on each call into th + callback. + + At the end of the result set, the same event is fired with + (row=undefined, rowNumber=null) to indicate that + the end of the result set has been reached. Note that the rows + arrive via worker-posted messages, with all the implications + of that. +*/ +self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ + // Inspired by: https://stackoverflow.com/a/52439530 + if(1===arguments.length && 'function'===typeof arguments[0]){ + const f = config; + config = Object.assign(Object.create(null), callee.defaultConfig); + config.onready = f; + }else{ + config = Object.assign(Object.create(null), callee.defaultConfig, config); + } + const handlerMap = Object.create(null); + const noop = function(){}; + const err = config.onerror + || noop /* config.onerror is intentionally undocumented + pending finding a less ambiguous name */; + const debug = config.debug || noop; + const idTypeMap = config.generateMessageId ? undefined : Object.create(null); + const genMsgId = config.generateMessageId || function(msg){ + return msg.type+'#'+(idTypeMap[msg.type] = (idTypeMap[msg.type]||0) + 1); + }; + const toss = (...args)=>{throw new Error(args.join(' '))}; + if(!config.worker) config.worker = callee.defaultConfig.worker; + if('function'===typeof config.worker) config.worker = config.worker(); + let dbId; + config.worker.onmessage = function(ev){ + ev = ev.data; + debug('worker1.onmessage',ev); + let msgHandler = handlerMap[ev.messageId]; + if(!msgHandler){ + if(ev && 'sqlite3-api'===ev.type && 'worker1-ready'===ev.result) { + /*fired one time when the Worker1 API initializes*/ + if(config.onready) config.onready(); + return; + } + msgHandler = handlerMap[ev.type] /* check for exec per-row callback */; + if(msgHandler && msgHandler.onrow){ + msgHandler.onrow(ev); + return; + } + if(config.onunhandled) config.onunhandled(arguments[0]); + else err("sqlite3Worker1Promiser() unhandled worker message:",ev); + return; + } + delete handlerMap[ev.messageId]; + switch(ev.type){ + case 'error': + msgHandler.reject(ev); + return; + case 'open': + if(!dbId) dbId = ev.dbId; + break; + case 'close': + if(ev.dbId===dbId) dbId = undefined; + break; + default: + break; + } + try {msgHandler.resolve(ev)} + catch(e){msgHandler.reject(e)} + }/*worker.onmessage()*/; + return function(/*(msgType, msgArgs) || (msgEnvelope)*/){ + let msg; + if(1===arguments.length){ + msg = arguments[0]; + }else if(2===arguments.length){ + msg = { + type: arguments[0], + args: arguments[1] + }; + }else{ + toss("Invalid arugments for sqlite3Worker1Promiser()-created factory."); + } + if(!msg.dbId) msg.dbId = dbId; + msg.messageId = genMsgId(msg); + msg.departureTime = performance.now(); + const proxy = Object.create(null); + proxy.message = msg; + let rowCallbackId /* message handler ID for exec on-row callback proxy */; + if('exec'===msg.type && msg.args){ + if('function'===typeof msg.args.callback){ + rowCallbackId = msg.messageId+':row'; + proxy.onrow = msg.args.callback; + msg.args.callback = rowCallbackId; + handlerMap[rowCallbackId] = proxy; + }else if('string' === typeof msg.args.callback){ + toss("exec callback may not be a string when using the Promise interface."); + /** + Design note: the reason for this limitation is that this + API takes over worker.onmessage() and the client has no way + of adding their own message-type handlers to it. Per-row + callbacks are implemented as short-lived message.type + mappings for worker.onmessage(). + + We "could" work around this by providing a new + config.fallbackMessageHandler (or some such) which contains + a map of event type names to callbacks. Seems like overkill + for now, seeing as the client can pass callback functions + to this interface (whereas the string-form "callback" is + needed for the over-the-Worker interface). + */ + } + } + //debug("requestWork", msg); + let p = new Promise(function(resolve, reject){ + proxy.resolve = resolve; + proxy.reject = reject; + handlerMap[msg.messageId] = proxy; + debug("Posting",msg.type,"message to Worker dbId="+(dbId||'default')+':',msg); + config.worker.postMessage(msg); + }); + if(rowCallbackId) p = p.finally(()=>delete handlerMap[rowCallbackId]); + return p; + }; +}/*sqlite3Worker1Promiser()*/; +self.sqlite3Worker1Promiser.defaultConfig = { + worker: function(){ + let theJs = "sqlite3-worker1.js"; + if(this.currentScript){ + const src = this.currentScript.src.split('/'); + src.pop(); + theJs = src.join('/')+'/' + theJs; + //console.warn("promiser currentScript, theJs =",this.currentScript,theJs); + }else{ + //console.warn("promiser self.location =",self.location); + const urlParams = new URL(self.location.href).searchParams; + if(urlParams.has('sqlite3.dir')){ + theJs = urlParams.get('sqlite3.dir') + '/' + theJs; + } + } + return new Worker(theJs + self.location.search); + }.bind({ + currentScript: self?.document?.currentScript + }), + onerror: (...args)=>console.error('worker1 promiser error',...args) +}; diff --git a/ext/wasm/api/sqlite3-worker.js b/ext/wasm/api/sqlite3-worker1.js similarity index 62% rename from ext/wasm/api/sqlite3-worker.js rename to ext/wasm/api/sqlite3-worker1.js index 48797de8ab..942437908f 100644 --- a/ext/wasm/api/sqlite3-worker.js +++ b/ext/wasm/api/sqlite3-worker1.js @@ -14,7 +14,7 @@ sqlite3.js, initializes the module, and postMessage()'s a message after the module is initialized: - {type: 'sqlite3-api', data: 'worker-ready'} + {type: 'sqlite3-api', result: 'worker1-ready'} This seemingly superfluous level of indirection is necessary when loading sqlite3.js via a Worker. Instantiating a worker with new @@ -25,7 +25,25 @@ Worker-specific API needs to pass _this_ file (or equivalent) to the Worker constructor and then listen for an event in the form shown above in order to know when the module has completed initialization. + + This file accepts a URL arguments to adjust how it loads sqlite3.js: + + - `sqlite3.dir`, if set, treats the given directory name as the + directory from which `sqlite3.js` will be loaded. */ "use strict"; -importScripts('sqlite3.js'); -sqlite3InitModule().then((EmscriptenModule)=>EmscriptenModule.sqlite3.initWorkerAPI()); +(()=>{ + const urlParams = new URL(self.location.href).searchParams; + let theJs = 'sqlite3.js'; + if(urlParams.has('sqlite3.dir')){ + theJs = urlParams.get('sqlite3.dir') + '/' + theJs; + } + //console.warn("worker1 theJs =",theJs); + importScripts(theJs); + sqlite3InitModule().then((sqlite3)=>{ + if(sqlite3.capi.sqlite3_wasmfs_opfs_dir){ + sqlite3.capi.sqlite3_wasmfs_opfs_dir(); + } + sqlite3.initWorker1API(); + }); +})(); diff --git a/ext/wasm/batch-runner.html b/ext/wasm/batch-runner.html new file mode 100644 index 0000000000..5258f9597e --- /dev/null +++ b/ext/wasm/batch-runner.html @@ -0,0 +1,90 @@ + + + + + + + + + sqlite3-api batch SQL runner + + +
sqlite3-api batch SQL runner
+ +
+ +
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
+

+ This page is for batch-running extracts from the output + of speedtest1 --script, as well as other standalone SQL + scripts. +

+

ACHTUNG: this file requires a generated input list + file. Run "make batch" from this directory to generate it. +

+ + +
+
+
+ + + + + + + + + + + +
+ +
+ + + + +
+ + + + + + diff --git a/ext/wasm/batch-runner.js b/ext/wasm/batch-runner.js new file mode 100644 index 0000000000..11c43217ff --- /dev/null +++ b/ext/wasm/batch-runner.js @@ -0,0 +1,588 @@ +/* + 2022-08-29 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic batch SQL runner for sqlite3-api.js. This file must be run in + main JS thread and sqlite3.js must have been loaded before it. +*/ +'use strict'; +(function(){ + const toss = function(...args){throw new Error(args.join(' '))}; + const warn = console.warn.bind(console); + let sqlite3; + const urlParams = new URL(self.location.href).searchParams; + const cacheSize = (()=>{ + if(urlParams.has('cachesize')) return +urlParams.get('cachesize'); + return 200; + })(); + + /** Throws if the given sqlite3 result code is not 0. */ + const checkSqliteRc = (dbh,rc)=>{ + if(rc) toss("Prepare failed:",sqlite3.capi.sqlite3_errmsg(dbh)); + }; + + const sqlToDrop = [ + "SELECT type,name FROM sqlite_schema ", + "WHERE name NOT LIKE 'sqlite\\_%' escape '\\' ", + "AND name NOT LIKE '\\_%' escape '\\'" + ].join(''); + + const clearDbWebSQL = function(db){ + db.handle.transaction(function(tx){ + const onErr = (e)=>console.error(e); + const callback = function(tx, result){ + const rows = result.rows; + let i, n; + i = n = rows.length; + while(i--){ + const row = rows.item(i); + const name = JSON.stringify(row.name); + const type = row.type; + switch(type){ + case 'index': case 'table': + case 'trigger': case 'view': { + const sql2 = 'DROP '+type+' '+name; + tx.executeSql(sql2, [], ()=>{}, onErr); + break; + } + default: + warn("Unhandled db entry type:",type,'name =',name); + break; + } + } + }; + tx.executeSql(sqlToDrop, [], callback, onErr); + db.handle.changeVersion(db.handle.version, "", ()=>{}, onErr, ()=>{}); + }); + }; + + const clearDbSqlite = function(db){ + // This would be SO much easier with the oo1 API, but we specifically want to + // inject metrics we can't get via that API, and we cannot reliably (OPFS) + // open the same DB twice to clear it using that API, so... + const rc = sqlite3.wasm.exports.sqlite3_wasm_db_reset(db.handle); + App.logHtml("reset db rc =",rc,db.id, db.filename); + }; + + + const E = (s)=>document.querySelector(s); + const App = { + e: { + output: E('#test-output'), + selSql: E('#sql-select'), + btnRun: E('#sql-run'), + btnRunNext: E('#sql-run-next'), + btnRunRemaining: E('#sql-run-remaining'), + btnExportMetrics: E('#export-metrics'), + btnClear: E('#output-clear'), + btnReset: E('#db-reset'), + cbReverseLog: E('#cb-reverse-log-order'), + selImpl: E('#select-impl'), + fsToolbar: E('#toolbar') + }, + db: Object.create(null), + dbs: Object.create(null), + cache:{}, + log: console.log.bind(console), + warn: console.warn.bind(console), + cls: function(){this.e.output.innerHTML = ''}, + logHtml2: function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + this.e.output.append(ln); + //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); + }, + logHtml: function(...args){ + console.log(...args); + if(1) this.logHtml2('', ...args); + }, + logErr: function(...args){ + console.error(...args); + if(1) this.logHtml2('error', ...args); + }, + + execSql: async function(name,sql){ + const db = this.getSelectedDb(); + const banner = "========================================"; + this.logHtml(banner, + "Running",name,'('+sql.length,'bytes) using',db.id); + const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm; + let pStmt = 0, pSqlBegin; + const stack = wasm.scopedAllocPush(); + const metrics = db.metrics = Object.create(null); + metrics.prepTotal = metrics.stepTotal = 0; + metrics.stmtCount = 0; + metrics.malloc = 0; + metrics.strcpy = 0; + this.blockControls(true); + if(this.gotErr){ + this.logErr("Cannot run SQL: error cleanup is pending."); + return; + } + // Run this async so that the UI can be updated for the above header... + const endRun = ()=>{ + metrics.evalSqlEnd = performance.now(); + metrics.evalTimeTotal = (metrics.evalSqlEnd - metrics.evalSqlStart); + this.logHtml(db.id,"metrics:",JSON.stringify(metrics, undefined, ' ')); + this.logHtml("prepare() count:",metrics.stmtCount); + this.logHtml("Time in prepare_v2():",metrics.prepTotal,"ms", + "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())"); + this.logHtml("Time in step():",metrics.stepTotal,"ms", + "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())"); + this.logHtml("Total runtime:",metrics.evalTimeTotal,"ms"); + this.logHtml("Overhead (time - prep - step):", + (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms"); + this.logHtml(banner,"End of",name); + }; + + let runner; + if('websql'===db.id){ + const who = this; + runner = function(resolve, reject){ + /* WebSQL cannot execute multiple statements, nor can it execute SQL without + an explicit transaction. Thus we have to do some fragile surgery on the + input SQL. Since we're only expecting carefully curated inputs, the hope is + that this will suffice. PS: it also can't run most SQL functions, e.g. even + instr() results in "not authorized". */ + if('string'!==typeof sql){ // assume TypedArray + sql = new TextDecoder().decode(sql); + } + sql = sql.replace(/-- [^\n]+\n/g,''); // comment lines interfere with our split() + const sqls = sql.split(/;+\n/); + const rxBegin = /^BEGIN/i, rxCommit = /^COMMIT/i; + try { + const nextSql = ()=>{ + let x = sqls.shift(); + while(sqls.length && !x) x = sqls.shift(); + return x && x.trim(); + }; + const who = this; + const transaction = function(tx){ + try { + let s; + /* Try to approximate the spirit of the input scripts + by running batches bound by BEGIN/COMMIT statements. */ + for(s = nextSql(); !!s; s = nextSql()){ + if(rxBegin.test(s)) continue; + else if(rxCommit.test(s)) break; + //console.log("websql sql again",sqls.length, s); + ++metrics.stmtCount; + const t = performance.now(); + tx.executeSql(s,[], ()=>{}, (t,e)=>{ + console.error("WebSQL error",e,"SQL =",s); + who.logErr(e.message); + //throw e; + return false; + }); + metrics.stepTotal += performance.now() - t; + } + }catch(e){ + who.logErr("transaction():",e.message); + throw e; + } + }; + const n = sqls.length; + const nextBatch = function(){ + if(sqls.length){ + console.log("websql sqls.length",sqls.length,'of',n); + db.handle.transaction(transaction, (e)=>{ + who.logErr("Ignoring and contiuing:",e.message) + //reject(e); + return false; + }, nextBatch); + }else{ + resolve(who); + } + }; + metrics.evalSqlStart = performance.now(); + nextBatch(); + }catch(e){ + //this.gotErr = e; + console.error("websql error:",e); + who.logErr(e.message); + //reject(e); + } + }.bind(this); + }else{/*sqlite3 db...*/ + runner = function(resolve, reject){ + metrics.evalSqlStart = performance.now(); + try { + let t; + let sqlByteLen = sql.byteLength; + const [ppStmt, pzTail] = wasm.scopedAllocPtr(2); + t = performance.now(); + pSqlBegin = wasm.scopedAlloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed"); + metrics.malloc = performance.now() - t; + metrics.byteLength = sqlByteLen; + let pSql = pSqlBegin; + const pSqlEnd = pSqlBegin + sqlByteLen; + t = performance.now(); + wasm.heap8().set(sql, pSql); + wasm.setMemValue(pSql + sqlByteLen, 0); + metrics.strcpy = performance.now() - t; + let breaker = 0; + while(pSql && wasm.getMemValue(pSql,'i8')){ + wasm.setPtrValue(ppStmt, 0); + wasm.setPtrValue(pzTail, 0); + t = performance.now(); + let rc = capi.sqlite3_prepare_v3( + db.handle, pSql, sqlByteLen, 0, ppStmt, pzTail + ); + metrics.prepTotal += performance.now() - t; + checkSqliteRc(db.handle, rc); + pStmt = wasm.getPtrValue(ppStmt); + pSql = wasm.getPtrValue(pzTail); + sqlByteLen = pSqlEnd - pSql; + if(!pStmt) continue/*empty statement*/; + ++metrics.stmtCount; + t = performance.now(); + rc = capi.sqlite3_step(pStmt); + capi.sqlite3_finalize(pStmt); + pStmt = 0; + metrics.stepTotal += performance.now() - t; + switch(rc){ + case capi.SQLITE_ROW: + case capi.SQLITE_DONE: break; + default: checkSqliteRc(db.handle, rc); toss("Not reached."); + } + } + resolve(this); + }catch(e){ + if(pStmt) capi.sqlite3_finalize(pStmt); + //this.gotErr = e; + reject(e); + }finally{ + capi.sqlite3_exec(db.handle,"rollback;",0,0,0); + wasm.scopedAllocPop(stack); + } + }.bind(this); + } + let p; + if(1){ + p = new Promise(function(res,rej){ + setTimeout(()=>runner(res, rej), 50)/*give UI a chance to output the "running" banner*/; + }); + }else{ + p = new Promise(runner); + } + return p.catch( + (e)=>this.logErr("Error via execSql("+name+",...):",e.message) + ).finally(()=>{ + endRun(); + this.blockControls(false); + }); + }, + + clearDb: function(){ + const db = this.getSelectedDb(); + if('websql'===db.id){ + this.logErr("TODO: clear websql db."); + return; + } + if(!db.handle) return; + const capi = this.sqlite3, wasm = this.sqlite3.wasm; + //const scope = wasm.scopedAllocPush( + this.logErr("TODO: clear db"); + }, + + /** + Loads batch-runner.list and populates the selection list from + it. Returns a promise which resolves to nothing in particular + when it completes. Only intended to be run once at the start + of the app. + */ + loadSqlList: async function(){ + const sel = this.e.selSql; + sel.innerHTML = ''; + this.blockControls(true); + const infile = 'batch-runner.list'; + this.logHtml("Loading list of SQL files:", infile); + let txt; + try{ + const r = await fetch(infile); + if(404 === r.status){ + toss("Missing file '"+infile+"'."); + } + if(!r.ok) toss("Loading",infile,"failed:",r.statusText); + txt = await r.text(); + const warning = E('#warn-list'); + if(warning) warning.remove(); + }catch(e){ + this.logErr(e.message); + throw e; + }finally{ + this.blockControls(false); + } + const list = txt.split(/\n+/); + let opt; + if(0){ + opt = document.createElement('option'); + opt.innerText = "Select file to evaluate..."; + opt.value = ''; + opt.disabled = true; + opt.selected = true; + sel.appendChild(opt); + } + list.forEach(function(fn){ + if(!fn) return; + opt = document.createElement('option'); + opt.value = fn; + opt.innerText = fn.split('/').pop(); + sel.appendChild(opt); + }); + this.logHtml("Loaded",infile); + }, + + /** Fetch ./fn and return its contents as a Uint8Array. */ + fetchFile: async function(fn, cacheIt=false){ + if(cacheIt && this.cache[fn]) return this.cache[fn]; + this.logHtml("Fetching",fn,"..."); + let sql; + try { + const r = await fetch(fn); + if(!r.ok) toss("Fetch failed:",r.statusText); + sql = new Uint8Array(await r.arrayBuffer()); + }catch(e){ + this.logErr(e.message); + throw e; + } + this.logHtml("Fetched",sql.length,"bytes from",fn); + if(cacheIt) this.cache[fn] = sql; + return sql; + }/*fetchFile()*/, + + /** Disable or enable certain UI controls. */ + blockControls: function(disable){ + //document.querySelectorAll('.disable-during-eval').forEach((e)=>e.disabled = disable); + this.e.fsToolbar.disabled = disable; + }, + + /** + Converts this.metrics() to a form which is suitable for easy conversion to + CSV. It returns an array of arrays. The first sub-array is the column names. + The 2nd and subsequent are the values, one per test file (only the most recent + metrics are kept for any given file). + */ + metricsToArrays: function(){ + const rc = []; + Object.keys(this.dbs).sort().forEach((k)=>{ + const d = this.dbs[k]; + const m = d.metrics; + delete m.evalSqlStart; + delete m.evalSqlEnd; + const mk = Object.keys(m).sort(); + if(!rc.length){ + rc.push(['db', ...mk]); + } + const row = [k.split('/').pop()/*remove dir prefix from filename*/]; + rc.push(row); + row.push(...mk.map((kk)=>m[kk])); + }); + return rc; + }, + + metricsToBlob: function(colSeparator='\t'){ + const ar = [], ma = this.metricsToArrays(); + if(!ma.length){ + this.logErr("Metrics are empty. Run something."); + return; + } + ma.forEach(function(row){ + ar.push(row.join(colSeparator),'\n'); + }); + return new Blob(ar); + }, + + downloadMetrics: function(){ + const b = this.metricsToBlob(); + if(!b) return; + const url = URL.createObjectURL(b); + const a = document.createElement('a'); + a.href = url; + a.download = 'batch-runner-js-'+((new Date().getTime()/1000) | 0)+'.csv'; + this.logHtml("Triggering download of",a.download); + document.body.appendChild(a); + a.click(); + setTimeout(()=>{ + document.body.removeChild(a); + URL.revokeObjectURL(url); + }, 500); + }, + + /** + Fetch file fn and eval it as an SQL blob. This is an async + operation and returns a Promise which resolves to this + object on success. + */ + evalFile: async function(fn){ + const sql = await this.fetchFile(fn); + return this.execSql(fn,sql); + }/*evalFile()*/, + + /** + Clears all DB tables in all _opened_ databases. Because of + disparities between backends, we cannot simply "unlink" the + databases to clean them up. + */ + clearStorage: function(onlySelectedDb=false){ + const list = onlySelectedDb + ? [('boolean'===typeof onlySelectedDb) + ? this.dbs[this.e.selImpl.value] + : onlySelectedDb] + : Object.values(this.dbs); + for(let db of list){ + if(db && db.handle){ + this.logHtml("Clearing db",db.id); + db.clear(); + } + } + }, + + /** + Fetches the handle of the db associated with + this.e.selImpl.value, opening it if needed. + */ + getSelectedDb: function(){ + if(!this.dbs.memdb){ + for(let opt of this.e.selImpl.options){ + const d = this.dbs[opt.value] = Object.create(null); + d.id = opt.value; + switch(d.id){ + case 'virtualfs': + d.filename = 'file:/virtualfs.sqlite3?vfs=unix-none'; + break; + case 'memdb': + d.filename = ':memory:'; + break; + case 'wasmfs-opfs': + d.filename = 'file:'+( + this.sqlite3.capi.sqlite3_wasmfs_opfs_dir() + )+'/wasmfs-opfs.sqlite3b'; + break; + case 'websql': + d.filename = 'websql.db'; + break; + default: + this.logErr("Unhandled db selection option (see details in the console).",opt); + toss("Unhandled db init option"); + } + } + }/*first-time init*/ + const dbId = this.e.selImpl.value; + const d = this.dbs[dbId]; + if(d.handle) return d; + if('websql' === dbId){ + d.handle = self.openDatabase('batch-runner', '0.1', 'foo', 1024 * 1024 * 50); + d.clear = ()=>clearDbWebSQL(d); + d.handle.transaction(function(tx){ + tx.executeSql("PRAGMA cache_size="+cacheSize); + App.logHtml(dbId,"cache_size =",cacheSize); + }); + }else{ + const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm; + const stack = wasm.scopedAllocPush(); + let pDb = 0; + try{ + const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + const ppDb = wasm.scopedAllocPtr(); + const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, null); + pDb = wasm.getPtrValue(ppDb) + if(rc) toss("sqlite3_open_v2() failed with code",rc); + capi.sqlite3_exec(pDb, "PRAGMA cache_size="+cacheSize, 0, 0, 0); + this.logHtml(dbId,"cache_size =",cacheSize); + }catch(e){ + if(pDb) capi.sqlite3_close_v2(pDb); + }finally{ + wasm.scopedAllocPop(stack); + } + d.handle = pDb; + d.clear = ()=>clearDbSqlite(d); + } + d.clear(); + this.logHtml("Opened db:",dbId,d.filename); + console.log("db =",d); + return d; + }, + + run: function(sqlite3){ + delete this.run; + this.sqlite3 = sqlite3; + const capi = sqlite3.capi, wasm = sqlite3.wasm; + this.logHtml("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + this.logHtml("WASM heap size =",wasm.heap8().length); + this.loadSqlList(); + if(capi.sqlite3_wasmfs_opfs_dir()){ + E('#warn-opfs').classList.remove('hidden'); + }else{ + E('#warn-opfs').remove(); + E('option[value=wasmfs-opfs]').disabled = true; + } + if('function' === typeof self.openDatabase){ + E('#warn-websql').classList.remove('hidden'); + }else{ + E('option[value=websql]').disabled = true; + E('#warn-websql').remove(); + } + const who = this; + if(this.e.cbReverseLog.checked){ + this.e.output.classList.add('reverse'); + } + this.e.cbReverseLog.addEventListener('change', function(){ + who.e.output.classList[this.checked ? 'add' : 'remove']('reverse'); + }, false); + this.e.btnClear.addEventListener('click', ()=>this.cls(), false); + this.e.btnRun.addEventListener('click', function(){ + if(!who.e.selSql.value) return; + who.evalFile(who.e.selSql.value); + }, false); + this.e.btnRunNext.addEventListener('click', function(){ + ++who.e.selSql.selectedIndex; + if(!who.e.selSql.value) return; + who.evalFile(who.e.selSql.value); + }, false); + this.e.btnReset.addEventListener('click', function(){ + who.clearStorage(true); + }, false); + this.e.btnExportMetrics.addEventListener('click', function(){ + who.logHtml2('warning',"Triggering download of metrics CSV. Check your downloads folder."); + who.downloadMetrics(); + //const m = who.metricsToArrays(); + //console.log("Metrics:",who.metrics, m); + }); + this.e.selImpl.addEventListener('change', function(){ + who.getSelectedDb(); + }); + this.e.btnRunRemaining.addEventListener('click', async function(){ + let v = who.e.selSql.value; + const timeStart = performance.now(); + while(v){ + await who.evalFile(v); + if(who.gotError){ + who.logErr("Error handling script",v,":",who.gotError.message); + break; + } + ++who.e.selSql.selectedIndex; + v = who.e.selSql.value; + } + const timeTotal = performance.now() - timeStart; + who.logHtml("Run-remaining time:",timeTotal,"ms ("+(timeTotal/1000/60)+" minute(s))"); + who.clearStorage(); + }, false); + }/*run()*/ + }/*App*/; + + self.sqlite3TestModule.initSqlite3().then(function(sqlite3_){ + sqlite3 = sqlite3_; + self.App = App /* only to facilitate dev console access */; + App.run(sqlite3); + }); +})(); diff --git a/ext/wasm/common/SqliteTestUtil.js b/ext/wasm/common/SqliteTestUtil.js index c7c99240e6..5ed423785b 100644 --- a/ext/wasm/common/SqliteTestUtil.js +++ b/ext/wasm/common/SqliteTestUtil.js @@ -113,6 +113,46 @@ ++this.counter; if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); return this; + }, + + /** + Parses window.location.search-style string into an object + containing key/value pairs of URL arguments (already + urldecoded). The object is created using Object.create(null), + so contains only parsed-out properties and has no prototype + (and thus no inherited properties). + + If the str argument is not passed (arguments.length==0) then + window.location.search.substring(1) is used by default. If + neither str is passed in nor window exists then false is returned. + + On success it returns an Object containing the key/value pairs + parsed from the string. Keys which have no value are treated + has having the boolean true value. + + Pedantic licensing note: this code has appeared in other source + trees, but was originally written by the same person who pasted + it into those trees. + */ + processUrlArgs: function(str) { + if( 0 === arguments.length ) { + if( ('undefined' === typeof window) || + !window.location || + !window.location.search ) return false; + else str = (''+window.location.search).substring(1); + } + if( ! str ) return false; + str = (''+str).split(/#/,2)[0]; // remove #... to avoid it being added as part of the last value. + const args = Object.create(null); + const sp = str.split(/&+/); + const rx = /^([^=]+)(=(.+))?/; + var i, m; + for( i in sp ) { + m = rx.exec( sp[i] ); + if( ! m ) continue; + args[decodeURIComponent(m[1])] = (m[3] ? decodeURIComponent(m[3]) : true); + } + return args; } }; @@ -122,23 +162,24 @@ sqlite3InitModule() factory function. */ self.sqlite3TestModule = { + /** + Array of functions to call after Emscripten has initialized the + wasm module. Each gets passed the Emscripten module object + (which is _this_ object). + */ postRun: [ /* function(theModule){...} */ ], //onRuntimeInitialized: function(){}, /* Proxy for C-side stdout output. */ - print: function(){ - console.log.apply(console, Array.prototype.slice.call(arguments)); - }, + print: (...args)=>{console.log(...args)}, /* Proxy for C-side stderr output. */ - printErr: function(){ - console.error.apply(console, Array.prototype.slice.call(arguments)); - }, + printErr: (...args)=>{console.error(...args)}, /** - Called by the module init bits to report loading - progress. It gets passed an empty argument when loading is - done (after onRuntimeInitialized() and any this.postRun - callbacks have been run). + Called by the Emscripten module init bits to report loading + progress. It gets passed an empty argument when loading is done + (after onRuntimeInitialized() and any this.postRun callbacks + have been run). */ setStatus: function f(text){ if(!f.last){ @@ -168,6 +209,28 @@ } f.ui.status.classList.add('hidden'); } + }, + /** + Config options used by the Emscripten-dependent initialization + which happens via this.initSqlite3(). This object gets + (indirectly) passed to sqlite3ApiBootstrap() to configure the + sqlite3 API. + */ + sqlite3ApiConfig: { + wasmfsOpfsDir: "/opfs" + }, + /** + Intended to be called by apps which need to call the + Emscripten-installed sqlite3InitModule() routine. This function + temporarily installs this.sqlite3ApiConfig into the self + object, calls it sqlite3InitModule(), and removes + self.sqlite3ApiConfig after initialization is done. Returns the + promise from sqlite3InitModule(), and the next then() handler + will get the sqlite3 API object as its argument. + */ + initSqlite3: function(){ + self.sqlite3ApiConfig = this.sqlite3ApiConfig; + return self.sqlite3InitModule(this).finally(()=>delete self.sqlite3ApiConfig); } }; })(self/*window or worker*/); diff --git a/ext/wasm/common/testing.css b/ext/wasm/common/testing.css index 09c570f48a..9438b330c9 100644 --- a/ext/wasm/common/testing.css +++ b/ext/wasm/common/testing.css @@ -1,3 +1,8 @@ +body { + display: flex; + flex-direction: column; + flex-wrap: wrap; +} textarea { font-family: monospace; } @@ -29,4 +34,30 @@ span.labeled-input { color: red; background-color: yellow; } -#test-output { font-family: monospace } +.strong { font-weight: 700 } +.warning { color: firebrick; } +.green { color: darkgreen; } +.tests-pass { background-color: green; color: white } +.tests-fail { background-color: red; color: yellow } +.faded { opacity: 0.5; } +.group-start { color: blue; } +.group-end { color: blue; } +.input-wrapper { + white-space: nowrap; + display: flex; + align-items: center; +} +#test-output { + border: 1px inset; + border-radius: 0.25em; + padding: 0.25em; + /*max-height: 30em;*/ + overflow: auto; + white-space: break-spaces; + display: flex; flex-direction: column; + font-family: monospace; +} +#test-output.reverse { + flex-direction: column-reverse; +} +label[for] { cursor: pointer } diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 5a1d425caf..7e5e7981f7 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -15,11 +15,17 @@ https://fossil.wanderinghorse.net/r/jaccwabyt + and sqlite3: + + https://sqlite.org + + This file is kept in sync between both of those trees. + Maintenance reminder: If you're reading this in a tree other than - the Jaccwabyt tree, note that this copy may be replaced with + one of those listed above, note that this copy may be replaced with upstream copies of that one from time to time. Thus the code - installed by this function "should not" be edited outside of that - project, else it risks getting overwritten. + installed by this function "should not" be edited outside of those + projects, else it risks getting overwritten. */ /** This function is intended to simplify porting around various bits @@ -63,7 +69,7 @@ - WASM-exported "indirect function table" access and manipulation. e.g. creating new WASM-side functions using JS functions, analog to Emscripten's addFunction() and - removeFunction() but slightly different. + uninstallFunction() but slightly different. - Get/set specific heap memory values, analog to Emscripten's getValue() and setValue(). @@ -109,8 +115,8 @@ following symbols: - `memory`: a WebAssembly.Memory object representing the WASM - memory. _Alternately_, the `memory` property can be set on the - target instance, in particular if the WASM heap memory is + memory. _Alternately_, the `memory` property can be set as + `target.memory`, in particular if the WASM heap memory is initialized in JS an _imported_ into WASM, as opposed to being initialized in WASM and exported to JS. @@ -132,7 +138,11 @@ false. If it is false, certain BigInt-related features will trigger an exception if invoked. This property, if not set when this is called, will get a default value of true only if the BigInt64Array - constructor is available, else it will default to false. + constructor is available, else it will default to false. Note that + having the BigInt type is not sufficient for full int64 integration + with WASM: the target WASM file must also have been built with + that support. In Emscripten that's done using the `-sWASM_BIGINT` + flag. Some optional APIs require that the target have the following methods: @@ -212,9 +222,10 @@ self.WhWasmUtilInstaller = function(target){ that will certainly change. */ const ptrIR = target.pointerIR || 'i32'; - const ptrSizeof = ('i32'===ptrIR ? 4 - : ('i64'===ptrIR - ? 8 : toss("Unhandled ptrSizeof:",ptrIR))); + const ptrSizeof = target.ptrSizeof = + ('i32'===ptrIR ? 4 + : ('i64'===ptrIR + ? 8 : toss("Unhandled ptrSizeof:",ptrIR))); /** Stores various cached state. */ const cache = Object.create(null); /** Previously-recorded size of cache.memory.buffer, noted so that @@ -294,7 +305,7 @@ self.WhWasmUtilInstaller = function(target){ an integer as the first argument and unsigned is truthy then the "U" (unsigned) variant of that view is returned, else the signed variant is returned. If passed a TypedArray value, the - 2nd argument is ignores. Note that Float32Array and + 2nd argument is ignored. Note that Float32Array and Float64Array views are not supported by this function. Note that growth of the heap will invalidate any references to @@ -326,7 +337,7 @@ self.WhWasmUtilInstaller = function(target){ if(c.HEAP64) return unsigned ? c.HEAP64U : c.HEAP64; break; default: - if(this.bigIntEnabled){ + if(target.bigIntEnabled){ if(n===self['BigUint64Array']) return c.HEAP64U; else if(n===self['BigInt64Array']) return c.HEAP64; break; @@ -334,7 +345,7 @@ self.WhWasmUtilInstaller = function(target){ } toss("Invalid heapForSize() size: expecting 8, 16, 32,", "or (if BigInt is enabled) 64."); - }.bind(target); + }; /** Returns the WASM-exported "indirect function table." @@ -346,31 +357,33 @@ self.WhWasmUtilInstaller = function(target){ - Use `__indirect_function_table` as the import name for the table, which is what LLVM does. */ - }.bind(target); + }; /** Given a function pointer, returns the WASM function table entry if found, else returns a falsy value. */ target.functionEntry = function(fptr){ - const ft = this.functionTable(); + const ft = target.functionTable(); return fptr < ft.length ? ft.get(fptr) : undefined; - }.bind(target); + }; /** Creates a WASM function which wraps the given JS function and returns the JS binding of that WASM function. The signature - argument must be the Jaccwabyt-format or Emscripten + string must be the Jaccwabyt-format or Emscripten addFunction()-format function signature string. In short: in may have one of the following formats: - - Emscripten: `x...`, where the first x is a letter representing + - Emscripten: `"x..."`, where the first x is a letter representing the result type and subsequent letters represent the argument - types. See below. + types. Functions with no arguments have only a single + letter. See below. - - Jaccwabyt: `x(...)` where `x` is the letter representing the + - Jaccwabyt: `"x(...)"` where `x` is the letter representing the result type and letters in the parens (if any) represent the - argument types. See below. + argument types. Functions with no arguments use `x()`. See + below. Supported letters: @@ -391,6 +404,9 @@ self.WhWasmUtilInstaller = function(target){ Sidebar: this code is developed together with Jaccwabyt, thus the support for its signature format. + + The arguments may be supplied in either order: (func,sig) or + (sig,func). */ target.jsFuncToWasm = function f(func, sig){ /** Attribution: adapted up from Emscripten-generated glue code, @@ -399,9 +415,14 @@ self.WhWasmUtilInstaller = function(target){ if(!f._){/*static init...*/ f._ = { // Map of signature letters to type IR values - sigTypes: Object.create(null), + sigTypes: Object.assign(Object.create(null),{ + i: 'i32', p: 'i32', P: 'i32', s: 'i32', + j: 'i64', f: 'f32', d: 'f64' + }), // Map of type IR values to WASM type code values - typeCodes: Object.create(null), + typeCodes: Object.assign(Object.create(null),{ + f64: 0x7c, f32: 0x7d, i64: 0x7e, i32: 0x7f + }), /** Encodes n, which must be <2^14 (16384), into target array tgt, as a little-endian value, using the given method ('push' or 'unshift'). */ @@ -430,9 +451,9 @@ self.WhWasmUtilInstaller = function(target){ // is not yet documented on MDN. sigToWasm: function(sig){ const rc = {parameters:[], results: []}; - if('v'!==sig[0]) rc.results.push(f._.letterType(sig[0])); + if('v'!==sig[0]) rc.results.push(f.sigTypes(sig[0])); for(const x of f._.sigParams(sig)){ - rc.parameters.push(f._.letterType(x)); + rc.parameters.push(f._.typeCodes(x)); } return rc; },************/ @@ -441,11 +462,12 @@ self.WhWasmUtilInstaller = function(target){ invalid. */ pushSigType: (dest, letter)=>dest.push(f._.typeCodes[f._.letterType(letter)]) }; - f._.sigTypes.i = f._.sigTypes.p = f._.sigTypes.P = f._.sigTypes.s = 'i32'; - f._.sigTypes.j = 'i64'; f._.sigTypes.f = 'f32'; f._.sigTypes.d = 'f64'; - f._.typeCodes['i32'] = 0x7f; f._.typeCodes['i64'] = 0x7e; - f._.typeCodes['f32'] = 0x7d; f._.typeCodes['f64'] = 0x7c; }/*static init*/ + if('string'===typeof func){ + const x = sig; + sig = func; + func = x; + } const sigParams = f._.sigParams(sig); const wasmCode = [0x01/*count: 1*/, 0x60/*function*/]; f._.uleb128Encode(wasmCode, 'push', sigParams.length); @@ -482,9 +504,12 @@ self.WhWasmUtilInstaller = function(target){ available slot of this.functionTable(), and returns the function's index in that table (which acts as a pointer to that function). The returned pointer can be passed to - removeFunction() to uninstall it and free up the table slot for + uninstallFunction() to uninstall it and free up the table slot for reuse. + If passed (string,function) arguments then it treats the first + argument as the signature and second as the function. + As a special case, if the passed-in function is a WASM-exported function then the signature argument is ignored and func is installed as-is, without requiring re-compilation/re-wrapping. @@ -498,13 +523,21 @@ self.WhWasmUtilInstaller = function(target){ Sidebar: this function differs from Emscripten's addFunction() _primarily_ in that it does not share that function's undocumented behavior of reusing a function if it's passed to - addFunction() more than once, which leads to removeFunction() + addFunction() more than once, which leads to uninstallFunction() breaking clients which do not take care to avoid that case: https://github.com/emscripten-core/emscripten/issues/17323 */ target.installFunction = function f(func, sig){ - const ft = this.functionTable(); + if(2!==arguments.length){ + toss("installFunction() requires exactly 2 arguments"); + } + if('string'===typeof func){ + const x = sig; + sig = func; + func = x; + } + const ft = target.functionTable(); const oldLen = ft.length; let ptr; while(cache.freeFuncIndexes.length){ @@ -532,13 +565,13 @@ self.WhWasmUtilInstaller = function(target){ } // It's not a WASM-exported function, so compile one... try { - ft.set(ptr, this.jsFuncToWasm(func, sig)); + ft.set(ptr, target.jsFuncToWasm(func, sig)); }catch(e){ if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); throw e; } return ptr; - }.bind(target); + }; /** Requires a pointer value previously returned from @@ -551,12 +584,12 @@ self.WhWasmUtilInstaller = function(target){ */ target.uninstallFunction = function(ptr){ const fi = cache.freeFuncIndexes; - const ft = this.functionTable(); + const ft = target.functionTable(); fi.push(ptr); const rc = ft.get(ptr); ft.set(ptr, null); return rc; - }.bind(target); + }; /** Given a WASM heap memory address and a data type name in the form @@ -602,6 +635,10 @@ self.WhWasmUtilInstaller = function(target){ out) the pointer's value, else it will contain an essentially random value. + ACHTUNG: calling this often, e.g. in a loop, can have a noticably + painful impact on performance. Rather than doing so, use + heapForSize() to fetch the heap object and read directly from it. + See: setMemValue() */ target.getMemValue = function(ptr, type='i8'){ @@ -614,14 +651,14 @@ self.WhWasmUtilInstaller = function(target){ case 'i16': return c.HEAP16[ptr>>1]; case 'i32': return c.HEAP32[ptr>>2]; case 'i64': - if(this.bigIntEnabled) return BigInt(c.HEAP64[ptr>>3]); + if(target.bigIntEnabled) return BigInt(c.HEAP64[ptr>>3]); break; case 'float': case 'f32': return c.HEAP32F[ptr>>2]; case 'double': case 'f64': return Number(c.HEAP64F[ptr>>3]); default: break; } toss('Invalid type for getMemValue():',type); - }.bind(target); + }; /** The counterpart of getMemValue(), this sets a numeric value at @@ -632,6 +669,10 @@ self.WhWasmUtilInstaller = function(target){ this function behaves as if the 3rd argument were `i32`. This function returns itself. + + ACHTUNG: calling this often, e.g. in a loop, can have a noticably + painful impact on performance. Rather than doing so, use + heapForSize() to fetch the heap object and assign directly to it. */ target.setMemValue = function f(ptr, value, type='i8'){ if (type.endsWith('*')) type = ptrIR; @@ -654,6 +695,32 @@ self.WhWasmUtilInstaller = function(target){ toss('Invalid type for setMemValue(): ' + type); }; + + /** Convenience form of getMemValue() intended for fetching + pointer-to-pointer values. */ + target.getPtrValue = (ptr)=>target.getMemValue(ptr, ptrIR); + + /** Convenience form of setMemValue() intended for setting + pointer-to-pointer values. */ + target.setPtrValue = (ptr, value)=>target.setMemValue(ptr, value, ptrIR); + + /** + Returns true if the given value appears to be legal for use as + a WASM pointer value. Its _range_ of values is not (cannot be) + validated except to ensure that it is a 32-bit integer with a + value of 0 or greater. Likewise, it cannot verify whether the + value actually refers to allocated memory in the WASM heap. + */ + target.isPtr32 = (ptr)=>('number'===typeof ptr && (ptr===(ptr|0)) && ptr>=0); + + /** + isPtr() is an alias for isPtr32(). If/when 64-bit WASM pointer + support becomes widespread, it will become an alias for either + isPtr32() or the as-yet-hypothetical isPtr64(), depending on a + configuration option. + */ + target.isPtr = target.isPtr32; + /** Expects ptr to be a pointer into the WASM heap memory which refers to a NUL-terminated C-style string encoded as UTF-8. @@ -669,6 +736,18 @@ self.WhWasmUtilInstaller = function(target){ return pos - ptr; }; + /** Internal helper to use in operations which need to distinguish + between SharedArrayBuffer heap memory and non-shared heap. */ + const __SAB = ('undefined'===typeof SharedArrayBuffer) + ? function(){} : SharedArrayBuffer; + const __utf8Decode = function(arrayBuffer, begin, end){ + return cache.utf8Decoder.decode( + (arrayBuffer.buffer instanceof __SAB) + ? arrayBuffer.slice(begin, end) + : arrayBuffer.subarray(begin, end) + ); + }; + /** Expects ptr to be a pointer into the WASM heap memory which refers to a NUL-terminated C-style string encoded as UTF-8. This @@ -677,13 +756,9 @@ self.WhWasmUtilInstaller = function(target){ ptr is falsy, `null` is returned. */ target.cstringToJs = function(ptr){ - const n = this.cstrlen(ptr); - if(null===n) return n; - return n - ? cache.utf8Decoder.decode( - new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) - ) : ""; - }.bind(target); + const n = target.cstrlen(ptr); + return n ? __utf8Decode(heapWrappers().HEAP8U, ptr, ptr+n) : (null===n ? n : ""); + }; /** Given a JS string, this function returns its UTF-8 length in @@ -811,16 +886,16 @@ self.WhWasmUtilInstaller = function(target){ */ target.cstrncpy = function(tgtPtr, srcPtr, n){ if(!tgtPtr || !srcPtr) toss("cstrncpy() does not accept NULL strings."); - if(n<0) n = this.cstrlen(strPtr)+1; + if(n<0) n = target.cstrlen(strPtr)+1; else if(!(n>0)) return 0; - const heap = this.heap8u(); + const heap = target.heap8u(); let i = 0, ch; for(; i < n && (ch = heap[srcPtr+i]); ++i){ heap[tgtPtr+i] = ch; } if(i__allocCStr(jstr, returnWithLength, target.scopedAlloc, 'scopedAllocCString()'); + // impl for allocMainArgv() and scopedAllocMainArgv(). + const __allocMainArgv = function(isScoped, list){ + if(!list.length) toss("Cannot allocate empty array."); + const pList = target[ + isScoped ? 'scopedAlloc' : 'alloc' + ](list.length * target.ptrSizeof); + let i = 0; + list.forEach((e)=>{ + target.setPtrValue(pList + (target.ptrSizeof * i++), + target[ + isScoped ? 'scopedAllocCString' : 'allocCString' + ](""+e)); + }); + return pList; + }; + + /** + Creates an array, using scopedAlloc(), suitable for passing to a + C-level main() routine. The input is a collection with a length + property and a forEach() method. A block of memory list.length + entries long is allocated and each pointer-sized block of that + memory is populated with a scopedAllocCString() conversion of the + (""+value) of each element. Returns a pointer to the start of the + list, suitable for passing as the 2nd argument to a C-style + main() function. + + Throws if list.length is falsy or scopedAllocPush() is not active. + */ + target.scopedAllocMainArgv = (list)=>__allocMainArgv(true, list); + + /** + Identical to scopedAllocMainArgv() but uses alloc() instead of + scopedAllocMainArgv + */ + target.allocMainArgv = (list)=>__allocMainArgv(false, list); + /** Wraps function call func() in a scopedAllocPush() and scopedAllocPop() block, such that all calls to scopedAlloc() and @@ -1013,33 +1124,41 @@ self.WhWasmUtilInstaller = function(target){ result of calling func(). */ target.scopedAllocCall = function(func){ - this.scopedAllocPush(); - try{ return func() } finally{ this.scopedAllocPop() } - }.bind(target); + target.scopedAllocPush(); + try{ return func() } finally{ target.scopedAllocPop() } + }; /** Internal impl for allocPtr() and scopedAllocPtr(). */ - const __allocPtr = function(howMany, method){ - __affirmAlloc(this, method); - let m = this[method](howMany * ptrSizeof); - this.setMemValue(m, 0, ptrIR) + const __allocPtr = function(howMany, safePtrSize, method){ + __affirmAlloc(target, method); + const pIr = safePtrSize ? 'i64' : ptrIR; + let m = target[method](howMany * (safePtrSize ? 8 : ptrSizeof)); + target.setMemValue(m, 0, pIr) if(1===howMany){ return m; } const a = [m]; for(let i = 1; i < howMany; ++i){ - m += ptrSizeof; + m += (safePtrSize ? 8 : ptrSizeof); a[i] = m; - this.setMemValue(m, 0, ptrIR); + target.setMemValue(m, 0, pIr); } return a; - }.bind(target); + }; /** - Allocates a single chunk of memory capable of holding `howMany` - pointers and zeroes them out. If `howMany` is 1 then the memory - chunk is returned directly, else an array of pointer addresses is - returned, which can optionally be used with "destructuring - assignment" like this: + Allocates one or more pointers as a single chunk of memory and + zeroes them out. + + The first argument is the number of pointers to allocate. The + second specifies whether they should use a "safe" pointer size (8 + bytes) or whether they may use the default pointer size + (typically 4 but also possibly 8). + + How the result is returned depends on its first argument: if + passed 1, it returns the allocated memory address. If passed more + than one then an array of pointer addresses is returned, which + can optionally be used with "destructuring assignment" like this: ``` const [p1, p2, p3] = allocPtr(3); @@ -1048,14 +1167,27 @@ self.WhWasmUtilInstaller = function(target){ ACHTUNG: when freeing the memory, pass only the _first_ result value to dealloc(). The others are part of the same memory chunk and must not be freed separately. + + The reason for the 2nd argument is.. + + When one of the returned pointers will refer to a 64-bit value, + e.g. a double or int64, an that value must be written or fetched, + e.g. using setMemValue() or getMemValue(), it is important that + the pointer in question be aligned to an 8-byte boundary or else + it will not be fetched or written properly and will corrupt or + read neighboring memory. It is only safe to pass false when the + client code is certain that it will only get/fetch 4-byte values + (or smaller). */ - target.allocPtr = (howMany=1)=>__allocPtr(howMany, 'alloc'); + target.allocPtr = + (howMany=1, safePtrSize=true)=>__allocPtr(howMany, safePtrSize, 'alloc'); /** Identical to allocPtr() except that it allocates using scopedAlloc() instead of alloc(). */ - target.scopedAllocPtr = (howMany=1)=>__allocPtr(howMany, 'scopedAlloc'); + target.scopedAllocPtr = + (howMany=1, safePtrSize=true)=>__allocPtr(howMany, safePtrSize, 'scopedAlloc'); /** If target.exports[name] exists, it is returned, else an @@ -1070,11 +1202,11 @@ self.WhWasmUtilInstaller = function(target){ /** Looks up a WASM-exported function named fname from - target.exports. If found, it is called, passed all remaining + target.exports. If found, it is called, passed all remaining arguments, and its return value is returned to xCall's caller. If not found, an exception is thrown. This function does no - conversion of argument or return types, but see xWrap() - and xCallWrapped() for variants which do. + conversion of argument or return types, but see xWrap() and + xCallWrapped() for variants which do. As a special case, if passed only 1 argument after the name and that argument in an Array, that array's entries become the @@ -1082,7 +1214,7 @@ self.WhWasmUtilInstaller = function(target){ not legal to pass an Array object to a WASM function.) */ target.xCall = function(fname, ...args){ - const f = this.xGet(fname); + const f = target.xGet(fname); if(!(f instanceof Function)) toss("Exported symbol",fname,"is not a function."); if(f.length!==args.length) __argcMismatch(fname,f.length) /* This is arguably over-pedantic but we want to help clients keep @@ -1090,7 +1222,7 @@ self.WhWasmUtilInstaller = function(target){ return (2===arguments.length && Array.isArray(arguments[1])) ? f.apply(null, arguments[1]) : f.apply(null, args); - }.bind(target); + }; /** State for use with xWrap() @@ -1102,21 +1234,28 @@ self.WhWasmUtilInstaller = function(target){ /** Map of type names to return result conversion functions. */ cache.xWrap.convert.result = Object.create(null); - xcv.arg.i64 = (i)=>BigInt(i); + if(target.bigIntEnabled){ + xcv.arg.i64 = (i)=>BigInt(i); + } xcv.arg.i32 = (i)=>(i | 0); xcv.arg.i16 = (i)=>((i | 0) & 0xFFFF); xcv.arg.i8 = (i)=>((i | 0) & 0xFF); xcv.arg.f32 = xcv.arg.float = (i)=>Number(i).valueOf(); xcv.arg.f64 = xcv.arg.double = xcv.arg.f32; xcv.arg.int = xcv.arg.i32; - xcv.result['*'] = xcv.result['pointer'] = xcv.arg[ptrIR]; + xcv.result['*'] = xcv.result['pointer'] = xcv.arg['**'] = xcv.arg[ptrIR]; + xcv.result['number'] = (v)=>Number(v); - for(const t of ['i8', 'i16', 'i32', 'int', 'i64', - 'f32', 'float', 'f64', 'double']){ - xcv.arg[t+'*'] = xcv.result[t+'*'] = xcv.arg[ptrIR] - xcv.result[t] = xcv.arg[t] || toss("Missing arg converter:",t); + { /* Copy certain xcv.arg[...] handlers to xcv.result[...] and + add pointer-style variants of them. */ + const copyToResult = ['i8', 'i16', 'i32', 'int', + 'f32', 'float', 'f64', 'double']; + if(target.bigIntEnabled) copyToResult.push('i64'); + for(const t of copyToResult){ + xcv.arg[t+'*'] = xcv.result[t+'*'] = xcv.arg[ptrIR]; + xcv.result[t] = xcv.arg[t] || toss("Missing arg converter:",t); + } } - xcv.arg['**'] = xcv.arg[ptrIR]; /** In order for args of type string to work in various contexts in @@ -1134,17 +1273,18 @@ self.WhWasmUtilInstaller = function(target){ Would that be too much magic concentrated in one place, ready to backfire? */ - xcv.arg.string = xcv.arg['pointer'] = xcv.arg['*'] = function(v){ - if('string'===typeof v) return target.scopedAllocCString(v); - return v ? xcv.arg[ptrIR](v) : null; - }; - xcv.result.string = (i)=>target.cstringToJs(i); - xcv.result['string:free'] = function(i){ + xcv.arg.string = xcv.arg.utf8 = xcv.arg['pointer'] = xcv.arg['*'] + = function(v){ + if('string'===typeof v) return target.scopedAllocCString(v); + return v ? xcv.arg[ptrIR](v) : null; + }; + xcv.result.string = xcv.result.utf8 = (i)=>target.cstringToJs(i); + xcv.result['string:free'] = xcv.result['utf8:free'] = (i)=>{ try { return i ? target.cstringToJs(i) : null } finally{ target.dealloc(i) } }; xcv.result.json = (i)=>JSON.parse(target.cstringToJs(i)); - xcv.result['json:free'] = function(i){ + xcv.result['json:free'] = (i)=>{ try{ return i ? JSON.parse(target.cstringToJs(i)) : null } finally{ target.dealloc(i) } } @@ -1164,19 +1304,19 @@ self.WhWasmUtilInstaller = function(target){ */ xcv.arg['func-ptr'] = function(v){ if(!(v instanceof Function)) return xcv.arg[ptrIR]; - const f = this.jsFuncToWasm(v, WHAT_SIGNATURE); - }.bind(target); + const f = target.jsFuncToWasm(v, WHAT_SIGNATURE); + }; } - const __xArgAdapter = + const __xArgAdapterCheck = (t)=>xcv.arg[t] || toss("Argument adapter not found:",t); - const __xResultAdapter = + const __xResultAdapterCheck = (t)=>xcv.result[t] || toss("Result adapter not found:",t); - cache.xWrap.convertArg = (t,v)=>__xArgAdapter(t)(v); + cache.xWrap.convertArg = (t,v)=>__xArgAdapterCheck(t)(v); cache.xWrap.convertResult = - (t,v)=>(null===t ? v : (t ? __xResultAdapter(t)(v) : undefined)); + (t,v)=>(null===t ? v : (t ? __xResultAdapterCheck(t)(v) : undefined)); /** Creates a wrapper for the WASM-exported function fname. Uses @@ -1239,43 +1379,63 @@ self.WhWasmUtilInstaller = function(target){ type. It's primarily intended to mark output-pointer arguments. - `i64` (args and results): passes the value to BigInt() to - convert it to an int64. + convert it to an int64. Only available if bigIntEnabled is + true. - `f32` (`float`), `f64` (`double`) (args and results): pass their argument to Number(). i.e. the adaptor does not currently distinguish between the two types of floating-point numbers. + - `number` (results): converts the result to a JS Number using + Number(theValue).valueOf(). Note that this is for result + conversions only, as it's not possible to generically know + which type of number to convert arguments to. + Non-numeric conversions include: - - `string` (args): has two different semantics in order to - accommodate various uses of certain C APIs (e.g. output-style - strings)... + - `string` or `utf8` (args): has two different semantics in order + to accommodate various uses of certain C APIs + (e.g. output-style strings)... - - If the arg is a string, it creates a _temporary_ C-string to - pass to the exported function, cleaning it up before the - wrapper returns. If a long-lived C-string pointer is - required, that requires client-side code to create the - string, then pass its pointer to the function. + - If the arg is a string, it creates a _temporary_ + UTF-8-encoded C-string to pass to the exported function, + cleaning it up before the wrapper returns. If a long-lived + C-string pointer is required, that requires client-side code + to create the string, then pass its pointer to the function. - Else the arg is assumed to be a pointer to a string the client has already allocated and it's passed on as a WASM pointer. - - `string` (results): treats the result value as a const C-string, - copies it to a JS string, and returns that JS string. + - `string` or `utf8` (results): treats the result value as a + const C-string, encoded as UTF-8, copies it to a JS string, + and returns that JS string. - - `string:free` (results): treats the result value as a non-const - C-string, ownership of which has just been transfered to the - caller. It copies the C-string to a JS string, frees the - C-string, and returns the JS string. If such a result value is - NULL, the JS result is `null`. + - `string:free` or `utf8:free) (results): treats the result value + as a non-const UTF-8 C-string, ownership of which has just been + transfered to the caller. It copies the C-string to a JS + string, frees the C-string, and returns the JS string. If such + a result value is NULL, the JS result is `null`. Achtung: when + using an API which returns results from a specific allocator, + e.g. `my_malloc()`, this conversion _is not legal_. Instead, an + equivalent conversion which uses the appropriate deallocator is + required. For example: + +```js + target.xWrap.resultAdaptor('string:my_free',(i)=>{ + try { return i ? target.cstringToJs(i) : null } + finally{ target.exports.my_free(i) } + }; +``` - `json` (results): treats the result as a const C-string and returns the result of passing the converted-to-JS string to JSON.parse(). Returns `null` if the C-string is a NULL pointer. - `json:free` (results): works exactly like `string:free` but - returns the same thing as the `json` adapter. + returns the same thing as the `json` adapter. Note the + warning in `string:free` regarding maching allocators and + deallocators. The type names for results and arguments are validated when xWrap() is called and any unknown names will trigger an @@ -1310,34 +1470,32 @@ self.WhWasmUtilInstaller = function(target){ if(3===arguments.length && Array.isArray(arguments[2])){ argTypes = arguments[2]; } - const xf = this.xGet(fname); - if(argTypes.length!==xf.length) __argcMismatch(fname, xf.length) + const xf = target.xGet(fname); + if(argTypes.length!==xf.length) __argcMismatch(fname, xf.length); if((null===resultType) && 0===xf.length){ /* Func taking no args with an as-is return. We don't need a wrapper. */ return xf; } /*Verify the arg type conversions are valid...*/; - if(undefined!==resultType && null!==resultType) __xResultAdapter(resultType); - argTypes.forEach(__xArgAdapter) + if(undefined!==resultType && null!==resultType) __xResultAdapterCheck(resultType); + argTypes.forEach(__xArgAdapterCheck); if(0===xf.length){ // No args to convert, so we can create a simpler wrapper... - return function(){ - return (arguments.length - ? __argcMismatch(fname, xf.length) - : cache.xWrap.convertResult(resultType, xf.call(null))); - }; + return (...args)=>(args.length + ? __argcMismatch(fname, xf.length) + : cache.xWrap.convertResult(resultType, xf.call(null))); } return function(...args){ if(args.length!==xf.length) __argcMismatch(fname, xf.length); - const scope = this.scopedAllocPush(); + const scope = target.scopedAllocPush(); try{ const rc = xf.apply(null,args.map((v,i)=>cache.xWrap.convertArg(argTypes[i], v))); return cache.xWrap.convertResult(resultType, rc); }finally{ - this.scopedAllocPop(scope); + target.scopedAllocPop(scope); } - }.bind(this); - }.bind(target)/*xWrap()*/; + }; + }/*xWrap()*/; /** Internal impl for xWrap.resultAdapter() and argAdaptor(). */ const __xAdapter = function(func, argc, typeName, adapter, modeName, xcvPart){ @@ -1428,11 +1586,10 @@ self.WhWasmUtilInstaller = function(target){ type name, as documented for xWrap() (use a falsy value or an empty array for nullary functions). The 4th+ arguments are arguments for the call, with the special case that if the 4th - argument is an array, it is used as the arguments for the call - (again, falsy or an empty array for nullary functions). Returns - the converted result of the call. + argument is an array, it is used as the arguments for the + call. Returns the converted result of the call. - This is just a thin wrapp around xWrap(). If the given function + This is just a thin wrapper around xWrap(). If the given function is to be called more than once, it's more efficient to use xWrap() to create a wrapper, then to call that wrapper as many times as needed. For one-shot calls, however, this variant is @@ -1441,9 +1598,9 @@ self.WhWasmUtilInstaller = function(target){ */ target.xCallWrapped = function(fname, resultType, argTypes, ...args){ if(Array.isArray(arguments[3])) args = arguments[3]; - return this.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]); - }.bind(target); - + return target.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]); + }; + return target; }; @@ -1455,11 +1612,11 @@ self.WhWasmUtilInstaller = function(target){ - `onload(loadResult,config)`: optional callback. The first argument is the result object from - WebAssembly.instanitate[Streaming](). The 2nd is the config + WebAssembly.instantiate[Streaming](). The 2nd is the config object passed to this function. Described in more detail below. - `imports`: optional imports object for - WebAssembly.instantiate[Streaming](). The default is am empty set + WebAssembly.instantiate[Streaming](). The default is an empty set of imports. If the module requires any imports, this object must include them. @@ -1522,10 +1679,11 @@ self.WhWasmUtilInstaller.yawl = function(config){ || toss("Missing 'memory' object!"); } if(!tgt.alloc && arg.instance.exports.malloc){ + const exports = arg.instance.exports; tgt.alloc = function(n){ - return this(n) || toss("Allocation of",n,"bytes failed."); - }.bind(arg.instance.exports.malloc); - tgt.dealloc = function(m){this(m)}.bind(arg.instance.exports.free); + return exports.malloc(n) || toss("Allocation of",n,"bytes failed."); + }; + tgt.dealloc = function(m){exports.free(m)}; } wui(tgt); } diff --git a/ext/wasm/demo-123-worker.html b/ext/wasm/demo-123-worker.html new file mode 100644 index 0000000000..692203d71e --- /dev/null +++ b/ext/wasm/demo-123-worker.html @@ -0,0 +1,44 @@ + + + + + + + Hello, sqlite3 + + + +

1-2-sqlite3 worker demo

+ + + diff --git a/ext/wasm/demo-123.html b/ext/wasm/demo-123.html new file mode 100644 index 0000000000..2046b076d4 --- /dev/null +++ b/ext/wasm/demo-123.html @@ -0,0 +1,24 @@ + + + + + + + Hello, sqlite3 + + + +

1-2-sqlite3 demo

+ + + + diff --git a/ext/wasm/demo-123.js b/ext/wasm/demo-123.js new file mode 100644 index 0000000000..311afcc827 --- /dev/null +++ b/ext/wasm/demo-123.js @@ -0,0 +1,289 @@ +/* + 2022-09-19 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic demonstration of the SQLite3 "OO#1" API. +*/ +'use strict'; +(function(){ + /** + Set up our output channel differently depending + on whether we are running in a worker thread or + the main (UI) thread. + */ + let logHtml; + if(self.window === self /* UI thread */){ + console.log("Running demo from main UI thread."); + logHtml = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + document.body.append(ln); + }; + }else{ /* Worker thread */ + console.log("Running demo from Worker thread."); + logHtml = function(cssClass,...args){ + postMessage({ + type:'log', + payload:{cssClass, args} + }); + }; + } + const log = (...args)=>logHtml('',...args); + const warn = (...args)=>logHtml('warning',...args); + const error = (...args)=>logHtml('error',...args); + + const demo1 = function(sqlite3){ + const capi = sqlite3.capi/*C-style API*/, + oo = sqlite3.oo1/*high-level OO API*/; + log("sqlite3 version",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + const db = new oo.DB("/mydb.sqlite3",'ct'); + log("transient db =",db.filename); + /** + Never(!) rely on garbage collection to clean up DBs and + (especially) prepared statements. Always wrap their lifetimes + in a try/finally construct, as demonstrated below. By and + large, client code can entirely avoid lifetime-related + complications of prepared statement objects by using the + DB.exec() method for SQL execution. + */ + try { + log("Create a table..."); + db.exec("CREATE TABLE IF NOT EXISTS t(a,b)"); + //Equivalent: + db.exec({ + sql:"CREATE TABLE IF NOT EXISTS t(a,b)" + // ... numerous other options ... + }); + // SQL can be either a string or a byte array + // or an array of strings which get concatenated + // together as-is (so be sure to end each statement + // with a semicolon). + + log("Insert some data using exec()..."); + let i; + for( i = 20; i <= 25; ++i ){ + db.exec({ + sql: "insert into t(a,b) values (?,?)", + // bind by parameter index... + bind: [i, i*2] + }); + db.exec({ + sql: "insert into t(a,b) values ($a,$b)", + // bind by parameter name... + bind: {$a: i * 10, $b: i * 20} + }); + } + + log("Insert using a prepared statement..."); + let q = db.prepare([ + // SQL may be a string or array of strings + // (concatenated w/o separators). + "insert into t(a,b) ", + "values(?,?)" + ]); + try { + for( i = 100; i < 103; ++i ){ + q.bind( [i, i*2] ).step(); + q.reset(); + } + // Equivalent... + for( i = 103; i <= 105; ++i ){ + q.bind(1, i).bind(2, i*2).stepReset(); + } + }finally{ + q.finalize(); + } + + log("Query data with exec() using rowMode 'array'..."); + db.exec({ + sql: "select a from t order by a limit 3", + rowMode: 'array', // 'array' (default), 'object', or 'stmt' + callback: function(row){ + log("row ",++this.counter,"=",row); + }.bind({counter: 0}) + }); + + log("Query data with exec() using rowMode 'object'..."); + db.exec({ + sql: "select a as aa, b as bb from t order by aa limit 3", + rowMode: 'object', + callback: function(row){ + log("row ",++this.counter,"=",JSON.stringify(row)); + }.bind({counter: 0}) + }); + + log("Query data with exec() using rowMode 'stmt'..."); + db.exec({ + sql: "select a from t order by a limit 3", + rowMode: 'stmt', + callback: function(row){ + log("row ",++this.counter,"get(0) =",row.get(0)); + }.bind({counter: 0}) + }); + + log("Query data with exec() using rowMode INTEGER (result column index)..."); + db.exec({ + sql: "select a, b from t order by a limit 3", + rowMode: 1, // === result column 1 + callback: function(row){ + log("row ",++this.counter,"b =",row); + }.bind({counter: 0}) + }); + + log("Query data with exec() using rowMode $COLNAME (result column name)..."); + db.exec({ + sql: "select a a, b from t order by a limit 3", + rowMode: '$a', + callback: function(value){ + log("row ",++this.counter,"a =",value); + }.bind({counter: 0}) + }); + + log("Query data with exec() without a callback..."); + let resultRows = []; + db.exec({ + sql: "select a, b from t order by a limit 3", + rowMode: 'object', + resultRows: resultRows + }); + log("Result rows:",JSON.stringify(resultRows,undefined,2)); + + log("Create a scalar UDF..."); + db.createFunction({ + name: 'twice', + xFunc: function(pCx, arg){ // note the call arg count + return arg + arg; + } + }); + log("Run scalar UDF and collect result column names..."); + let columnNames = []; + db.exec({ + sql: "select a, twice(a), twice(''||a) from t order by a desc limit 3", + columnNames: columnNames, + rowMode: 'stmt', + callback: function(row){ + log("a =",row.get(0), "twice(a) =", row.get(1), + "twice(''||a) =",row.get(2)); + } + }); + log("Result column names:",columnNames); + + try{ + log("The following use of the twice() UDF will", + "fail because of incorrect arg count..."); + db.exec("select twice(1,2,3)"); + }catch(e){ + warn("Got expected exception:",e.message); + } + + try { + db.transaction( function(D) { + D.exec("delete from t"); + log("In transaction: count(*) from t =",db.selectValue("select count(*) from t")); + throw new sqlite3.SQLite3Error("Demonstrating transaction() rollback"); + }); + }catch(e){ + if(e instanceof sqlite3.SQLite3Error){ + log("Got expected exception from db.transaction():",e.message); + log("count(*) from t =",db.selectValue("select count(*) from t")); + }else{ + throw e; + } + } + + try { + db.savepoint( function(D) { + D.exec("delete from t"); + log("In savepoint: count(*) from t =",db.selectValue("select count(*) from t")); + D.savepoint(function(DD){ + const rows = []; + DD.exec({ + sql: ["insert into t(a,b) values(99,100);", + "select count(*) from t"], + rowMode: 0, + resultRows: rows + }); + log("In nested savepoint. Row count =",rows[0]); + throw new sqlite3.SQLite3Error("Demonstrating nested savepoint() rollback"); + }) + }); + }catch(e){ + if(e instanceof sqlite3.SQLite3Error){ + log("Got expected exception from nested db.savepoint():",e.message); + log("count(*) from t =",db.selectValue("select count(*) from t")); + }else{ + throw e; + } + } + }finally{ + db.close(); + } + + log("That's all, folks!"); + + /** + Some of the features of the OO API not demonstrated above... + + - get change count (total or statement-local, 32- or 64-bit) + - get a DB's file name + + Misc. Stmt features: + + - Various forms of bind() + - clearBindings() + - reset() + - Various forms of step() + - Variants of get() for explicit type treatment/conversion, + e.g. getInt(), getFloat(), getBlob(), getJSON() + - getColumnName(ndx), getColumnNames() + - getParamIndex(name) + */ + }/*demo1()*/; + + log("Loading and initializing sqlite3 module..."); + if(self.window!==self) /*worker thread*/{ + /* + If sqlite3.js is in a directory other than this script, in order + to get sqlite3.js to resolve sqlite3.wasm properly, we have to + explicitly tell it where sqlite3.js is being loaded from. We do + that by passing the `sqlite3.dir=theDirName` URL argument to + _this_ script. That URL argument will be seen by the JS/WASM + loader and it will adjust the sqlite3.wasm path accordingly. If + sqlite3.js/.wasm are in the same directory as this script then + that's not needed. + + URL arguments passed as part of the filename via importScripts() + are simply lost, and such scripts see the self.location of + _this_ script. + */ + let sqlite3Js = 'sqlite3.js'; + const urlParams = new URL(self.location.href).searchParams; + if(urlParams.has('sqlite3.dir')){ + sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js; + } + importScripts(sqlite3Js); + } + self.sqlite3InitModule({ + // We can redirect any stdout/stderr from the module + // like so... + print: log, + printErr: error + }).then(function(sqlite3){ + //console.log('sqlite3 =',sqlite3); + log("Done initializing. Running demo..."); + try { + demo1(sqlite3); + }catch(e){ + error("Exception:",e.message); + } + }); +})(); diff --git a/ext/wasm/demo-jsstorage.html b/ext/wasm/demo-jsstorage.html new file mode 100644 index 0000000000..79f4a3b4be --- /dev/null +++ b/ext/wasm/demo-jsstorage.html @@ -0,0 +1,49 @@ + + + + + + + + + sqlite3-kvvfs.js tests + + +
sqlite3-kvvfs.js tests
+ +
+ +
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
+
+ Options +
+ + + + + +
+
+
+ + + + + + diff --git a/ext/wasm/demo-jsstorage.js b/ext/wasm/demo-jsstorage.js new file mode 100644 index 0000000000..cf820e4033 --- /dev/null +++ b/ext/wasm/demo-jsstorage.js @@ -0,0 +1,114 @@ +/* + 2022-09-12 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3.wasm with kvvfs support. This file + must be run in main JS thread and sqlite3.js must have been loaded + before it. +*/ +'use strict'; +(function(){ + const T = self.SqliteTestUtil; + const toss = function(...args){throw new Error(args.join(' '))}; + const debug = console.debug.bind(console); + const eOutput = document.querySelector('#test-output'); + const logC = console.log.bind(console) + const logE = function(domElement){ + eOutput.append(domElement); + }; + const logHtml = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + logE(ln); + } + const log = function(...args){ + logC(...args); + logHtml('',...args); + }; + const warn = function(...args){ + logHtml('warning',...args); + }; + const error = function(...args){ + logHtml('error',...args); + }; + + const runTests = function(sqlite3){ + const capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = sqlite3.wasm; + log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + T.assert( 0 !== capi.sqlite3_vfs_find(null) ); + if(!capi.sqlite3_vfs_find('kvvfs')){ + error("This build is not kvvfs-capable."); + return; + } + + const dbStorage = 0 ? 'session' : 'local'; + const theStore = 's'===dbStorage[0] ? sessionStorage : localStorage; + const db = new oo.JsStorageDb( dbStorage ); + // Or: oo.DB(dbStorage, 'c', 'kvvfs') + log("db.storageSize():",db.storageSize()); + document.querySelector('#btn-clear-storage').addEventListener('click',function(){ + const sz = db.clearStorage(); + log("kvvfs",db.filename+"Storage cleared:",sz,"entries."); + }); + document.querySelector('#btn-clear-log').addEventListener('click',function(){ + eOutput.innerText = ''; + }); + document.querySelector('#btn-init-db').addEventListener('click',function(){ + try{ + const saveSql = []; + db.exec({ + sql: ["drop table if exists t;", + "create table if not exists t(a);", + "insert into t(a) values(?),(?),(?)"], + bind: [performance.now() >> 0, + (performance.now() * 2) >> 0, + (performance.now() / 2) >> 0], + saveSql + }); + console.log("saveSql =",saveSql,theStore); + log("DB (re)initialized."); + }catch(e){ + error(e.message); + } + }); + const btnSelect = document.querySelector('#btn-select1'); + btnSelect.addEventListener('click',function(){ + log("DB rows:"); + try{ + db.exec({ + sql: "select * from t order by a", + rowMode: 0, + callback: (v)=>log(v) + }); + }catch(e){ + error(e.message); + } + }); + document.querySelector('#btn-storage-size').addEventListener('click',function(){ + log("size.storageSize(",dbStorage,") says", db.storageSize(), + "bytes"); + }); + log("Storage backend:",db.filename); + if(0===db.selectValue('select count(*) from sqlite_master')){ + log("DB is empty. Use the init button to populate it."); + }else{ + log("DB contains data from a previous session. Use the Clear Ctorage button to delete it."); + btnSelect.click(); + } + }; + + sqlite3InitModule(self.sqlite3TestModule).then((sqlite3)=>{ + runTests(sqlite3); + }); +})(); diff --git a/ext/wasm/testing1.html b/ext/wasm/demo-worker1-promiser.html similarity index 84% rename from ext/wasm/testing1.html rename to ext/wasm/demo-worker1-promiser.html index 0c64470221..e99131e6c9 100644 --- a/ext/wasm/testing1.html +++ b/ext/wasm/demo-worker1-promiser.html @@ -6,10 +6,10 @@ - sqlite3-api.js tests + worker-promise tests -
sqlite3-api.js tests
+
worker-promise tests
@@ -27,8 +27,8 @@
Most stuff on this page happens in the dev console.
- - + + diff --git a/ext/wasm/demo-worker1-promiser.js b/ext/wasm/demo-worker1-promiser.js new file mode 100644 index 0000000000..a65cc31b6e --- /dev/null +++ b/ext/wasm/demo-worker1-promiser.js @@ -0,0 +1,270 @@ +/* + 2022-08-23 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + Demonstration of the sqlite3 Worker API #1 Promiser: a Promise-based + proxy for for the sqlite3 Worker #1 API. +*/ +'use strict'; +(function(){ + const T = self.SqliteTestUtil; + const eOutput = document.querySelector('#test-output'); + const warn = console.warn.bind(console); + const error = console.error.bind(console); + const log = console.log.bind(console); + const logHtml = async function(cssClass,...args){ + log.apply(this, args); + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + eOutput.append(ln); + }; + + let startTime; + const testCount = async ()=>{ + logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); + }; + + //why is this triggered even when we catch() a Promise? + //window.addEventListener('unhandledrejection', function(event) { + // warn('unhandledrejection',event); + //}); + + const promiserConfig = { + worker: ()=>{ + const w = new Worker("jswasm/sqlite3-worker1.js"); + w.onerror = (event)=>error("worker.onerror",event); + return w; + }, + debug: 1 ? undefined : (...args)=>console.debug('worker debug',...args), + onunhandled: function(ev){ + error("Unhandled worker message:",ev.data); + }, + onready: function(){ + self.sqlite3TestModule.setStatus(null)/*hide the HTML-side is-loading spinner*/; + runTests(); + }, + onerror: function(ev){ + error("worker1 error:",ev); + } + }; + const workerPromise = self.sqlite3Worker1Promiser(promiserConfig); + delete self.sqlite3Worker1Promiser; + + const wtest = async function(msgType, msgArgs, callback){ + if(2===arguments.length && 'function'===typeof msgArgs){ + callback = msgArgs; + msgArgs = undefined; + } + const p = workerPromise({type: msgType, args:msgArgs}); + return callback ? p.then(callback).finally(testCount) : p; + }; + + const runTests = async function(){ + const dbFilename = '/testing2.sqlite3'; + startTime = performance.now(); + + let sqConfig; + await wtest('config-get', (ev)=>{ + const r = ev.result; + log('sqlite3.config subset:', r); + T.assert('boolean' === typeof r.bigIntEnabled) + .assert('string'===typeof r.wasmfsOpfsDir) + .assert('boolean' === typeof r.wasmfsOpfsEnabled); + sqConfig = r; + }); + logHtml('', + "Sending 'open' message and waiting for its response before continuing..."); + + await wtest('open', { + filename: dbFilename, + simulateError: 0 /* if true, fail the 'open' */, + }, function(ev){ + const r = ev.result; + log("then open result",r); + T.assert(ev.dbId === r.dbId) + .assert(ev.messageId) + .assert('string' === typeof r.vfs); + promiserConfig.dbId = ev.dbId; + }).then(runTests2); + }; + + const runTests2 = async function(){ + const mustNotReach = ()=>toss("This is not supposed to be reached."); + + await wtest('exec',{ + sql: ["create table t(a,b)", + "insert into t(a,b) values(1,2),(3,4),(5,6)" + ].join(';'), + multi: true, + resultRows: [], columnNames: [] + }, function(ev){ + ev = ev.result; + T.assert(0===ev.resultRows.length) + .assert(0===ev.columnNames.length); + }); + + await wtest('exec',{ + sql: 'select a a, b b from t order by a', + resultRows: [], columnNames: [], + }, function(ev){ + ev = ev.result; + T.assert(3===ev.resultRows.length) + .assert(1===ev.resultRows[0][0]) + .assert(6===ev.resultRows[2][1]) + .assert(2===ev.columnNames.length) + .assert('b'===ev.columnNames[1]); + }); + + await wtest('exec',{ + sql: 'select a a, b b from t order by a', + resultRows: [], columnNames: [], + rowMode: 'object' + }, function(ev){ + ev = ev.result; + T.assert(3===ev.resultRows.length) + .assert(1===ev.resultRows[0].a) + .assert(6===ev.resultRows[2].b) + }); + + await wtest( + 'exec', + {sql:'intentional_error'}, + mustNotReach + ).catch((e)=>{ + warn("Intentional error:",e); + }); + + await wtest('exec',{ + sql:'select 1 union all select 3', + resultRows: [], + }, function(ev){ + ev = ev.result; + T.assert(2 === ev.resultRows.length) + .assert(1 === ev.resultRows[0][0]) + .assert(3 === ev.resultRows[1][0]); + }); + + const resultRowTest1 = function f(ev){ + if(undefined === f.counter) f.counter = 0; + if(null === ev.rowNumber){ + /* End of result set. */ + T.assert(undefined === ev.row) + .assert(2===ev.columnNames.length) + .assert('a'===ev.columnNames[0]) + .assert('B'===ev.columnNames[1]); + }else{ + T.assert(ev.rowNumber > 0); + ++f.counter; + } + log("exec() result row:",ev); + T.assert(null === ev.rowNumber || 'number' === typeof ev.row.B); + }; + await wtest('exec',{ + sql: 'select a a, b B from t order by a limit 3', + callback: resultRowTest1, + rowMode: 'object' + }, function(ev){ + T.assert(3===resultRowTest1.counter); + resultRowTest1.counter = 0; + }); + + const resultRowTest2 = function f(ev){ + if(null === ev.rowNumber){ + /* End of result set. */ + T.assert(undefined === ev.row) + .assert(1===ev.columnNames.length) + .assert('a'===ev.columnNames[0]) + }else{ + T.assert(ev.rowNumber > 0); + f.counter = ev.rowNumber; + } + log("exec() result row:",ev); + T.assert(null === ev.rowNumber || 'number' === typeof ev.row); + }; + await wtest('exec',{ + sql: 'select a a from t limit 3', + callback: resultRowTest2, + rowMode: 0 + }, function(ev){ + T.assert(3===resultRowTest2.counter); + }); + + const resultRowTest3 = function f(ev){ + if(null === ev.rowNumber){ + T.assert(3===ev.columnNames.length) + .assert('foo'===ev.columnNames[0]) + .assert('bar'===ev.columnNames[1]) + .assert('baz'===ev.columnNames[2]); + }else{ + f.counter = ev.rowNumber; + T.assert('number' === typeof ev.row); + } + }; + await wtest('exec',{ + sql: "select 'foo' foo, a bar, 'baz' baz from t limit 2", + callback: resultRowTest3, + columnNames: [], + rowMode: ':bar' + }, function(ev){ + log("exec() result row:",ev); + T.assert(2===resultRowTest3.counter); + }); + + await wtest('exec',{ + multi: true, + sql:[ + 'pragma foreign_keys=0;', + // ^^^ arbitrary query with no result columns + 'select a, b from t order by a desc; select a from t;' + // multi-exec only honors results from the first + // statement with result columns (regardless of whether) + // it has any rows). + ], + rowMode: 1, + resultRows: [] + },function(ev){ + const rows = ev.result.resultRows; + T.assert(3===rows.length). + assert(6===rows[0]); + }); + + await wtest('exec',{sql: 'delete from t where a>3'}); + + await wtest('exec',{ + sql: 'select count(a) from t', + resultRows: [] + },function(ev){ + ev = ev.result; + T.assert(1===ev.resultRows.length) + .assert(2===ev.resultRows[0][0]); + }); + + await wtest('export', function(ev){ + ev = ev.result; + T.assert('string' === typeof ev.filename) + .assert(ev.byteArray instanceof Uint8Array) + .assert(ev.byteArray.length > 1024) + .assert('application/x-sqlite3' === ev.mimetype); + }); + + /***** close() tests must come last. *****/ + await wtest('close',{},function(ev){ + T.assert('string' === typeof ev.result.filename); + }); + + await wtest('close', (ev)=>{ + T.assert(undefined === ev.result.filename); + }).finally(()=>logHtml('',"That's all, folks!")); + }/*runTests2()*/; + + log("Init complete, but async init bits may still be running."); +})(); diff --git a/ext/wasm/testing2.html b/ext/wasm/demo-worker1.html similarity index 87% rename from ext/wasm/testing2.html rename to ext/wasm/demo-worker1.html index 739c7f66be..c766ffd445 100644 --- a/ext/wasm/testing2.html +++ b/ext/wasm/demo-worker1.html @@ -1,3 +1,4 @@ + @@ -6,10 +7,10 @@ - sqlite3-worker.js tests + sqlite3-worker1.js tests -
sqlite3-worker.js tests
+
sqlite3-worker1.js tests
@@ -28,6 +29,6 @@
- + diff --git a/ext/wasm/testing2.js b/ext/wasm/demo-worker1.js similarity index 73% rename from ext/wasm/testing2.js rename to ext/wasm/demo-worker1.js index 3a279513f8..cc63f3a7cc 100644 --- a/ext/wasm/testing2.js +++ b/ext/wasm/demo-worker1.js @@ -10,17 +10,21 @@ *********************************************************************** - A basic test script for sqlite3-worker.js. + A basic test script for sqlite3-worker1.js. + + Note that the wrapper interface demonstrated in + demo-worker1-promiser.js is much easier to use from client code, as it + lacks the message-passing acrobatics demonstrated in this file. */ 'use strict'; (function(){ const T = self.SqliteTestUtil; - const SW = new Worker("api/sqlite3-worker.js"); + const SW = new Worker("jswasm/sqlite3-worker1.js"); const DbState = { id: undefined }; const eOutput = document.querySelector('#test-output'); - const log = console.log.bind(console) + const log = console.log.bind(console); const logHtml = function(cssClass,...args){ log.apply(this, args); const ln = document.createElement('div'); @@ -31,24 +35,13 @@ const warn = console.warn.bind(console); const error = console.error.bind(console); const toss = (...args)=>{throw new Error(args.join(' '))}; - /** Posts a worker message as {type:type, data:data}. */ - const wMsg = function(type,data){ - log("Posting message to worker dbId="+(DbState.id||'default')+':',data); - SW.postMessage({ - type, - dbId: DbState.id, - data, - departureTime: performance.now() - }); - return SW; - }; SW.onerror = function(event){ error("onerror",event); }; let startTime; - + /** A queue for callbacks which are to be run in response to async DB commands. See the notes in runTests() for why we need @@ -74,28 +67,37 @@ logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); }; - const logEventResult = function(evd){ + const logEventResult = function(ev){ + const evd = ev.result; logHtml(evd.errorClass ? 'error' : '', - "runOneTest",evd.messageId,"Worker time =", - (evd.workerRespondTime - evd.workerReceivedTime),"ms.", + "runOneTest",ev.messageId,"Worker time =", + (ev.workerRespondTime - ev.workerReceivedTime),"ms.", "Round-trip event time =", - (performance.now() - evd.departureTime),"ms.", - (evd.errorClass ? evd.message : "") + (performance.now() - ev.departureTime),"ms.", + (evd.errorClass ? ev.message : "")//, JSON.stringify(evd) ); }; - const runOneTest = function(eventType, eventData, callback){ - T.assert(eventData && 'object'===typeof eventData); + const runOneTest = function(eventType, eventArgs, callback){ + T.assert(eventArgs && 'object'===typeof eventArgs); /* ^^^ that is for the testing and messageId-related code, not a hard requirement of all of the Worker-exposed APIs. */ - eventData.messageId = MsgHandlerQueue.push(eventType,function(ev){ - logEventResult(ev.data); + const messageId = MsgHandlerQueue.push(eventType,function(ev){ + logEventResult(ev); if(callback instanceof Function){ callback(ev); testCount(); } }); - wMsg(eventType, eventData); + const msg = { + type: eventType, + args: eventArgs, + dbId: DbState.id, + messageId: messageId, + departureTime: performance.now() + }; + log("Posting",eventType,"message to worker dbId="+(DbState.id||'default')+':',msg); + SW.postMessage(msg); }; /** Methods which map directly to onmessage() event.type keys. @@ -103,23 +105,31 @@ const dbMsgHandler = { open: function(ev){ DbState.id = ev.dbId; - log("open result",ev.data); + log("open result",ev); }, exec: function(ev){ - log("exec result",ev.data); + log("exec result",ev); }, export: function(ev){ - log("export result",ev.data); + log("export result",ev); }, error: function(ev){ - error("ERROR from the worker:",ev.data); - logEventResult(ev.data); + error("ERROR from the worker:",ev); + logEventResult(ev); }, resultRowTest1: function f(ev){ if(undefined === f.counter) f.counter = 0; - if(ev.data) ++f.counter; - //log("exec() result row:",ev.data); - T.assert(null===ev.data || 'number' === typeof ev.data.b); + if(null === ev.rowNumber){ + /* End of result set. */ + T.assert(undefined === ev.row) + .assert(Array.isArray(ev.columnNames)) + .assert(ev.columnNames.length); + }else{ + T.assert(ev.rowNumber > 0); + ++f.counter; + } + //log("exec() result row:",ev); + T.assert(null === ev.rowNumber || 'number' === typeof ev.row.b); } }; @@ -143,33 +153,33 @@ throw new Error("This is not supposed to be reached."); }; runOneTest('exec',{ - sql: ["create table t(a,b)", + sql: ["create table t(a,b);", "insert into t(a,b) values(1,2),(3,4),(5,6)" - ].join(';'), - multi: true, + ], resultRows: [], columnNames: [] }, function(ev){ - ev = ev.data; + ev = ev.result; T.assert(0===ev.resultRows.length) .assert(0===ev.columnNames.length); }); runOneTest('exec',{ sql: 'select a a, b b from t order by a', - resultRows: [], columnNames: [], + resultRows: [], columnNames: [], saveSql:[] }, function(ev){ - ev = ev.data; + ev = ev.result; T.assert(3===ev.resultRows.length) .assert(1===ev.resultRows[0][0]) .assert(6===ev.resultRows[2][1]) .assert(2===ev.columnNames.length) .assert('b'===ev.columnNames[1]); }); + //if(1){ error("Returning prematurely for testing."); return; } runOneTest('exec',{ sql: 'select a a, b b from t order by a', resultRows: [], columnNames: [], rowMode: 'object' }, function(ev){ - ev = ev.data; + ev = ev.result; T.assert(3===ev.resultRows.length) .assert(1===ev.resultRows[0].a) .assert(6===ev.resultRows[2].b) @@ -181,7 +191,7 @@ resultRows: [], //rowMode: 'array', // array is the default in the Worker interface }, function(ev){ - ev = ev.data; + ev = ev.result; T.assert(1 === ev.resultRows.length) .assert(1 === ev.resultRows[0][0]); }); @@ -194,19 +204,19 @@ dbMsgHandler.resultRowTest1.counter = 0; }); runOneTest('exec',{ - multi: true, sql:[ - 'pragma foreign_keys=0;', + "pragma foreign_keys=0;", // ^^^ arbitrary query with no result columns - 'select a, b from t order by a desc; select a from t;' - // multi-exec only honors results from the first + "select a, b from t order by a desc;", + "select a from t;" + // multi-statement exec only honors results from the first // statement with result columns (regardless of whether) // it has any rows). ], rowMode: 1, resultRows: [] },function(ev){ - const rows = ev.data.resultRows; + const rows = ev.result.resultRows; T.assert(3===rows.length). assert(6===rows[0]); }); @@ -215,29 +225,29 @@ sql: 'select count(a) from t', resultRows: [] },function(ev){ - ev = ev.data; + ev = ev.result; T.assert(1===ev.resultRows.length) .assert(2===ev.resultRows[0][0]); }); - if(0){ - // export requires reimpl. for portability reasons. - runOneTest('export',{}, function(ev){ - ev = ev.data; - T.assert('string' === typeof ev.filename) - .assert(ev.buffer instanceof Uint8Array) - .assert(ev.buffer.length > 1024) - .assert('application/x-sqlite3' === ev.mimetype); - }); - } + runOneTest('export',{}, function(ev){ + ev = ev.result; + log("export result:",ev); + T.assert('string' === typeof ev.filename) + .assert(ev.byteArray instanceof Uint8Array) + .assert(ev.byteArray.length > 1024) + .assert('application/x-sqlite3' === ev.mimetype); + }); /***** close() tests must come last. *****/ runOneTest('close',{unlink:true},function(ev){ - ev = ev.data; + ev = ev.result; T.assert('string' === typeof ev.filename); }); runOneTest('close',{unlink:true},function(ev){ - ev = ev.data; + ev = ev.result; T.assert(undefined === ev.filename); + logHtml('warning',"This is the final test."); }); + logHtml('warning',"Finished posting tests. Waiting on async results."); }; const runTests = function(){ @@ -261,16 +271,8 @@ will fail and we have no way of cancelling them once they've been posted to the worker. - We currently do (2) because (A) it's certainly the most - client-friendly thing to do and (B) it seems likely that most - apps using this API will only have a single db to work with so - won't need to juggle multiple DB ids. If we revert to (1) then - the following call to runTests2() needs to be moved into the - callback function of the runOneTest() check for the 'open' - command. Note, also, that using approach (2) does not keep the - user from instead using approach (1), noting that doing so - requires explicit handling of the 'open' message to account for - it. + Which approach we use below depends on the boolean value of + waitForOpen. */ const waitForOpen = 1, simulateOpenError = 0 /* if true, the remaining tests will @@ -284,11 +286,12 @@ filename:'testing2.sqlite3', simulateError: simulateOpenError }, function(ev){ - //log("open result",ev); - T.assert('testing2.sqlite3'===ev.data.filename) - .assert(ev.data.dbId) - .assert(ev.data.messageId); - DbState.id = ev.data.dbId; + log("open result",ev); + T.assert('testing2.sqlite3'===ev.result.filename) + .assert(ev.dbId) + .assert(ev.messageId) + .assert('string' === typeof ev.result.vfs); + DbState.id = ev.dbId; if(waitForOpen) setTimeout(runTests2, 0); }); if(!waitForOpen) runTests2(); @@ -301,7 +304,7 @@ } ev = ev.data/*expecting a nested object*/; //log("main window onmessage:",ev); - if(ev.data && ev.data.messageId){ + if(ev.result && ev.messageId){ /* We're expecting a queued-up callback handler. */ const f = MsgHandlerQueue.shift(); if('error'===ev.type){ @@ -314,8 +317,8 @@ } switch(ev.type){ case 'sqlite3-api': - switch(ev.data){ - case 'worker-ready': + switch(ev.result){ + case 'worker1-ready': log("Message:",ev); self.sqlite3TestModule.setStatus(null); runTests(); @@ -337,4 +340,6 @@ } }; log("Init complete, but async init bits may still be running."); + log("Installing Worker into global scope SW for dev purposes."); + self.SW = SW; })(); diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make new file mode 100644 index 0000000000..5aee8af779 --- /dev/null +++ b/ext/wasm/dist.make @@ -0,0 +1,101 @@ +#!/do/not/make +#^^^ help emacs select edit mode +# +# Intended to include'd by ./GNUmakefile. +# +# 'make dist' rules for creating a distribution archive of the WASM/JS +# pieces, noting that we only build a dist of the built files, not the +# numerous pieces required to build them. +####################################################################### +MAKEFILE.dist := $(lastword $(MAKEFILE_LIST)) + +######################################################################## +# Chicken/egg situation: we need $(bin.version-info) to get the version +# info for the archive name, but that binary may not yet be built, and +# won't be built until we expand the dependencies. We have to use a +# temporary name for the archive. +dist-name = sqlite-wasm-TEMP +#ifeq (0,1) +# $(info WARNING *******************************************************************) +# $(info ** Be sure to create the desired build configuration before creating the) +# $(info ** distribution archive. Use one of the following targets to do so:) +# $(info **) +# $(info ** o2: builds with -O2, resulting in the fastest builds) +# $(info ** oz: builds with -Oz, resulting in the smallest builds) +# $(info /WARNING *******************************************************************) +#endif + +######################################################################## +# dist.build must be the name of a target which triggers the +# build of the files to be packed into the dist archive. The +# intention is that it be one of (o0, o1, o2, o3, os, oz), each of +# which uses like-named -Ox optimization level flags. The o2 target +# provides the best overall runtime speeds. The oz target provides +# slightly slower speeds (roughly 10%) with significantly smaller WASM +# file sizes. Note that -O2 (the o2 target) results in faster binaries +# than both -O3 and -Os (the o3 and os targets) in all tests run to +# date. +dist.build ?= oz + +dist-dir.top := $(dist-name) +dist-dir.jswasm := $(dist-dir.top)/$(notdir $(dir.dout)) +dist-dir.common := $(dist-dir.top)/common +dist.top.extras := \ + demo-123.html demo-123-worker.html demo-123.js \ + tester1.html tester1-worker.html tester1.js \ + demo-jsstorage.html demo-jsstorage.js \ + demo-worker1.html demo-worker1.js \ + demo-worker1-promiser.html demo-worker1-promiser.js +dist.jswasm.extras := $(sqlite3-api.ext.jses) $(sqlite3.wasm) +dist.common.extras := \ + $(wildcard $(dir.common)/*.css) \ + $(dir.common)/SqliteTestUtil.js + +.PHONY: dist +######################################################################## +# dist: create the end-user deliverable archive. +# +# Maintenance reminder: because dist depends on $(dist.build), and +# $(dist.build) will depend on clean, having any deps on +# $(dist-archive) which themselves may be cleaned up by the clean +# target will lead to grief in parallel builds (-j #). Thus +# $(dist-target)'s deps must be trimmed to non-generated files or +# files which are _not_ cleaned up by the clean target. +# +# Note that we require $(bin.version-info) in order to figure out the +# dist file's name, so cannot (without a recursive make) have the +# target name equal to the archive name. +dist: \ + $(bin.stripccomments) $(bin.version-info) \ + $(dist.build) \ + $(MAKEFILE) $(MAKEFILE.dist) + @echo "Making end-user deliverables..." + @rm -fr $(dist-dir.top) + @mkdir -p $(dist-dir.jswasm) $(dist-dir.common) + @cp -p $(dist.top.extras) $(dist-dir.top) + @cp -p README-dist.txt $(dist-dir.top)/README.txt + @cp -p index-dist.html $(dist-dir.top)/index.html + @cp -p $(dist.jswasm.extras) $(dist-dir.jswasm) + @$(bin.stripccomments) -k -k < $(sqlite3.js) \ + > $(dist-dir.jswasm)/$(notdir $(sqlite3.js)) + @cp -p $(dist.common.extras) $(dist-dir.common) + @set -e; \ + vnum=$$($(bin.version-info) --download-version); \ + vdir=sqlite-wasm-$$vnum; \ + arczip=$$vdir.zip; \ + echo "Making $$arczip ..."; \ + rm -fr $$arczip $$vdir; \ + mv $(dist-dir.top) $$vdir; \ + zip -qr $$arczip $$vdir; \ + rm -fr $$vdir; \ + ls -la $$arczip; \ + set +e; \ + unzip -lv $$arczip || echo "Missing unzip app? Not fatal." + +# We need a separate `clean` rule to account for weirdness in +# a sub-make, where we get a copy of the $(dist-name) dir +# copied into the new $(dist-name) dir. +.PHONY: dist-clean +clean: dist-clean +dist-clean: + rm -fr $(dist-name) $(wildcard sqlite-wasm-*.zip) diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make new file mode 100644 index 0000000000..43e6941f55 --- /dev/null +++ b/ext/wasm/fiddle.make @@ -0,0 +1,194 @@ +#!/do/not/make +#^^^ help emacs select edit mode +# +# Intended to include'd by ./GNUmakefile. +####################################################################### +MAKEFILE.fiddle := $(lastword $(MAKEFILE_LIST)) + +######################################################################## +# shell.c and its build flags... +make-np-0 := make -C $(dir.top) -n -p +make-np-1 := sed -e 's/(TOP)/(dir.top)/g' +$(eval $(shell $(make-np-0) | grep -e '^SHELL_OPT ' | $(make-np-1))) +$(eval $(shell $(make-np-0) | grep -e '^SHELL_SRC ' | $(make-np-1))) +# ^^^ can't do that in 1 invocation b/c newlines get stripped +ifeq (,$(SHELL_OPT)) +$(error Could not parse SHELL_OPT from $(dir.top)/Makefile.) +endif +ifeq (,$(SHELL_SRC)) +$(error Could not parse SHELL_SRC from $(dir.top)/Makefile.) +endif +$(dir.top)/shell.c: $(SHELL_SRC) $(dir.top)/tool/mkshellc.tcl + $(MAKE) -C $(dir.top) shell.c +# /shell.c +######################################################################## + +EXPORTED_FUNCTIONS.fiddle := $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle +fiddle.emcc-flags = \ + $(emcc.cflags) $(emcc_opt_full) \ + --minify 0 \ + -sALLOW_TABLE_GROWTH \ + -sABORTING_MALLOC \ + -sSTRICT_JS \ + -sENVIRONMENT=web,worker \ + -sMODULARIZE \ + -sDYNAMIC_EXECUTION=0 \ + -sWASM_BIGINT=$(emcc.WASM_BIGINT) \ + -sEXPORT_NAME=$(sqlite3.js.init-func) \ + -Wno-limited-postlink-optimizations \ + $(sqlite3.js.flags.--post-js) \ + $(emcc.exportedRuntimeMethods) \ + -sEXPORTED_FUNCTIONS=@$(abspath $(EXPORTED_FUNCTIONS.fiddle)) \ + $(SQLITE_OPT) $(SHELL_OPT) \ + -DSQLITE_SHELL_FIDDLE +# -D_POSIX_C_SOURCE is needed for strdup() with emcc + +fiddle.EXPORTED_FUNCTIONS.in := \ + EXPORTED_FUNCTIONS.fiddle.in \ + $(EXPORTED_FUNCTIONS.api) + +$(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE.fiddle) + sort -u $(fiddle.EXPORTED_FUNCTIONS.in) > $@ + +fiddle-module.js := $(dir.fiddle)/fiddle-module.js +fiddle-module.wasm := $(subst .js,.wasm,$(fiddle-module.js)) +fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.c) + +fiddle.SOAP.js := $(dir.fiddle)/$(notdir $(SOAP.js)) +$(fiddle.SOAP.js): $(SOAP.js) + cp $< $@ + +$(eval $(call call-make-pre-js,fiddle-module)) +$(fiddle-module.js): $(MAKEFILE) $(MAKEFILE.fiddle) \ + $(EXPORTED_FUNCTIONS.fiddle) \ + $(fiddle.cses) $(pre-post-fiddle-module.deps) $(fiddle.SOAP.js) + $(emcc.bin) -o $@ $(fiddle.emcc-flags) \ + $(pre-post-common.flags) $(pre-post-fiddle-module.flags) \ + $(fiddle.cses) + $(maybe-wasm-strip) $(fiddle-module.wasm) + gzip < $@ > $@.gz + gzip < $(fiddle-module.wasm) > $(fiddle-module.wasm).gz + +$(dir.fiddle)/fiddle.js.gz: $(dir.fiddle)/fiddle.js + gzip < $< > $@ + +clean: clean-fiddle +clean-fiddle: + rm -f $(fiddle-module.js) $(fiddle-module.js).gz \ + $(fiddle-module.wasm) $(fiddle-module.wasm).gz \ + $(dir.fiddle)/$(SOAP.js) \ + $(dir.fiddle)/fiddle-module.worker.js \ + EXPORTED_FUNCTIONS.fiddle +.PHONY: fiddle +fiddle: $(fiddle-module.js) $(dir.fiddle)/fiddle.js.gz +all: fiddle + +######################################################################## +# fiddle_remote is the remote destination for the fiddle app. It +# must be a [user@]HOST:/path for rsync. +# Note that the target "should probably" contain a symlink of +# index.html -> fiddle.html. +fiddle_remote ?= +ifeq (,$(fiddle_remote)) +ifneq (,$(wildcard /home/stephan)) + fiddle_remote = wh:www/wh/sqlite3/. +else ifneq (,$(wildcard /home/drh)) + #fiddle_remote = if appropriate, add that user@host:/path here +endif +endif +push-fiddle: fiddle + @if [ x = "x$(fiddle_remote)" ]; then \ + echo "fiddle_remote must be a [user@]HOST:/path for rsync"; \ + exit 1; \ + fi + rsync -va fiddle/ $(fiddle_remote) +# end fiddle remote push +######################################################################## + + +######################################################################## +# Explanation of the emcc build flags follows. Full docs for these can +# be found at: +# +# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js +# +# -sENVIRONMENT=web: elides bootstrap code related to non-web JS +# environments like node.js. Removing this makes the output a tiny +# tick larger but hypothetically makes it more portable to +# non-browser JS environments. +# +# -sMODULARIZE: changes how the generated code is structured to avoid +# declaring a global Module object and instead installing a function +# which loads and initializes the module. The function is named... +# +# -sEXPORT_NAME=jsFunctionName (see -sMODULARIZE) +# +# -sEXPORTED_RUNTIME_METHODS=@/absolute/path/to/file: a file +# containing a list of emscripten-supplied APIs, one per line, which +# must be exported into the generated JS. Must be an absolute path! +# +# -sEXPORTED_FUNCTIONS=@/absolute/path/to/file: a file containing a +# list of C functions, one per line, which must be exported via wasm +# so they're visible to JS. C symbols names in that file must all +# start with an underscore for reasons known only to the emcc +# developers. e.g., _sqlite3_open_v2 and _sqlite3_finalize. Must be +# an absolute path! +# +# -sSTRICT_JS ensures that the emitted JS code includes the 'use +# strict' option. Note that -sSTRICT is more broadly-scoped and +# results in build errors. +# +# -sALLOW_TABLE_GROWTH is required for (at a minimum) the UDF-binding +# feature. Without it, JS functions cannot be made to proxy C-side +# callbacks. +# +# -sABORTING_MALLOC causes the JS-bound _malloc() to abort rather than +# return 0 on OOM. If set to 0 then all code which uses _malloc() +# must, just like in C, check the result before using it, else +# they're likely to corrupt the JS/WASM heap by writing to its +# address of 0. It is, as of this writing, enabled in Emscripten by +# default but we enable it explicitly in case that default changes. +# +# -sDYNAMIC_EXECUTION=0 disables eval() and the Function constructor. +# If the build runs without these, it's preferable to use this flag +# because certain execution environments disallow those constructs. +# This flag is not strictly necessary, however. +# +# -sWASM_BIGINT is UNTESTED but "should" allow the int64-using C APIs +# to work with JS/wasm, insofar as the JS environment supports the +# BigInt type. That support requires an extremely recent browser: +# Safari didn't get that support until late 2020. +# +# --no-entry: for compiling library code with no main(). If this is +# not supplied and the code has a main(), it is called as part of the +# module init process. Note that main() is #if'd out of shell.c +# (renamed) when building in wasm mode. +# +# --pre-js/--post-js=FILE relative or absolute paths to JS files to +# prepend/append to the emcc-generated bootstrapping JS. It's +# easier/faster to develop with separate JS files (reduces rebuilding +# requirements) but certain configurations, namely -sMODULARIZE, may +# require using at least a --pre-js file. They can be used +# individually and need not be paired. +# +# -O0..-O3 and -Oz: optimization levels affect not only C-style +# optimization but whether or not the resulting generated JS code +# gets minified. -O0 compiles _much_ more quickly than -O3 or -Oz, +# and doesn't minimize any JS code, so is recommended for +# development. -O3 or -Oz are recommended for deployment, but +# primarily because -Oz will shrink the wasm file notably. JS-side +# minification makes little difference in terms of overall +# distributable size. +# +# --minify 0: disables minification of the generated JS code, +# regardless of optimization level. Minification of the JS has +# minimal overall effect in the larger scheme of things and results +# in JS files which can neither be edited nor viewed as text files in +# Fossil (which flags them as binary because of their extreme line +# lengths). Interestingly, whether or not the comments in the +# generated JS file get stripped is unaffected by this setting and +# depends entirely on the optimization level. Higher optimization +# levels reduce the size of the JS considerably even without +# minification. +# +######################################################################## diff --git a/ext/wasm/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js index ca562323ce..a60b79ab2e 100644 --- a/ext/wasm/fiddle/fiddle-worker.js +++ b/ext/wasm/fiddle/fiddle-worker.js @@ -89,213 +89,291 @@ */ "use strict"; (function(){ - /** - Posts a message in the form {type,data} unless passed more than 2 - args, in which case it posts {type, data:[arg1...argN]}. - */ - const wMsg = function(type,data){ - postMessage({ - type, - data: arguments.length<3 - ? data - : Array.prototype.slice.call(arguments,1) - }); - }; - - const stdout = function(){wMsg('stdout', Array.prototype.slice.call(arguments));}; - const stderr = function(){wMsg('stderr', Array.prototype.slice.call(arguments));}; - - self.onerror = function(/*message, source, lineno, colno, error*/) { - const err = arguments[4]; - if(err && 'ExitStatus'==err.name){ - /* This is relevant for the sqlite3 shell binding but not the - lower-level binding. */ - fiddleModule.isDead = true; - stderr("FATAL ERROR:", err.message); - stderr("Restarting the app requires reloading the page."); - wMsg('error', err); - } - console.error(err); - fiddleModule.setStatus('Exception thrown, see JavaScript console: '+err); - }; - - const Sqlite3Shell = { - /** Returns the name of the currently-opened db. */ - dbFilename: function f(){ - if(!f._) f._ = fiddleModule.cwrap('fiddle_db_filename', "string", ['string']); - return f._(); - }, - /** - Runs the given text through the shell as if it had been typed - in by a user. Fires a working/start event before it starts and - working/end event when it finishes. - */ - exec: function f(sql){ - if(!f._) f._ = fiddleModule.cwrap('fiddle_exec', null, ['string']); - if(fiddleModule.isDead){ - stderr("shell module has exit()ed. Cannot run SQL."); - return; - } - wMsg('working','start'); - try { - if(f._running){ - stderr('Cannot run multiple commands concurrently.'); - }else{ - f._running = true; - f._(sql); - } - } finally { - delete f._running; - wMsg('working','end'); - } - }, - resetDb: function f(){ - if(!f._) f._ = fiddleModule.cwrap('fiddle_reset_db', null); - stdout("Resetting database."); - f._(); - stdout("Reset",this.dbFilename()); - }, - /* Interrupt can't work: this Worker is tied up working, so won't get the - interrupt event which would be needed to perform the interrupt. */ - interrupt: function f(){ - if(!f._) f._ = fiddleModule.cwrap('fiddle_interrupt', null); - stdout("Requesting interrupt."); - f._(); - } - }; - - self.onmessage = function f(ev){ - ev = ev.data; - if(!f.cache){ - f.cache = { - prevFilename: null - }; - } - //console.debug("worker: onmessage.data",ev); - switch(ev.type){ - case 'shellExec': Sqlite3Shell.exec(ev.data); return; - case 'db-reset': Sqlite3Shell.resetDb(); return; - case 'interrupt': Sqlite3Shell.interrupt(); return; - /** Triggers the export of the current db. Fires an - event in the form: - - {type:'db-export', - data:{ - filename: name of db, - buffer: contents of the db file (Uint8Array), - error: on error, a message string and no buffer property. - } - } - */ - case 'db-export': { - const fn = Sqlite3Shell.dbFilename(); - stdout("Exporting",fn+"."); - const fn2 = fn ? fn.split(/[/\\]/).pop() : null; - try{ - if(!fn2) throw new Error("DB appears to be closed."); - wMsg('db-export',{ - filename: fn2, - buffer: fiddleModule.FS.readFile(fn, {encoding:"binary"}) - }); - }catch(e){ - /* Post a failure message so that UI elements disabled - during the export can be re-enabled. */ - wMsg('db-export',{ - filename: fn, - error: e.message - }); - } - return; - } - case 'open': { - /* Expects: { - buffer: ArrayBuffer | Uint8Array, - filename: for logging/informational purposes only - } */ - const opt = ev.data; - let buffer = opt.buffer; - if(buffer instanceof Uint8Array){ - }else if(buffer instanceof ArrayBuffer){ - buffer = new Uint8Array(buffer); - }else{ - stderr("'open' expects {buffer:Uint8Array} containing an uploaded db."); - return; - } - const fn = ( - opt.filename - ? opt.filename.split(/[/\\]/).pop().replace('"','_') - : ("db-"+((Math.random() * 10000000) | 0)+ - "-"+((Math.random() * 10000000) | 0)+".sqlite3") - ); - /* We cannot delete the existing db file until the new one - is installed, which means that we risk overflowing our - quota (if any) by having both the previous and current - db briefly installed in the virtual filesystem. */ - fiddleModule.FS.createDataFile("/", fn, buffer, true, true); - const oldName = Sqlite3Shell.dbFilename(); - Sqlite3Shell.exec('.open "/'+fn+'"'); - if(oldName && oldName !== fn){ - try{fiddleModule.FS.unlink(oldName);} - catch(e){/*ignored*/} - } - stdout("Replaced DB with",fn+"."); - return; - } + /** + Posts a message in the form {type,data}. If passed more than 2 + args, the 3rd must be an array of "transferable" values to pass + as the 2nd argument to postMessage(). */ + const wMsg = + (type,data,transferables)=>{ + postMessage({type, data}, transferables || []); }; - console.warn("Unknown fiddle-worker message type:",ev); - }; - + const stdout = (...args)=>wMsg('stdout', args); + const stderr = (...args)=>wMsg('stderr', args); + const toss = (...args)=>{ + throw new Error(args.join(' ')); + }; + const fixmeOPFS = "(FIXME: won't work with OPFS-over-sqlite3_vfs.)"; + let sqlite3 /* gets assigned when the wasm module is loaded */; + + self.onerror = function(/*message, source, lineno, colno, error*/) { + const err = arguments[4]; + if(err && 'ExitStatus'==err.name){ + /* This is relevant for the sqlite3 shell binding but not the + lower-level binding. */ + fiddleModule.isDead = true; + stderr("FATAL ERROR:", err.message); + stderr("Restarting the app requires reloading the page."); + wMsg('error', err); + } + console.error(err); + fiddleModule.setStatus('Exception thrown, see JavaScript console: '+err); + }; + + const Sqlite3Shell = { + /** Returns the name of the currently-opened db. */ + dbFilename: function f(){ + if(!f._) f._ = sqlite3.wasm.xWrap('fiddle_db_filename', "string", ['string']); + return f._(0); + }, + dbHandle: function f(){ + if(!f._) f._ = sqlite3.wasm.xWrap("fiddle_db_handle", "sqlite3*"); + return f._(); + }, + dbIsOpfs: function f(){ + return sqlite3.opfs && sqlite3.capi.sqlite3_js_db_uses_vfs( + this.dbHandle(), "opfs" + ); + }, + runMain: function f(){ + if(f.argv) return 0===f.argv.rc; + const dbName = "/fiddle.sqlite3"; + f.argv = [ + 'sqlite3-fiddle.wasm', + '-bail', '-safe', + dbName + /* Reminder: because of how we run fiddle, we have to ensure + that any argv strings passed to its main() are valid until + the wasm environment shuts down. */ + ]; + const capi = sqlite3.capi, wasm = sqlite3.wasm; + /* We need to call sqlite3_shutdown() in order to avoid numerous + legitimate warnings from the shell about it being initialized + after sqlite3_initialize() has been called. This means, + however, that any initialization done by the JS code may need + to be re-done (e.g. re-registration of dynamically-loaded + VFSes). We need a more generic approach to running such + init-level code. */ + capi.sqlite3_shutdown(); + f.argv.pArgv = wasm.allocMainArgv(f.argv); + f.argv.rc = wasm.exports.fiddle_main( + f.argv.length, f.argv.pArgv + ); + if(f.argv.rc){ + stderr("Fatal error initializing sqlite3 shell."); + fiddleModule.isDead = true; + return false; + } + stdout("SQLite version", capi.sqlite3_libversion(), + capi.sqlite3_sourceid().substr(0,19)); + stdout('Welcome to the "fiddle" shell.'); + if(sqlite3.opfs){ + stdout("\nOPFS is available. To open a persistent db, use:\n\n", + " .open file:name?vfs=opfs\n\nbut note that some", + "features (e.g. upload) do not yet work with OPFS."); + sqlite3.opfs.registerVfs(); + } + stdout('\nEnter ".help" for usage hints.'); + this.exec([ // initialization commands... + '.nullvalue NULL', + '.headers on' + ].join('\n')); + return true; + }, /** - emscripten module for use with build mode -sMODULARIZE. + Runs the given text through the shell as if it had been typed + in by a user. Fires a working/start event before it starts and + working/end event when it finishes. */ - const fiddleModule = { - print: stdout, - printErr: stderr, - /** - Intercepts status updates from the emscripting module init - and fires worker events with a type of 'status' and a - payload of: + exec: function f(sql){ + if(!f._){ + if(!this.runMain()) return; + f._ = sqlite3.wasm.xWrap('fiddle_exec', null, ['string']); + } + if(fiddleModule.isDead){ + stderr("shell module has exit()ed. Cannot run SQL."); + return; + } + wMsg('working','start'); + try { + if(f._running){ + stderr('Cannot run multiple commands concurrently.'); + }else if(sql){ + if(Array.isArray(sql)) sql = sql.join(''); + f._running = true; + f._(sql); + } + }finally{ + delete f._running; + wMsg('working','end'); + } + }, + resetDb: function f(){ + if(!f._) f._ = sqlite3.wasm.xWrap('fiddle_reset_db', null); + stdout("Resetting database."); + f._(); + stdout("Reset",this.dbFilename()); + }, + /* Interrupt can't work: this Worker is tied up working, so won't get the + interrupt event which would be needed to perform the interrupt. */ + interrupt: function f(){ + if(!f._) f._ = sqlite3.wasm.xWrap('fiddle_interrupt', null); + stdout("Requesting interrupt."); + f._(); + } + }; + + self.onmessage = function f(ev){ + ev = ev.data; + if(!f.cache){ + f.cache = { + prevFilename: null + }; + } + //console.debug("worker: onmessage.data",ev); + switch(ev.type){ + case 'shellExec': Sqlite3Shell.exec(ev.data); return; + case 'db-reset': Sqlite3Shell.resetDb(); return; + case 'interrupt': Sqlite3Shell.interrupt(); return; + /** Triggers the export of the current db. Fires an + event in the form: - { - text: string | null, // null at end of load process - step: integer // starts at 1, increments 1 per call - } - - We have no way of knowing in advance how many steps will - be processed/posted, so creating a "percentage done" view is - not really practical. One can be approximated by giving it a - current value of message.step and max value of message.step+1, - though. - - When work is finished, a message with a text value of null is - submitted. - - After a message with text==null is posted, the module may later - post messages about fatal problems, e.g. an exit() being - triggered, so it is recommended that UI elements for posting - status messages not be outright removed from the DOM when - text==null, and that they instead be hidden until/unless - text!=null. - */ - setStatus: function f(text){ - if(!f.last) f.last = { step: 0, text: '' }; - else if(text === f.last.text) return; - f.last.text = text; - wMsg('module',{ - type:'status', - data:{step: ++f.last.step, text: text||null} + {type:'db-export', + data:{ + filename: name of db, + buffer: contents of the db file (Uint8Array), + error: on error, a message string and no buffer property. + } + } + */ + case 'db-export': { + const fn = Sqlite3Shell.dbFilename(); + stdout("Exporting",fn+"."); + const fn2 = fn ? fn.split(/[/\\]/).pop() : null; + try{ + if(!fn2) toss("DB appears to be closed."); + const buffer = sqlite3.capi.sqlite3_js_db_export( + Sqlite3Shell.dbHandle() + ); + wMsg('db-export',{filename: fn2, buffer: buffer.buffer}, [buffer.buffer]); + }catch(e){ + console.error("Export failed:",e); + /* Post a failure message so that UI elements disabled + during the export can be re-enabled. */ + wMsg('db-export',{ + filename: fn, + error: e.message }); + } + return; + } + case 'open': { + /* Expects: { + buffer: ArrayBuffer | Uint8Array, + filename: the filename for the db. Any dir part is + stripped. + } + */ + const opt = ev.data; + let buffer = opt.buffer; + stderr('open():',fixmeOPFS); + if(buffer instanceof ArrayBuffer){ + buffer = new Uint8Array(buffer); + }else if(!(buffer instanceof Uint8Array)){ + stderr("'open' expects {buffer:Uint8Array} containing an uploaded db."); + return; + } + const fn = ( + opt.filename + ? opt.filename.split(/[/\\]/).pop().replace('"','_') + : ("db-"+((Math.random() * 10000000) | 0)+ + "-"+((Math.random() * 10000000) | 0)+".sqlite3") + ); + try { + /* We cannot delete the existing db file until the new one + is installed, which means that we risk overflowing our + quota (if any) by having both the previous and current + db briefly installed in the virtual filesystem. */ + const fnAbs = '/'+fn; + const oldName = Sqlite3Shell.dbFilename(); + if(oldName && oldName===fnAbs){ + /* We cannot create the replacement file while the current file + is opened, nor does the shell have a .close command, so we + must temporarily switch to another db... */ + Sqlite3Shell.exec('.open :memory:'); + fiddleModule.FS.unlink(fnAbs); + } + fiddleModule.FS.createDataFile("/", fn, buffer, true, true); + Sqlite3Shell.exec('.open "'+fnAbs+'"'); + if(oldName && oldName!==fnAbs){ + try{fiddleModule.fsUnlink(oldName)} + catch(e){/*ignored*/} + } + stdout("Replaced DB with",fn+"."); + }catch(e){ + stderr("Error installing db",fn+":",e.message); + } + return; } }; - - importScripts('fiddle-module.js'); + console.warn("Unknown fiddle-worker message type:",ev); + }; + + /** + emscripten module for use with build mode -sMODULARIZE. + */ + const fiddleModule = { + print: stdout, + printErr: stderr, /** - initFiddleModule() is installed via fiddle-module.js due to - building with: + Intercepts status updates from the emscripting module init + and fires worker events with a type of 'status' and a + payload of: - emcc ... -sMODULARIZE=1 -sEXPORT_NAME=initFiddleModule + { + text: string | null, // null at end of load process + step: integer // starts at 1, increments 1 per call + } + + We have no way of knowing in advance how many steps will + be processed/posted, so creating a "percentage done" view is + not really practical. One can be approximated by giving it a + current value of message.step and max value of message.step+1, + though. + + When work is finished, a message with a text value of null is + submitted. + + After a message with text==null is posted, the module may later + post messages about fatal problems, e.g. an exit() being + triggered, so it is recommended that UI elements for posting + status messages not be outright removed from the DOM when + text==null, and that they instead be hidden until/unless + text!=null. */ - initFiddleModule(fiddleModule).then(function(thisModule){ - wMsg('fiddle-ready'); - }); + setStatus: function f(text){ + if(!f.last) f.last = { step: 0, text: '' }; + else if(text === f.last.text) return; + f.last.text = text; + wMsg('module',{ + type:'status', + data:{step: ++f.last.step, text: text||null} + }); + } + }; + + importScripts('fiddle-module.js'+self.location.search); + /** + initFiddleModule() is installed via fiddle-module.js due to + building with: + + emcc ... -sMODULARIZE=1 -sEXPORT_NAME=initFiddleModule + */ + sqlite3InitModule(fiddleModule).then((_sqlite3)=>{ + sqlite3 = _sqlite3; + const dbVfs = sqlite3.wasm.xWrap('fiddle_db_vfs', "*", ['string']); + fiddleModule.fsUnlink = (fn)=>{ + return sqlite3.wasm.sqlite3_wasm_vfs_unlink(dbVfs(0), fn); + }; + wMsg('fiddle-ready'); + })/*then()*/; })(); diff --git a/ext/wasm/fiddle/fiddle.js b/ext/wasm/fiddle/fiddle.js index 619ce4eca8..2a3d1746f3 100644 --- a/ext/wasm/fiddle/fiddle.js +++ b/ext/wasm/fiddle/fiddle.js @@ -15,195 +15,195 @@ communication between the UI and worker. */ (function(){ + 'use strict'; + /* Recall that the 'self' symbol, except where locally + overwritten, refers to the global window or worker object. */ + + const storage = (function(NS/*namespace object in which to store this module*/){ + /* Pedantic licensing note: this code originated in the Fossil SCM + source tree, where it has a different license, but the person who + ported it into sqlite is the same one who wrote it for fossil. */ 'use strict'; - /* Recall that the 'self' symbol, except where locally - overwritten, refers to the global window or worker object. */ - - const storage = (function(NS/*namespace object in which to store this module*/){ - /* Pedantic licensing note: this code originated in the Fossil SCM - source tree, where it has a different license, but the person who - ported it into sqlite is the same one who wrote it for fossil. */ - 'use strict'; - NS = NS||{}; - - /** - This module provides a basic wrapper around localStorage - or sessionStorage or a dummy proxy object if neither - of those are available. - */ - const tryStorage = function f(obj){ - if(!f.key) f.key = 'storage.access.check'; - try{ - obj.setItem(f.key, 'f'); - const x = obj.getItem(f.key); - obj.removeItem(f.key); - if(x!=='f') throw new Error(f.key+" failed") - return obj; - }catch(e){ - return undefined; - } - }; - - /** Internal storage impl for this module. */ - const $storage = - tryStorage(window.localStorage) - || tryStorage(window.sessionStorage) - || tryStorage({ - // A basic dummy xyzStorage stand-in - $$$:{}, - setItem: function(k,v){this.$$$[k]=v}, - getItem: function(k){ - return this.$$$.hasOwnProperty(k) ? this.$$$[k] : undefined; - }, - removeItem: function(k){delete this.$$$[k]}, - clear: function(){this.$$$={}} - }); - - /** - For the dummy storage we need to differentiate between - $storage and its real property storage for hasOwnProperty() - to work properly... - */ - const $storageHolder = $storage.hasOwnProperty('$$$') ? $storage.$$$ : $storage; - - /** - A prefix which gets internally applied to all storage module - property keys so that localStorage and sessionStorage across the - same browser profile instance do not "leak" across multiple apps - being hosted by the same origin server. Such cross-polination is - still there but, with this key prefix applied, it won't be - immediately visible via the storage API. - - With this in place we can justify using localStorage instead of - sessionStorage. - - One implication of using localStorage and sessionStorage is that - their scope (the same "origin" and client application/profile) - allows multiple apps on the same origin to use the same - storage. Thus /appA/foo could then see changes made via - /appB/foo. The data do not cross user- or browser boundaries, - though, so it "might" arguably be called a - feature. storageKeyPrefix was added so that we can sandbox that - state for each separate app which shares an origin. - - See: https://fossil-scm.org/forum/forumpost/4afc4d34de - - Sidebar: it might seem odd to provide a key prefix and stick all - properties in the topmost level of the storage object. We do that - because adding a layer of object to sandbox each app would mean - (de)serializing that whole tree on every storage property change. - e.g. instead of storageObject.projectName.foo we have - storageObject[storageKeyPrefix+'foo']. That's soley for - efficiency's sake (in terms of battery life and - environment-internal storage-level effort). - */ - const storageKeyPrefix = ( - $storageHolder===$storage/*localStorage or sessionStorage*/ - ? ( - (NS.config ? - (NS.config.projectCode || NS.config.projectName - || NS.config.shortProjectName) - : false) - || window.location.pathname - )+'::' : ( - '' /* transient storage */ - ) - ); - - /** - A proxy for localStorage or sessionStorage or a - page-instance-local proxy, if neither one is availble. - - Which exact storage implementation is uses is unspecified, and - apps must not rely on it. - */ - NS.storage = { - storageKeyPrefix: storageKeyPrefix, - /** Sets the storage key k to value v, implicitly converting - it to a string. */ - set: (k,v)=>$storage.setItem(storageKeyPrefix+k,v), - /** Sets storage key k to JSON.stringify(v). */ - setJSON: (k,v)=>$storage.setItem(storageKeyPrefix+k,JSON.stringify(v)), - /** Returns the value for the given storage key, or - dflt if the key is not found in the storage. */ - get: (k,dflt)=>$storageHolder.hasOwnProperty( - storageKeyPrefix+k - ) ? $storage.getItem(storageKeyPrefix+k) : dflt, - /** Returns true if the given key has a value of "true". If the - key is not found, it returns true if the boolean value of dflt - is "true". (Remember that JS persistent storage values are all - strings.) */ - getBool: function(k,dflt){ - return 'true'===this.get(k,''+(!!dflt)); - }, - /** Returns the JSON.parse()'d value of the given - storage key's value, or dflt is the key is not - found or JSON.parse() fails. */ - getJSON: function f(k,dflt){ - try { - const x = this.get(k,f); - return x===f ? dflt : JSON.parse(x); - } - catch(e){return dflt} - }, - /** Returns true if the storage contains the given key, - else false. */ - contains: (k)=>$storageHolder.hasOwnProperty(storageKeyPrefix+k), - /** Removes the given key from the storage. Returns this. */ - remove: function(k){ - $storage.removeItem(storageKeyPrefix+k); - return this; - }, - /** Clears ALL keys from the storage. Returns this. */ - clear: function(){ - this.keys().forEach((k)=>$storage.removeItem(/*w/o prefix*/k)); - return this; - }, - /** Returns an array of all keys currently in the storage. */ - keys: ()=>Object.keys($storageHolder).filter((v)=>(v||'').startsWith(storageKeyPrefix)), - /** Returns true if this storage is transient (only available - until the page is reloaded), indicating that fileStorage - and sessionStorage are unavailable. */ - isTransient: ()=>$storageHolder!==$storage, - /** Returns a symbolic name for the current storage mechanism. */ - storageImplName: function(){ - if($storage===window.localStorage) return 'localStorage'; - else if($storage===window.sessionStorage) return 'sessionStorage'; - else return 'transient'; - }, - - /** - Returns a brief help text string for the currently-selected - storage type. - */ - storageHelpDescription: function(){ - return { - localStorage: "Browser-local persistent storage with an "+ - "unspecified long-term lifetime (survives closing the browser, "+ - "but maybe not a browser upgrade).", - sessionStorage: "Storage local to this browser tab, "+ - "lost if this tab is closed.", - "transient": "Transient storage local to this invocation of this page." - }[this.storageImplName()]; - } - }; - return NS.storage; - })({})/*storage API setup*/; - - - /** Name of the stored copy of SqliteFiddle.config. */ - const configStorageKey = 'sqlite3-fiddle-config'; + NS = NS||{}; /** - The SqliteFiddle object is intended to be the primary - app-level object for the main-thread side of the sqlite - fiddle application. It uses a worker thread to load the - sqlite WASM module and communicate with it. + This module provides a basic wrapper around localStorage + or sessionStorage or a dummy proxy object if neither + of those are available. */ - const SF/*local convenience alias*/ - = window.SqliteFiddle/*canonical name*/ = { - /* Config options. */ - config: { + const tryStorage = function f(obj){ + if(!f.key) f.key = 'storage.access.check'; + try{ + obj.setItem(f.key, 'f'); + const x = obj.getItem(f.key); + obj.removeItem(f.key); + if(x!=='f') throw new Error(f.key+" failed") + return obj; + }catch(e){ + return undefined; + } + }; + + /** Internal storage impl for this module. */ + const $storage = + tryStorage(window.localStorage) + || tryStorage(window.sessionStorage) + || tryStorage({ + // A basic dummy xyzStorage stand-in + $$$:{}, + setItem: function(k,v){this.$$$[k]=v}, + getItem: function(k){ + return this.$$$.hasOwnProperty(k) ? this.$$$[k] : undefined; + }, + removeItem: function(k){delete this.$$$[k]}, + clear: function(){this.$$$={}} + }); + + /** + For the dummy storage we need to differentiate between + $storage and its real property storage for hasOwnProperty() + to work properly... + */ + const $storageHolder = $storage.hasOwnProperty('$$$') ? $storage.$$$ : $storage; + + /** + A prefix which gets internally applied to all storage module + property keys so that localStorage and sessionStorage across the + same browser profile instance do not "leak" across multiple apps + being hosted by the same origin server. Such cross-polination is + still there but, with this key prefix applied, it won't be + immediately visible via the storage API. + + With this in place we can justify using localStorage instead of + sessionStorage. + + One implication of using localStorage and sessionStorage is that + their scope (the same "origin" and client application/profile) + allows multiple apps on the same origin to use the same + storage. Thus /appA/foo could then see changes made via + /appB/foo. The data do not cross user- or browser boundaries, + though, so it "might" arguably be called a + feature. storageKeyPrefix was added so that we can sandbox that + state for each separate app which shares an origin. + + See: https://fossil-scm.org/forum/forumpost/4afc4d34de + + Sidebar: it might seem odd to provide a key prefix and stick all + properties in the topmost level of the storage object. We do that + because adding a layer of object to sandbox each app would mean + (de)serializing that whole tree on every storage property change. + e.g. instead of storageObject.projectName.foo we have + storageObject[storageKeyPrefix+'foo']. That's soley for + efficiency's sake (in terms of battery life and + environment-internal storage-level effort). + */ + const storageKeyPrefix = ( + $storageHolder===$storage/*localStorage or sessionStorage*/ + ? ( + (NS.config ? + (NS.config.projectCode || NS.config.projectName + || NS.config.shortProjectName) + : false) + || window.location.pathname + )+'::' : ( + '' /* transient storage */ + ) + ); + + /** + A proxy for localStorage or sessionStorage or a + page-instance-local proxy, if neither one is availble. + + Which exact storage implementation is uses is unspecified, and + apps must not rely on it. + */ + NS.storage = { + storageKeyPrefix: storageKeyPrefix, + /** Sets the storage key k to value v, implicitly converting + it to a string. */ + set: (k,v)=>$storage.setItem(storageKeyPrefix+k,v), + /** Sets storage key k to JSON.stringify(v). */ + setJSON: (k,v)=>$storage.setItem(storageKeyPrefix+k,JSON.stringify(v)), + /** Returns the value for the given storage key, or + dflt if the key is not found in the storage. */ + get: (k,dflt)=>$storageHolder.hasOwnProperty( + storageKeyPrefix+k + ) ? $storage.getItem(storageKeyPrefix+k) : dflt, + /** Returns true if the given key has a value of "true". If the + key is not found, it returns true if the boolean value of dflt + is "true". (Remember that JS persistent storage values are all + strings.) */ + getBool: function(k,dflt){ + return 'true'===this.get(k,''+(!!dflt)); + }, + /** Returns the JSON.parse()'d value of the given + storage key's value, or dflt is the key is not + found or JSON.parse() fails. */ + getJSON: function f(k,dflt){ + try { + const x = this.get(k,f); + return x===f ? dflt : JSON.parse(x); + } + catch(e){return dflt} + }, + /** Returns true if the storage contains the given key, + else false. */ + contains: (k)=>$storageHolder.hasOwnProperty(storageKeyPrefix+k), + /** Removes the given key from the storage. Returns this. */ + remove: function(k){ + $storage.removeItem(storageKeyPrefix+k); + return this; + }, + /** Clears ALL keys from the storage. Returns this. */ + clear: function(){ + this.keys().forEach((k)=>$storage.removeItem(/*w/o prefix*/k)); + return this; + }, + /** Returns an array of all keys currently in the storage. */ + keys: ()=>Object.keys($storageHolder).filter((v)=>(v||'').startsWith(storageKeyPrefix)), + /** Returns true if this storage is transient (only available + until the page is reloaded), indicating that fileStorage + and sessionStorage are unavailable. */ + isTransient: ()=>$storageHolder!==$storage, + /** Returns a symbolic name for the current storage mechanism. */ + storageImplName: function(){ + if($storage===window.localStorage) return 'localStorage'; + else if($storage===window.sessionStorage) return 'sessionStorage'; + else return 'transient'; + }, + + /** + Returns a brief help text string for the currently-selected + storage type. + */ + storageHelpDescription: function(){ + return { + localStorage: "Browser-local persistent storage with an "+ + "unspecified long-term lifetime (survives closing the browser, "+ + "but maybe not a browser upgrade).", + sessionStorage: "Storage local to this browser tab, "+ + "lost if this tab is closed.", + "transient": "Transient storage local to this invocation of this page." + }[this.storageImplName()]; + } + }; + return NS.storage; + })({})/*storage API setup*/; + + + /** Name of the stored copy of SqliteFiddle.config. */ + const configStorageKey = 'sqlite3-fiddle-config'; + + /** + The SqliteFiddle object is intended to be the primary + app-level object for the main-thread side of the sqlite + fiddle application. It uses a worker thread to load the + sqlite WASM module and communicate with it. + */ + const SF/*local convenience alias*/ + = window.SqliteFiddle/*canonical name*/ = { + /* Config options. */ + config: { /* If true, SqliteFiddle.echo() will auto-scroll the output widget to the bottom when it receives output, else it won't. */ @@ -219,24 +219,24 @@ sideBySide: true, /* If true, swap positions of the input/output areas. */ swapInOut: false - }, - /** - Emits the given text, followed by a line break, to the - output widget. If given more than one argument, they are - join()'d together with a space between each. As a special - case, if passed a single array, that array is used in place - of the arguments array (this is to facilitate receiving - lists of arguments via worker events). - */ - echo: function f(text) { + }, + /** + Emits the given text, followed by a line break, to the + output widget. If given more than one argument, they are + join()'d together with a space between each. As a special + case, if passed a single array, that array is used in place + of the arguments array (this is to facilitate receiving + lists of arguments via worker events). + */ + echo: function f(text) { /* Maintenance reminder: we currently require/expect a textarea output element. It might be nice to extend this to behave differently if the output element is a non-textarea element, in which case it would need to append the given text as a TEXT node and add a line break. */ if(!f._){ - f._ = document.getElementById('output'); - f._.value = ''; // clear browser cache + f._ = document.getElementById('output'); + f._.value = ''; // clear browser cache } if(arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' '); else if(1===arguments.length && Array.isArray(text)) text = text.join(' '); @@ -246,564 +246,570 @@ //text = text.replace(/>/g, ">"); //text = text.replace('\n', '
', 'g'); if(null===text){/*special case: clear output*/ - f._.value = ''; - return; + f._.value = ''; + return; }else if(this.echo._clearPending){ - delete this.echo._clearPending; - f._.value = ''; + delete this.echo._clearPending; + f._.value = ''; } if(this.config.echoToConsole) console.log(text); if(this.jqTerm) this.jqTerm.echo(text); f._.value += text + "\n"; if(this.config.autoScrollOutput){ - f._.scrollTop = f._.scrollHeight; + f._.scrollTop = f._.scrollHeight; } - }, - _msgMap: {}, - /** Adds a worker message handler for messages of the given - type. */ - addMsgHandler: function f(type,callback){ + }, + _msgMap: {}, + /** Adds a worker message handler for messages of the given + type. */ + addMsgHandler: function f(type,callback){ if(Array.isArray(type)){ - type.forEach((t)=>this.addMsgHandler(t, callback)); - return this; + type.forEach((t)=>this.addMsgHandler(t, callback)); + return this; } (this._msgMap.hasOwnProperty(type) ? this._msgMap[type] : (this._msgMap[type] = [])).push(callback); return this; - }, - /** Given a worker message, runs all handlers for msg.type. */ - runMsgHandlers: function(msg){ + }, + /** Given a worker message, runs all handlers for msg.type. */ + runMsgHandlers: function(msg){ const list = (this._msgMap.hasOwnProperty(msg.type) ? this._msgMap[msg.type] : false); if(!list){ - console.warn("No handlers found for message type:",msg); - return false; + console.warn("No handlers found for message type:",msg); + return false; } //console.debug("runMsgHandlers",msg); list.forEach((f)=>f(msg)); return true; - }, - /** Removes all message handlers for the given message type. */ - clearMsgHandlers: function(type){ + }, + /** Removes all message handlers for the given message type. */ + clearMsgHandlers: function(type){ delete this._msgMap[type]; return this; - }, - /* Posts a message in the form {type, data} to the db worker. Returns this. */ - wMsg: function(type,data){ - this.worker.postMessage({type, data}); + }, + /* Posts a message in the form {type, data} to the db worker. Returns this. */ + wMsg: function(type,data,transferables){ + this.worker.postMessage({type, data}, transferables || []); return this; - }, - /** - Prompts for confirmation and, if accepted, deletes - all content and tables in the (transient) database. - */ - resetDb: function(){ + }, + /** + Prompts for confirmation and, if accepted, deletes + all content and tables in the (transient) database. + */ + resetDb: function(){ if(window.confirm("Really destroy all content and tables " +"in the (transient) db?")){ - this.wMsg('db-reset'); + this.wMsg('db-reset'); } return this; - }, - /** Stores this object's config in the browser's storage. */ - storeConfig: function(){ + }, + /** Stores this object's config in the browser's storage. */ + storeConfig: function(){ storage.setJSON(configStorageKey,this.config); - } - }; + } + }; - if(1){ /* Restore SF.config */ - const storedConfig = storage.getJSON(configStorageKey); - if(storedConfig){ - /* Copy all properties to SF.config which are currently in - storedConfig. We don't bother copying any other - properties: those have been removed from the app in the - meantime. */ - Object.keys(SF.config).forEach(function(k){ - if(storedConfig.hasOwnProperty(k)){ - SF.config[k] = storedConfig[k]; - } - }); + if(1){ /* Restore SF.config */ + const storedConfig = storage.getJSON(configStorageKey); + if(storedConfig){ + /* Copy all properties to SF.config which are currently in + storedConfig. We don't bother copying any other + properties: those have been removed from the app in the + meantime. */ + Object.keys(SF.config).forEach(function(k){ + if(storedConfig.hasOwnProperty(k)){ + SF.config[k] = storedConfig[k]; } + }); } + } - SF.worker = new Worker('fiddle-worker.js'); - SF.worker.onmessage = (ev)=>SF.runMsgHandlers(ev.data); - SF.addMsgHandler(['stdout', 'stderr'], (ev)=>SF.echo(ev.data)); + SF.worker = new Worker('fiddle-worker.js'+self.location.search); + SF.worker.onmessage = (ev)=>SF.runMsgHandlers(ev.data); + SF.addMsgHandler(['stdout', 'stderr'], (ev)=>SF.echo(ev.data)); - /* querySelectorAll() proxy */ - const EAll = function(/*[element=document,] cssSelector*/){ - return (arguments.length>1 ? arguments[0] : document) - .querySelectorAll(arguments[arguments.length-1]); - }; - /* querySelector() proxy */ - const E = function(/*[element=document,] cssSelector*/){ - return (arguments.length>1 ? arguments[0] : document) - .querySelector(arguments[arguments.length-1]); - }; + /* querySelectorAll() proxy */ + const EAll = function(/*[element=document,] cssSelector*/){ + return (arguments.length>1 ? arguments[0] : document) + .querySelectorAll(arguments[arguments.length-1]); + }; + /* querySelector() proxy */ + const E = function(/*[element=document,] cssSelector*/){ + return (arguments.length>1 ? arguments[0] : document) + .querySelector(arguments[arguments.length-1]); + }; - /** Handles status updates from the Module object. */ - SF.addMsgHandler('module', function f(ev){ - ev = ev.data; - if('status'!==ev.type){ - console.warn("Unexpected module-type message:",ev); - return; - } - if(!f.ui){ - f.ui = { - status: E('#module-status'), - progress: E('#module-progress'), - spinner: E('#module-spinner') - }; - } - const msg = ev.data; - if(f.ui.progres){ - progress.value = msg.step; - progress.max = msg.step + 1/*we don't know how many steps to expect*/; - } - if(1==msg.step){ - f.ui.progress.classList.remove('hidden'); - f.ui.spinner.classList.remove('hidden'); - } - if(msg.text){ - f.ui.status.classList.remove('hidden'); - f.ui.status.innerText = msg.text; - }else{ - if(f.ui.progress){ - f.ui.progress.remove(); - f.ui.spinner.remove(); - delete f.ui.progress; - delete f.ui.spinner; - } - f.ui.status.classList.add('hidden'); - /* The module can post messages about fatal problems, - e.g. an exit() being triggered or assertion failure, - after the last "load" message has arrived, so - leave f.ui.status and message listener intact. */ - } - }); + /** Handles status updates from the Emscripten Module object. */ + SF.addMsgHandler('module', function f(ev){ + ev = ev.data; + if('status'!==ev.type){ + console.warn("Unexpected module-type message:",ev); + return; + } + if(!f.ui){ + f.ui = { + status: E('#module-status'), + progress: E('#module-progress'), + spinner: E('#module-spinner') + }; + } + const msg = ev.data; + if(f.ui.progres){ + progress.value = msg.step; + progress.max = msg.step + 1/*we don't know how many steps to expect*/; + } + if(1==msg.step){ + f.ui.progress.classList.remove('hidden'); + f.ui.spinner.classList.remove('hidden'); + } + if(msg.text){ + f.ui.status.classList.remove('hidden'); + f.ui.status.innerText = msg.text; + }else{ + if(f.ui.progress){ + f.ui.progress.remove(); + f.ui.spinner.remove(); + delete f.ui.progress; + delete f.ui.spinner; + } + f.ui.status.classList.add('hidden'); + /* The module can post messages about fatal problems, + e.g. an exit() being triggered or assertion failure, + after the last "load" message has arrived, so + leave f.ui.status and message listener intact. */ + } + }); - /** - The 'fiddle-ready' event is fired (with no payload) when the - wasm module has finished loading. Interestingly, that happens - _before_ the final module:status event */ - SF.addMsgHandler('fiddle-ready', function(){ - SF.clearMsgHandlers('fiddle-ready'); - self.onSFLoaded(); - }); - - /** - Performs all app initialization which must wait until after the - worker module is loaded. This function removes itself when it's - called. - */ - self.onSFLoaded = function(){ - delete this.onSFLoaded; - // Unhide all elements which start out hidden - EAll('.initially-hidden').forEach((e)=>e.classList.remove('initially-hidden')); - E('#btn-reset').addEventListener('click',()=>SF.resetDb()); - const taInput = E('#input'); - const btnClearIn = E('#btn-clear'); - btnClearIn.addEventListener('click',function(){ - taInput.value = ''; - },false); - // Ctrl-enter and shift-enter both run the current SQL. - taInput.addEventListener('keydown',function(ev){ - if((ev.ctrlKey || ev.shiftKey) && 13 === ev.keyCode){ - ev.preventDefault(); - ev.stopPropagation(); - btnShellExec.click(); - } - }, false); - const taOutput = E('#output'); - const btnClearOut = E('#btn-clear-output'); - btnClearOut.addEventListener('click',function(){ - taOutput.value = ''; - if(SF.jqTerm) SF.jqTerm.clear(); - },false); - const btnShellExec = E('#btn-shell-exec'); - btnShellExec.addEventListener('click',function(ev){ - let sql; - ev.preventDefault(); - if(taInput.selectionStart e.addEventListener('click', cmdClick, false) - ); - - btnInterrupt.addEventListener('click',function(){ - SF.wMsg('interrupt'); - }); - - /** Initiate a download of the db. */ - const btnExport = E('#btn-export'); - const eLoadDb = E('#load-db'); - const btnLoadDb = E('#btn-load-db'); - btnLoadDb.addEventListener('click', ()=>eLoadDb.click()); - /** - Enables (if passed true) or disables all UI elements which - "might," if timed "just right," interfere with an - in-progress db import/export/exec operation. - */ - const enableMutatingElements = function f(enable){ - if(!f._elems){ - f._elems = [ - /* UI elements to disable while import/export are - running. Normally the export is fast enough - that this won't matter, but we really don't - want to be reading (from outside of sqlite) the - db when the user taps btnShellExec. */ - btnShellExec, btnExport, eLoadDb - ]; - } - f._elems.forEach( enable - ? (e)=>e.removeAttribute('disabled') - : (e)=>e.setAttribute('disabled','disabled') ); - }; - btnExport.addEventListener('click',function(){ - enableMutatingElements(false); - SF.wMsg('db-export'); - }); - SF.addMsgHandler('db-export', function(ev){ - enableMutatingElements(true); - ev = ev.data; - if(ev.error){ - SF.echo("Export failed:",ev.error); - return; - } - const blob = new Blob([ev.buffer], {type:"application/x-sqlite3"}); - const a = document.createElement('a'); - document.body.appendChild(a); - a.href = window.URL.createObjectURL(blob); - a.download = ev.filename; - a.addEventListener('click',function(){ - setTimeout(function(){ - SF.echo("Exported (possibly auto-downloaded):",ev.filename); - window.URL.revokeObjectURL(a.href); - a.remove(); - },500); - }); - a.click(); - }); - /** - Handle load/import of an external db file. - */ - eLoadDb.addEventListener('change',function(){ - const f = this.files[0]; - const r = new FileReader(); - const status = {loaded: 0, total: 0}; - enableMutatingElements(false); - r.addEventListener('loadstart', function(){ - SF.echo("Loading",f.name,"..."); - }); - r.addEventListener('progress', function(ev){ - SF.echo("Loading progress:",ev.loaded,"of",ev.total,"bytes."); - }); - const that = this; - r.addEventListener('load', function(){ - enableMutatingElements(true); - SF.echo("Loaded",f.name+". Opening db..."); - SF.wMsg('open',{ - filename: f.name, - buffer: this.result - }); - }); - r.addEventListener('error',function(){ - enableMutatingElements(true); - SF.echo("Loading",f.name,"failed for unknown reasons."); - }); - r.addEventListener('abort',function(){ - enableMutatingElements(true); - SF.echo("Cancelled loading of",f.name+"."); - }); - r.readAsArrayBuffer(f); - }); - - EAll('fieldset.collapsible').forEach(function(fs){ - const btnToggle = E(fs,'legend > .fieldset-toggle'), - content = EAll(fs,':scope > div'); - btnToggle.addEventListener('click', function(){ - fs.classList.toggle('collapsed'); - content.forEach((d)=>d.classList.toggle('hidden')); - }, false); - }); - - /** - Given a DOM element, this routine measures its "effective - height", which is the bounding top/bottom range of this element - and all of its children, recursively. For some DOM structure - cases, a parent may have a reported height of 0 even though - children have non-0 sizes. - - Returns 0 if !e or if the element really has no height. - */ - const effectiveHeight = function f(e){ - if(!e) return 0; - if(!f.measure){ - f.measure = function callee(e, depth){ - if(!e) return; - const m = e.getBoundingClientRect(); - if(0===depth){ - callee.top = m.top; - callee.bottom = m.bottom; - }else{ - callee.top = m.top ? Math.min(callee.top, m.top) : callee.top; - callee.bottom = Math.max(callee.bottom, m.bottom); - } - Array.prototype.forEach.call(e.children,(e)=>callee(e,depth+1)); - if(0===depth){ - //console.debug("measure() height:",e.className, callee.top, callee.bottom, (callee.bottom - callee.top)); - f.extra += callee.bottom - callee.top; - } - return f.extra; - }; - } - f.extra = 0; - f.measure(e,0); - return f.extra; - }; - - /** - Returns a function, that, as long as it continues to be invoked, - will not be triggered. The function will be called after it stops - being called for N milliseconds. If `immediate` is passed, call - the callback immediately and hinder future invocations until at - least the given time has passed. - - If passed only 1 argument, or passed a falsy 2nd argument, - the default wait time set in this function's $defaultDelay - property is used. - - Source: underscore.js, by way of https://davidwalsh.name/javascript-debounce-function - */ - const debounce = function f(func, wait, immediate) { - var timeout; - if(!wait) wait = f.$defaultDelay; - return function() { - const context = this, args = Array.prototype.slice.call(arguments); - const later = function() { - timeout = undefined; - if(!immediate) func.apply(context, args); - }; - const callNow = immediate && !timeout; - clearTimeout(timeout); - timeout = setTimeout(later, wait); - if(callNow) func.apply(context, args); - }; - }; - debounce.$defaultDelay = 500 /*arbitrary*/; - - const ForceResizeKludge = (function(){ - /* Workaround for Safari mayhem regarding use of vh CSS - units.... We cannot use vh units to set the main view - size because Safari chokes on that, so we calculate - that height here. Larger than ~95% is too big for - Firefox on Android, causing the input area to move - off-screen. */ - const appViews = EAll('.app-view'); - const elemsToCount = [ - /* Elements which we need to always count in the - visible body size. */ - E('body > header'), - E('body > footer') - ]; - const resized = function f(){ - if(f.$disabled) return; - const wh = window.innerHeight; - var ht; - var extra = 0; - elemsToCount.forEach((e)=>e ? extra += effectiveHeight(e) : false); - ht = wh - extra; - appViews.forEach(function(e){ - e.style.height = - e.style.maxHeight = [ - "calc(", (ht>=100 ? ht : 100), "px", - " - 2em"/*fudge value*/,")" - /* ^^^^ hypothetically not needed, but both - Chrome/FF on Linux will force scrollbars on the - body if this value is too small. */ - ].join(''); - }); - }; - resized.$disabled = true/*gets deleted when setup is finished*/; - window.addEventListener('resize', debounce(resized, 250), false); - return resized; - })(); - - /** Set up a selection list of examples */ - (function(){ - const xElem = E('#select-examples'); - const examples = [ - {name: "Help", sql: -`-- ================================================ --- Use ctrl-enter or shift-enter to execute sqlite3 --- shell commands and SQL. --- If a subset of the text is currently selected, --- only that part is executed. --- ================================================ -.help`}, - {name: "Timer on", sql: ".timer on"}, - {name: "Setup table T", sql:`.nullvalue NULL -CREATE TABLE t(a,b); -INSERT INTO t(a,b) VALUES('abc',123),('def',456),(NULL,789),('ghi',012); -SELECT * FROM t;`}, - {name: "Table list", sql: ".tables"}, - {name: "Box Mode", sql: ".mode box"}, - {name: "JSON Mode", sql: ".mode json"}, - {name: "Mandlebrot", sql: `WITH RECURSIVE - xaxis(x) AS (VALUES(-2.0) UNION ALL SELECT x+0.05 FROM xaxis WHERE x<1.2), - yaxis(y) AS (VALUES(-1.0) UNION ALL SELECT y+0.1 FROM yaxis WHERE y<1.0), - m(iter, cx, cy, x, y) AS ( - SELECT 0, x, y, 0.0, 0.0 FROM xaxis, yaxis - UNION ALL - SELECT iter+1, cx, cy, x*x-y*y + cx, 2.0*x*y + cy FROM m - WHERE (x*x + y*y) < 4.0 AND iter<28 - ), - m2(iter, cx, cy) AS ( - SELECT max(iter), cx, cy FROM m GROUP BY cx, cy - ), - a(t) AS ( - SELECT group_concat( substr(' .+*#', 1+min(iter/7,4), 1), '') - FROM m2 GROUP BY cy - ) -SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a;`} - ]; - const newOpt = function(lbl,val){ - const o = document.createElement('option'); - o.value = val; - if(!val) o.setAttribute('disabled',true); - o.appendChild(document.createTextNode(lbl)); - xElem.appendChild(o); - }; - newOpt("Examples (replaces input!)"); - examples.forEach((o)=>newOpt(o.name, o.sql)); - //xElem.setAttribute('disabled',true); - xElem.selectedIndex = 0; - xElem.addEventListener('change', function(){ - taInput.value = '-- ' + - this.selectedOptions[0].innerText + - '\n' + this.value; - SF.dbExec(this.value); - }); - })()/* example queries */; - - SF.echo(null/*clear any output generated by the init process*/); - if(window.jQuery && window.jQuery.terminal){ - /* Set up the terminal-style view... */ - const eTerm = window.jQuery('#view-terminal').empty(); - SF.jqTerm = eTerm.terminal(SF.dbExec.bind(SF),{ - prompt: 'sqlite> ', - greetings: false /* note that the docs incorrectly call this 'greeting' */ - }); - /* Set up a button to toggle the views... */ - const head = E('header#titlebar'); - const btnToggleView = document.createElement('button'); - btnToggleView.appendChild(document.createTextNode("Toggle View")); - head.appendChild(btnToggleView); - btnToggleView.addEventListener('click',function f(){ - EAll('.app-view').forEach(e=>e.classList.toggle('hidden')); - if(document.body.classList.toggle('terminal-mode')){ - ForceResizeKludge(); - } - }, false); - btnToggleView.click()/*default to terminal view*/; - } - SF.dbExec(null/*init the db and output the header*/); - SF.echo('This experimental app is provided in the hope that it', - 'may prove interesting or useful but is not an officially', - 'supported deliverable of the sqlite project. It is subject to', - 'any number of changes or outright removal at any time.\n'); - delete ForceResizeKludge.$disabled; - ForceResizeKludge(); + /** + The 'fiddle-ready' event is fired (with no payload) when the + wasm module has finished loading. Interestingly, that happens + _before_ the final module:status event */ + SF.addMsgHandler('fiddle-ready', function(){ + SF.clearMsgHandlers('fiddle-ready'); + self.onSFLoaded(); + }); + /** + Performs all app initialization which must wait until after the + worker module is loaded. This function removes itself when it's + called. + */ + self.onSFLoaded = function(){ + delete this.onSFLoaded; + // Unhide all elements which start out hidden + EAll('.initially-hidden').forEach((e)=>e.classList.remove('initially-hidden')); + E('#btn-reset').addEventListener('click',()=>SF.resetDb()); + const taInput = E('#input'); + const btnClearIn = E('#btn-clear'); + btnClearIn.addEventListener('click',function(){ + taInput.value = ''; + },false); + // Ctrl-enter and shift-enter both run the current SQL. + taInput.addEventListener('keydown',function(ev){ + if((ev.ctrlKey || ev.shiftKey) && 13 === ev.keyCode){ + ev.preventDefault(); + ev.stopPropagation(); btnShellExec.click(); - }/*onSFLoaded()*/; + } + }, false); + const taOutput = E('#output'); + const btnClearOut = E('#btn-clear-output'); + btnClearOut.addEventListener('click',function(){ + taOutput.value = ''; + if(SF.jqTerm) SF.jqTerm.clear(); + },false); + const btnShellExec = E('#btn-shell-exec'); + btnShellExec.addEventListener('click',function(ev){ + let sql; + ev.preventDefault(); + if(taInput.selectionStart e.addEventListener('click', cmdClick, false) + ); + + btnInterrupt.addEventListener('click',function(){ + SF.wMsg('interrupt'); + }); + + /** Initiate a download of the db. */ + const btnExport = E('#btn-export'); + const eLoadDb = E('#load-db'); + const btnLoadDb = E('#btn-load-db'); + btnLoadDb.addEventListener('click', ()=>eLoadDb.click()); + /** + Enables (if passed true) or disables all UI elements which + "might," if timed "just right," interfere with an + in-progress db import/export/exec operation. + */ + const enableMutatingElements = function f(enable){ + if(!f._elems){ + f._elems = [ + /* UI elements to disable while import/export are + running. Normally the export is fast enough + that this won't matter, but we really don't + want to be reading (from outside of sqlite) the + db when the user taps btnShellExec. */ + btnShellExec, btnExport, eLoadDb + ]; + } + f._elems.forEach( enable + ? (e)=>e.removeAttribute('disabled') + : (e)=>e.setAttribute('disabled','disabled') ); + }; + btnExport.addEventListener('click',function(){ + enableMutatingElements(false); + SF.wMsg('db-export'); + }); + SF.addMsgHandler('db-export', function(ev){ + enableMutatingElements(true); + ev = ev.data; + if(ev.error){ + SF.echo("Export failed:",ev.error); + return; + } + const blob = new Blob([ev.buffer], + {type:"application/x-sqlite3"}); + const a = document.createElement('a'); + document.body.appendChild(a); + a.href = window.URL.createObjectURL(blob); + a.download = ev.filename; + a.addEventListener('click',function(){ + setTimeout(function(){ + SF.echo("Exported (possibly auto-downloaded):",ev.filename); + window.URL.revokeObjectURL(a.href); + a.remove(); + },500); + }); + a.click(); + }); + /** + Handle load/import of an external db file. + */ + eLoadDb.addEventListener('change',function(){ + const f = this.files[0]; + const r = new FileReader(); + const status = {loaded: 0, total: 0}; + enableMutatingElements(false); + r.addEventListener('loadstart', function(){ + SF.echo("Loading",f.name,"..."); + }); + r.addEventListener('progress', function(ev){ + SF.echo("Loading progress:",ev.loaded,"of",ev.total,"bytes."); + }); + const that = this; + r.addEventListener('load', function(){ + enableMutatingElements(true); + SF.echo("Loaded",f.name+". Opening db..."); + SF.wMsg('open',{ + filename: f.name, + buffer: this.result + }, [this.result]); + }); + r.addEventListener('error',function(){ + enableMutatingElements(true); + SF.echo("Loading",f.name,"failed for unknown reasons."); + }); + r.addEventListener('abort',function(){ + enableMutatingElements(true); + SF.echo("Cancelled loading of",f.name+"."); + }); + r.readAsArrayBuffer(f); + }); + + EAll('fieldset.collapsible').forEach(function(fs){ + const btnToggle = E(fs,'legend > .fieldset-toggle'), + content = EAll(fs,':scope > div'); + btnToggle.addEventListener('click', function(){ + fs.classList.toggle('collapsed'); + content.forEach((d)=>d.classList.toggle('hidden')); + }, false); + }); + + /** + Given a DOM element, this routine measures its "effective + height", which is the bounding top/bottom range of this element + and all of its children, recursively. For some DOM structure + cases, a parent may have a reported height of 0 even though + children have non-0 sizes. + + Returns 0 if !e or if the element really has no height. + */ + const effectiveHeight = function f(e){ + if(!e) return 0; + if(!f.measure){ + f.measure = function callee(e, depth){ + if(!e) return; + const m = e.getBoundingClientRect(); + if(0===depth){ + callee.top = m.top; + callee.bottom = m.bottom; + }else{ + callee.top = m.top ? Math.min(callee.top, m.top) : callee.top; + callee.bottom = Math.max(callee.bottom, m.bottom); + } + Array.prototype.forEach.call(e.children,(e)=>callee(e,depth+1)); + if(0===depth){ + //console.debug("measure() height:",e.className, callee.top, callee.bottom, (callee.bottom - callee.top)); + f.extra += callee.bottom - callee.top; + } + return f.extra; + }; + } + f.extra = 0; + f.measure(e,0); + return f.extra; + }; + + /** + Returns a function, that, as long as it continues to be invoked, + will not be triggered. The function will be called after it stops + being called for N milliseconds. If `immediate` is passed, call + the callback immediately and hinder future invocations until at + least the given time has passed. + + If passed only 1 argument, or passed a falsy 2nd argument, + the default wait time set in this function's $defaultDelay + property is used. + + Source: underscore.js, by way of https://davidwalsh.name/javascript-debounce-function + */ + const debounce = function f(func, wait, immediate) { + var timeout; + if(!wait) wait = f.$defaultDelay; + return function() { + const context = this, args = Array.prototype.slice.call(arguments); + const later = function() { + timeout = undefined; + if(!immediate) func.apply(context, args); + }; + const callNow = immediate && !timeout; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + if(callNow) func.apply(context, args); + }; + }; + debounce.$defaultDelay = 500 /*arbitrary*/; + + const ForceResizeKludge = (function(){ + /* Workaround for Safari mayhem regarding use of vh CSS + units.... We cannot use vh units to set the main view + size because Safari chokes on that, so we calculate + that height here. Larger than ~95% is too big for + Firefox on Android, causing the input area to move + off-screen. */ + const appViews = EAll('.app-view'); + const elemsToCount = [ + /* Elements which we need to always count in the + visible body size. */ + E('body > header'), + E('body > footer') + ]; + const resized = function f(){ + if(f.$disabled) return; + const wh = window.innerHeight; + var ht; + var extra = 0; + elemsToCount.forEach((e)=>e ? extra += effectiveHeight(e) : false); + ht = wh - extra; + appViews.forEach(function(e){ + e.style.height = + e.style.maxHeight = [ + "calc(", (ht>=100 ? ht : 100), "px", + " - 2em"/*fudge value*/,")" + /* ^^^^ hypothetically not needed, but both + Chrome/FF on Linux will force scrollbars on the + body if this value is too small. */ + ].join(''); + }); + }; + resized.$disabled = true/*gets deleted when setup is finished*/; + window.addEventListener('resize', debounce(resized, 250), false); + return resized; + })(); + + /** Set up a selection list of examples */ + (function(){ + const xElem = E('#select-examples'); + const examples = [ + {name: "Help", sql: [ + "-- ================================================\n", + "-- Use ctrl-enter or shift-enter to execute sqlite3\n", + "-- shell commands and SQL.\n", + "-- If a subset of the text is currently selected,\n", + "-- only that part is executed.\n", + "-- ================================================\n", + ".help\n" + ]}, + //{name: "Timer on", sql: ".timer on"}, + // ^^^ re-enable if emscripten re-enables getrusage() + {name: "Setup table T", sql:[ + ".nullvalue NULL\n", + "CREATE TABLE t(a,b);\n", + "INSERT INTO t(a,b) VALUES('abc',123),('def',456),(NULL,789),('ghi',012);\n", + "SELECT * FROM t;\n" + ]}, + {name: "Table list", sql: ".tables"}, + {name: "Box Mode", sql: ".mode box"}, + {name: "JSON Mode", sql: ".mode json"}, + {name: "Mandlebrot", sql:[ + "WITH RECURSIVE", + " xaxis(x) AS (VALUES(-2.0) UNION ALL SELECT x+0.05 FROM xaxis WHERE x<1.2),\n", + " yaxis(y) AS (VALUES(-1.0) UNION ALL SELECT y+0.1 FROM yaxis WHERE y<1.0),\n", + " m(iter, cx, cy, x, y) AS (\n", + " SELECT 0, x, y, 0.0, 0.0 FROM xaxis, yaxis\n", + " UNION ALL\n", + " SELECT iter+1, cx, cy, x*x-y*y + cx, 2.0*x*y + cy FROM m \n", + " WHERE (x*x + y*y) < 4.0 AND iter<28\n", + " ),\n", + " m2(iter, cx, cy) AS (\n", + " SELECT max(iter), cx, cy FROM m GROUP BY cx, cy\n", + " ),\n", + " a(t) AS (\n", + " SELECT group_concat( substr(' .+*#', 1+min(iter/7,4), 1), '') \n", + " FROM m2 GROUP BY cy\n", + " )\n", + "SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a;\n", + ]} + ]; + const newOpt = function(lbl,val){ + const o = document.createElement('option'); + if(Array.isArray(val)) val = val.join(''); + o.value = val; + if(!val) o.setAttribute('disabled',true); + o.appendChild(document.createTextNode(lbl)); + xElem.appendChild(o); + }; + newOpt("Examples (replaces input!)"); + examples.forEach((o)=>newOpt(o.name, o.sql)); + //xElem.setAttribute('disabled',true); + xElem.selectedIndex = 0; + xElem.addEventListener('change', function(){ + taInput.value = '-- ' + + this.selectedOptions[0].innerText + + '\n' + this.value; + SF.dbExec(this.value); + }); + })()/* example queries */; + + //SF.echo(null/*clear any output generated by the init process*/); + if(window.jQuery && window.jQuery.terminal){ + /* Set up the terminal-style view... */ + const eTerm = window.jQuery('#view-terminal').empty(); + SF.jqTerm = eTerm.terminal(SF.dbExec.bind(SF),{ + prompt: 'sqlite> ', + greetings: false /* note that the docs incorrectly call this 'greeting' */ + }); + /* Set up a button to toggle the views... */ + const head = E('header#titlebar'); + const btnToggleView = document.createElement('button'); + btnToggleView.appendChild(document.createTextNode("Toggle View")); + head.appendChild(btnToggleView); + btnToggleView.addEventListener('click',function f(){ + EAll('.app-view').forEach(e=>e.classList.toggle('hidden')); + if(document.body.classList.toggle('terminal-mode')){ + ForceResizeKludge(); + } + }, false); + btnToggleView.click()/*default to terminal view*/; + } + SF.echo('This experimental app is provided in the hope that it', + 'may prove interesting or useful but is not an officially', + 'supported deliverable of the sqlite project. It is subject to', + 'any number of changes or outright removal at any time.\n'); + const urlParams = new URL(self.location.href).searchParams; + SF.dbExec(urlParams.get('sql') || null); + delete ForceResizeKludge.$disabled; + ForceResizeKludge(); + }/*onSFLoaded()*/; })(); diff --git a/ext/wasm/index-dist.html b/ext/wasm/index-dist.html new file mode 100644 index 0000000000..6b038c82a9 --- /dev/null +++ b/ext/wasm/index-dist.html @@ -0,0 +1,90 @@ + + + + + + + sqlite3 WASM Demo Page Index + + + +
sqlite3 WASM demo pages
+
+
Below is the list of demo pages for the sqlite3 WASM + builds. The intent is that this page be run + using the functional equivalent of:
+ 
althttpd -enable-sab -page index.html
+
and the individual pages be started in their own tab. + Warnings and Caveats: +
    +
  • Some of these pages require that the web server emit the + so-called + COOP + and + COEP + headers. althttpd requires the + -enable-sab flag for that. +
    • +
+
+
The tests and demos... +
    +
  • Core-most tests +
      +
    • tester1: Core unit and + regression tests for the various APIs and surrounding + utility code.
      • +
    • tester1-worker: same thing + but running in a Worker.
      • +
    +
    • +
  • Higher-level apps and demos... +
      +
    • demo-123 provides a + no-nonsense example of adding sqlite3 support to a web + page in the UI thread.
      • +
    • demo-123-worker is + the same as demo-123 but loads and runs + sqlite3 from a Worker thread.
      • +
    • demo-jsstorage: very basic + demo of using the key-value VFS for storing a persistent db + in JS localStorage or sessionStorage.
      • +
    • demo-worker1: + Worker-based wrapper of the OO API #1. Its Promise-based + wrapper is significantly easier to use, however.
      • +
    • demo-worker1-promiser: + a demo of the Promise-based wrapper of the Worker1 API.
      • +
    +
    • +
+
+ + + + diff --git a/ext/wasm/index.html b/ext/wasm/index.html new file mode 100644 index 0000000000..044cd1360d --- /dev/null +++ b/ext/wasm/index.html @@ -0,0 +1,115 @@ + + + + + + + + sqlite3 WASM Testing Page Index + + +
sqlite3 WASM test pages
+
+
Below is the list of test pages for the sqlite3 WASM + builds. All of them require that this directory have been + "make"d first. The intent is that this page be run + using:
+ 
althttpd -enable-sab -page index.html
+
and the individual tests be started in their own tab. + Warnings and Caveats: +
    +
  • Some of these pages require that + the web server emit the so-called + COOP + and + COEP + headers. althttpd requires the + -enable-sab flag for that. +
    • +
  • Any OPFS-related pages require very recent version of + Chrome or Chromium (v102 at least, possibly newer). OPFS + support in the other major browsers is pending. Development + and testing is currently done against a dev-channel release + of Chrome (v107 as of 2022-09-26). +
    • +
  • Whether or not WASMFS/OPFS support is enabled on any given + page may depend on build-time options which are off by + default. +
    • +
+
+
The tests and demos... +
    +
  • Core-most tests +
      +
    • tester1: Core unit and + regression tests for the various APIs and surrounding + utility code.
      • +
    • tester1-worker: same thing + but running in a Worker.
      • +
    +
    • +
  • High-level apps and demos... +
      +
    • fiddle is an HTML front-end + to a wasm build of the sqlite3 shell.
      • +
    • demo-123 provides a + no-nonsense example of adding sqlite3 support to a web + page in the UI thread.
      • +
    • demo-123-worker is + the same as demo-123 but loads and runs + sqlite3 from a Worker thread.
      • +
    • demo-jsstorage: very basic + demo of using the key-value VFS for storing a persistent db + in JS localStorage or sessionStorage.
      • +
    • demo-worker1: + Worker-based wrapper of the OO API #1. Its Promise-based + wrapper is significantly easier to use, however.
      • +
    • demo-worker1-promiser: + a demo of the Promise-based wrapper of the Worker1 API.
      • +
    +
    • +
  • speedtest1 ports (sqlite3's primary benchmarking tool)... + +
    • +
  • The obligatory "misc." category... + +
    • + +
+
+ + + + diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index a018658579..dee7258c51 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -361,7 +361,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ framework's native format or in Emscripten format. */ const __memberSignature = function f(obj,memberName,emscriptenFormat=false){ - if(!f._) f._ = (x)=>x.replace(/[^vipPsjrd]/g,'').replace(/[pPs]/g,'i'); + if(!f._) f._ = (x)=>x.replace(/[^vipPsjrd]/g,"").replace(/[pPs]/g,'i'); const m = __lookupMember(obj.structInfo, memberName, true); return emscriptenFormat ? f._(m.signature) : m.signature; }; @@ -394,7 +394,17 @@ self.Jaccwabyt = function StructBinderFactory(config){ const __utf8Decoder = new TextDecoder('utf-8'); const __utf8Encoder = new TextEncoder(); - + /** Internal helper to use in operations which need to distinguish + between SharedArrayBuffer heap memory and non-shared heap. */ + const __SAB = ('undefined'===typeof SharedArrayBuffer) + ? function(){} : SharedArrayBuffer; + const __utf8Decode = function(arrayBuffer, begin, end){ + return __utf8Decoder.decode( + (arrayBuffer.buffer instanceof __SAB) + ? arrayBuffer.slice(begin, end) + : arrayBuffer.subarray(begin, end) + ); + }; /** Uses __lookupMember() to find the given obj.structInfo key. Returns that member if it is a string, else returns false. If the @@ -437,8 +447,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ //log("mem[",pos,"]",mem[pos]); }; //log("addr =",addr,"pos =",pos); - if(addr===pos) return ""; - return __utf8Decoder.decode(new Uint8Array(mem.buffer, addr, pos-addr)); + return (addr===pos) ? "" : __utf8Decode(mem, addr, pos); }; /** diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index 2bb39e636f..edcba260a7 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -809,9 +809,7 @@ common: A read-only numeric property which is the "pointer" returned by the configured allocator when this object is constructed. After `dispose()` (inherited from [StructType][]) is called, this property - has the `undefined` value. When passing instances of this struct to - C-bound code, `pointer` is the value which must be passed in place - of a C-side struct pointer. When calling C-side code which takes a + has the `undefined` value. When calling C-side code which takes a pointer to a struct of this type, simply pass it `myStruct.pointer`. diff --git a/ext/wasm/jaccwabyt/jaccwabyt_test.c b/ext/wasm/jaccwabyt/jaccwabyt_test.c deleted file mode 100644 index 7e2db394c6..0000000000 --- a/ext/wasm/jaccwabyt/jaccwabyt_test.c +++ /dev/null @@ -1,178 +0,0 @@ -#include -#include /* memset() */ -#include /* offsetof() */ -#include /* snprintf() */ -#include /* int64_t */ -/*#include */ /* malloc/free(), needed for emscripten exports. */ -extern void * malloc(size_t); -extern void free(void *); - -/* -** 2022-06-25 -** -** The author disclaims copyright to this source code. In place of a -** legal notice, here is a blessing: -** -** * May you do good and not evil. -** * May you find forgiveness for yourself and forgive others. -** * May you share freely, never taking more than you give. -** -*********************************************************************** -** -** Utility functions for use with the emscripten/WASM bits. These -** functions ARE NOT part of the sqlite3 public API. They are strictly -** for internal use by the JS/WASM bindings. -** -** This file is intended to be WASM-compiled together with sqlite3.c, -** e.g.: -** -** emcc ... sqlite3.c wasm_util.c -*/ - -/* -** Experimenting with output parameters. -*/ -int jaccwabyt_test_intptr(int * p){ - if(1==((int)p)%3){ - /* kludge to get emscripten to export malloc() and free() */; - free(malloc(0)); - } - return *p = *p * 2; -} -int64_t jaccwabyt_test_int64_max(void){ - return (int64_t)0x7fffffffffffffff; -} -int64_t jaccwabyt_test_int64_min(void){ - return ~jaccwabyt_test_int64_max(); -} -int64_t jaccwabyt_test_int64_times2(int64_t x){ - return x * 2; -} - -void jaccwabyt_test_int64_minmax(int64_t * min, int64_t *max){ - *max = jaccwabyt_test_int64_max(); - *min = jaccwabyt_test_int64_min(); - /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ -} -int64_t jaccwabyt_test_int64ptr(int64_t * p){ - /*printf("jaccwabyt_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ - return *p = *p * 2; -} - -void jaccwabyt_test_stack_overflow(int recurse){ - if(recurse) jaccwabyt_test_stack_overflow(recurse); -} - -struct WasmTestStruct { - int v4; - void * ppV; - const char * cstr; - int64_t v8; - void (*xFunc)(void*); -}; -typedef struct WasmTestStruct WasmTestStruct; -void jaccwabyt_test_struct(WasmTestStruct * s){ - if(s){ - s->v4 *= 2; - s->v8 = s->v4 * 2; - s->ppV = s; - s->cstr = __FILE__; - if(s->xFunc) s->xFunc(s); - } - return; -} - -/** For testing the 'string-free' whwasmutil.xWrap() conversion. */ -char * jaccwabyt_test_str_hello(int fail){ - char * s = fail ? 0 : (char *)malloc(6); - if(s){ - memcpy(s, "hello", 5); - s[5] = 0; - } - return s; -} - -/* -** Returns a NUL-terminated string containing a JSON-format metadata -** regarding C structs, for use with the StructBinder API. The -** returned memory is static and is only written to the first time -** this is called. -*/ -const char * jaccwabyt_test_ctype_json(void){ - static char strBuf[1024 * 8] = {0}; - int n = 0, structCount = 0, groupCount = 0; - char * pos = &strBuf[1] /* skip first byte for now to help protect - against a small race condition */; - char const * const zEnd = pos + sizeof(strBuf); - if(strBuf[0]) return strBuf; - /* Leave first strBuf[0] at 0 until the end to help guard against a - tiny race condition. If this is called twice concurrently, they - might end up both writing to strBuf, but they'll both write the - same thing, so that's okay. If we set byte 0 up front then the - 2nd instance might return a partially-populated string. */ - - //////////////////////////////////////////////////////////////////// - // First we need to build up our macro framework... - //////////////////////////////////////////////////////////////////// - // Core output macros... -#define lenCheck assert(pos < zEnd - 100) -#define outf(format,...) \ - pos += snprintf(pos, ((size_t)(zEnd - pos)), format, __VA_ARGS__); \ - lenCheck -#define out(TXT) outf("%s",TXT) -#define CloseBrace(LEVEL) \ - assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck - - //////////////////////////////////////////////////////////////////// - // Macros for emitting StructBinder descriptions... -#define StructBinder__(TYPE) \ - n = 0; \ - outf("%s{", (structCount++ ? ", " : "")); \ - out("\"name\": \"" # TYPE "\","); \ - outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ - out(",\"members\": {"); -#define StructBinder_(T) StructBinder__(T) -// ^^^ indirection needed to expand CurrentStruct -#define StructBinder StructBinder_(CurrentStruct) -#define _StructBinder CloseBrace(2) -#define M(MEMBER,SIG) \ - outf("%s\"%s\": " \ - "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ - (n++ ? ", " : ""), #MEMBER, \ - (int)offsetof(CurrentStruct,MEMBER), \ - (int)sizeof(((CurrentStruct*)0)->MEMBER), \ - SIG) - // End of macros - //////////////////////////////////////////////////////////////////// - - out("\"structs\": ["); { - -#define CurrentStruct WasmTestStruct - StructBinder { - M(v4,"i"); - M(cstr,"s"); - M(ppV,"p"); - M(v8,"j"); - M(xFunc,"v(p)"); - } _StructBinder; -#undef CurrentStruct - - } out( "]"/*structs*/); - out("}"/*top-level object*/); - *pos = 0; - strBuf[0] = '{'/*end of the race-condition workaround*/; - return strBuf; -#undef DefGroup -#undef Def -#undef _DefGroup -#undef StructBinder -#undef StructBinder_ -#undef StructBinder__ -#undef M -#undef _StructBinder -#undef CurrentStruct -#undef CloseBrace -#undef out -#undef outf -#undef lenCheck -} diff --git a/ext/wasm/jaccwabyt/jaccwabyt_test.exports b/ext/wasm/jaccwabyt/jaccwabyt_test.exports deleted file mode 100644 index b6182207b5..0000000000 --- a/ext/wasm/jaccwabyt/jaccwabyt_test.exports +++ /dev/null @@ -1,10 +0,0 @@ -_jaccwabyt_test_intptr -_jaccwabyt_test_int64ptr -_jaccwabyt_test_int64_max -_jaccwabyt_test_int64_min -_jaccwabyt_test_int64_minmax -_jaccwabyt_test_int64_times2 -_jaccwabyt_test_struct -_jaccwabyt_test_ctype_json -_jaccwabyt_test_stack_overflow -_jaccwabyt_test_str_hello diff --git a/ext/wasm/module-symbols.html b/ext/wasm/module-symbols.html new file mode 100644 index 0000000000..bebefffac8 --- /dev/null +++ b/ext/wasm/module-symbols.html @@ -0,0 +1,333 @@ + + + + + + + sqlite3 Module Symbols + + + +
+ + +

Loading WASM module... + If this takes "a long time" it may have failed and the browser's + dev console may contain hints as to why. +

+ +

+ This page lists the SQLite3 APIs exported + by sqlite3.wasm and exposed to clients + by sqlite3.js. These lists are generated dynamically + by loading the JS/WASM module and introspecting it, with the following + caveats: +

+ +
    +
  • Some APIs are explicitly filtered out of these lists because + they are strictly for internal use within the JS/WASM APIs and + its own test code. +
    • +
  • This page runs in the main UI thread so cannot see features + which are only available in a Worker thread. If this page were + to function via a Worker, it would not be able to see + functionality only available in the main thread. Starting a + Worker here to fetch those symbols requires loading a second + copy of the sqlite3 WASM module and JS code. +
    • +
+ +
+ +

This page exposes a global symbol named sqlite3 + which can be inspected using the browser's dev tools. +

+ +

Jump to...

+ + + +

sqlite3 Namespace

+

+ The sqlite3 namespace object exposes the following... +

+
+ + +

sqlite3.version Object

+

+ The sqlite3.version object exposes the following... +

+
+ + +

sqlite3_...() Function List

+ +

The sqlite3.capi namespace exposes the following + sqlite3_...() + functions... +

+
+

+ = function is specific to the JS/WASM + bindings, not part of the C API. +

+ + +

SQLITE_... Constants

+ +

The sqlite3.capi namespace exposes the following + SQLITE_... + constants... +

+
+ + +

sqlite3.oo1 Namespace

+

+ The sqlite3.oo1 namespace exposes the following... +

+
+ + +

sqlite3.wasm Namespace

+

+ The sqlite3.wasm namespace exposes the + following... +

+
+ + +

sqlite3.wasm.pstack Namespace

+

+ The sqlite3.wasm.pstack namespace exposes the + following... +

+
+ + +

Compilation Options

+

+ SQLITE_... compilation options used in this build + of sqlite3.wasm... +

+
+ +
+ + +
+ diff --git a/ext/wasm/scratchpad-wasmfs-main.html b/ext/wasm/scratchpad-wasmfs-main.html new file mode 100644 index 0000000000..91f61526cd --- /dev/null +++ b/ext/wasm/scratchpad-wasmfs-main.html @@ -0,0 +1,40 @@ + + + + + + + + + sqlite3 WASMFS/OPFS Main-thread Scratchpad + + +
sqlite3 WASMFS/OPFS Main-thread Scratchpad
+ +
+ +
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
+

Scratchpad/test app for the WASMF/OPFS integration in the + main window thread. This page requires that the sqlite3 API have + been built with WASMFS support. If OPFS support is available then + it "should" persist a database across reloads (watch the dev console + output), otherwise it will not. +

+

All stuff on this page happens in the dev console.

+
+
+ + + + + diff --git a/ext/wasm/scratchpad-wasmfs-main.js b/ext/wasm/scratchpad-wasmfs-main.js new file mode 100644 index 0000000000..56f9325de5 --- /dev/null +++ b/ext/wasm/scratchpad-wasmfs-main.js @@ -0,0 +1,70 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-api.js. This file must be run in + main JS thread and sqlite3.js must have been loaded before it. +*/ +'use strict'; +(function(){ + const toss = function(...args){throw new Error(args.join(' '))}; + const log = console.log.bind(console), + warn = console.warn.bind(console), + error = console.error.bind(console); + + const stdout = log; + const stderr = error; + + const test1 = function(db){ + db.exec("create table if not exists t(a);") + .transaction(function(db){ + db.prepare("insert into t(a) values(?)") + .bind(new Date().getTime()) + .stepFinalize(); + stdout("Number of values in table t:", + db.selectValue("select count(*) from t")); + }); + }; + + const runTests = function(sqlite3){ + const capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = sqlite3.wasm; + stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + const persistentDir = capi.sqlite3_wasmfs_opfs_dir(); + if(persistentDir){ + stdout("Persistent storage dir:",persistentDir); + }else{ + stderr("No persistent storage available."); + } + const startTime = performance.now(); + let db; + try { + db = new oo.DB(persistentDir+'/foo.db'); + stdout("DB filename:",db.filename); + const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', + banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; + [ + test1 + ].forEach((f)=>{ + const n = performance.now(); + stdout(banner1,"Running",f.name+"()..."); + f(db, sqlite3); + stdout(banner2,f.name+"() took ",(performance.now() - n),"ms"); + }); + }finally{ + if(db) db.close(); + } + stdout("Total test time:",(performance.now() - startTime),"ms"); + }; + + sqlite3InitModule(self.sqlite3TestModule).then(runTests); +})(); diff --git a/ext/wasm/speedtest1-wasmfs.html b/ext/wasm/speedtest1-wasmfs.html new file mode 100644 index 0000000000..e35546702e --- /dev/null +++ b/ext/wasm/speedtest1-wasmfs.html @@ -0,0 +1,149 @@ + + + + + + + + + speedtest1-wasmfs.wasm + + +
speedtest1-wasmfs.wasm
+ + +
+ +
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
+
This page starts running the main exe when it loads, which will + block the UI until it finishes! Adding UI controls to manually configure and start it + are TODO.
+ +
Achtung: running it with the dev tools open may + drastically slow it down. For faster results, keep the dev + tools closed when running it! +
+
Output is delayed/buffered because we cannot update the UI while the + speedtest is running. Output will appear below when ready... +
+ + + + + diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html new file mode 100644 index 0000000000..5bdd32acf5 --- /dev/null +++ b/ext/wasm/speedtest1-worker.html @@ -0,0 +1,369 @@ + + + + + + + + + speedtest1.wasm Worker + + +
speedtest1.wasm Worker
+ + +
+ +
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
+ +
+ + + + +
+
+
+
+ Tips: +
    +
  • Control-click the flags to (de)select multiple flags.
    • +
  • The --big-transactions flag is important for two + of the bigger tests. Without it, those tests create a + combined total of 140k implicit transactions, reducing their + speed to an absolute crawl, especially when WASMFS is + activated. +
    • +
  • The easiest way to try different optimization levels is, + from this directory: + 
    $ rm -f speedtest1.js; make -e emcc_opt='-O2' speedtest1.js
    + Then reload this page. -O2 seems to consistently produce the fastest results. +
    • +
+
+ + + + diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js new file mode 100644 index 0000000000..c61cab9190 --- /dev/null +++ b/ext/wasm/speedtest1-worker.js @@ -0,0 +1,99 @@ +'use strict'; +(function(){ + let speedtestJs = 'speedtest1.js'; + const urlParams = new URL(self.location.href).searchParams; + if(urlParams.has('sqlite3.dir')){ + speedtestJs = urlParams.get('sqlite3.dir') + '/' + speedtestJs; + } + importScripts('common/whwasmutil.js', speedtestJs); + /** + If this environment contains OPFS, this function initializes it and + returns the name of the dir on which OPFS is mounted, else it returns + an empty string. + */ + const wasmfsDir = function f(wasmUtil){ + if(undefined !== f._) return f._; + const pdir = '/opfs'; + if( !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return f._ = ""; + } + try{ + if(0===wasmUtil.xCallWrapped( + 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir + )){ + return f._ = pdir; + }else{ + return f._ = ""; + } + }catch(e){ + // sqlite3_wasm_init_wasmfs() is not available + return f._ = ""; + } + }; + wasmfsDir._ = undefined; + + const mPost = function(msgType,payload){ + postMessage({type: msgType, data: payload}); + }; + + const App = Object.create(null); + App.logBuffer = []; + const logMsg = (type,msgArgs)=>{ + const msg = msgArgs.join(' '); + App.logBuffer.push(msg); + mPost(type,msg); + }; + const log = (...args)=>logMsg('stdout',args); + const logErr = (...args)=>logMsg('stderr',args); + + const runSpeedtest = function(cliFlagsArray){ + const scope = App.wasm.scopedAllocPush(); + const dbFile = App.pDir+"/speedtest1.sqlite3"; + try{ + const argv = [ + "speedtest1.wasm", ...cliFlagsArray, dbFile + ]; + App.logBuffer.length = 0; + mPost('run-start', [...argv]); + App.wasm.xCall('wasm_main', argv.length, + App.wasm.scopedAllocMainArgv(argv)); + }catch(e){ + mPost('error',e.message); + }finally{ + App.wasm.scopedAllocPop(scope); + mPost('run-end', App.logBuffer.join('\n')); + App.logBuffer.length = 0; + } + }; + + self.onmessage = function(msg){ + msg = msg.data; + switch(msg.type){ + case 'run': runSpeedtest(msg.data || []); break; + default: + logErr("Unhandled worker message type:",msg.type); + break; + } + }; + + const EmscriptenModule = { + print: log, + printErr: logErr, + setStatus: (text)=>mPost('load-status',text) + }; + self.sqlite3InitModule(EmscriptenModule).then((sqlite3)=>{ + const S = sqlite3; + App.vfsUnlink = function(pDb, fname){ + const pVfs = S.wasm.sqlite3_wasm_db_vfs(pDb, 0); + if(pVfs) S.wasm.sqlite3_wasm_vfs_unlink(pVfs, fname||0); + }; + App.pDir = wasmfsDir(S.wasm); + App.wasm = S.wasm; + //if(App.pDir) log("Persistent storage:",pDir); + //else log("Using transient storage."); + mPost('ready',true); + log("Registered VFSes:", ...S.capi.sqlite3_js_vfs_list()); + }); +})(); diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html new file mode 100644 index 0000000000..5286b9e482 --- /dev/null +++ b/ext/wasm/speedtest1.html @@ -0,0 +1,174 @@ + + + + + + + + + speedtest1.wasm + + +
speedtest1.wasm
+ + +
+ +
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
+
This page starts running the main exe when it loads, which will + block the UI until it finishes! Adding UI controls to manually configure and start it + are TODO.
+
+
Achtung: running it with the dev tools open may + drastically slow it down. For faster results, keep the dev + tools closed when running it! +
+
Output is delayed/buffered because we cannot update the UI while the + speedtest is running. Output will appear below when ready... +
+ + + + + diff --git a/ext/wasm/split-speedtest1-script.sh b/ext/wasm/split-speedtest1-script.sh new file mode 100755 index 0000000000..e072d08a1e --- /dev/null +++ b/ext/wasm/split-speedtest1-script.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Expects $1 to be a (speedtest1 --script) output file. Output is a +# series of SQL files extracted from that file. +infile=${1:?arg = speedtest1 --script output file} +testnums=$(grep -e '^-- begin test' "$infile" | cut -d' ' -f4) +if [ x = "x${testnums}" ]; then + echo "Could not parse any begin/end blocks out of $infile" 1>&2 + exit 1 +fi +odir=${infile%%/*} +if [ "$odir" = "$infile" ]; then odir="."; fi +#echo testnums=$testnums +for n in $testnums; do + ofile=$odir/$(printf "speedtest1-%03d.sql" $n) + sed -n -e "/^-- begin test $n /,/^-- end test $n\$/p" $infile > $ofile + echo -e "$n\t$ofile" +done diff --git a/ext/wasm/sql/000-mandelbrot.sql b/ext/wasm/sql/000-mandelbrot.sql new file mode 100644 index 0000000000..3aa5f57156 --- /dev/null +++ b/ext/wasm/sql/000-mandelbrot.sql @@ -0,0 +1,17 @@ +WITH RECURSIVE + xaxis(x) AS (VALUES(-2.0) UNION ALL SELECT x+0.05 FROM xaxis WHERE x<1.2), + yaxis(y) AS (VALUES(-1.0) UNION ALL SELECT y+0.1 FROM yaxis WHERE y<1.0), + m(iter, cx, cy, x, y) AS ( + SELECT 0, x, y, 0.0, 0.0 FROM xaxis, yaxis + UNION ALL + SELECT iter+1, cx, cy, x*x-y*y + cx, 2.0*x*y + cy FROM m + WHERE (x*x + y*y) < 4.0 AND iter<28 + ), + m2(iter, cx, cy) AS ( + SELECT max(iter), cx, cy FROM m GROUP BY cx, cy + ), + a(t) AS ( + SELECT group_concat( substr(' .+*#', 1+min(iter/7,4), 1), '') + FROM m2 GROUP BY cy + ) +SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a; diff --git a/ext/wasm/sql/001-sudoku.sql b/ext/wasm/sql/001-sudoku.sql new file mode 100644 index 0000000000..53661b1c35 --- /dev/null +++ b/ext/wasm/sql/001-sudoku.sql @@ -0,0 +1,28 @@ +WITH RECURSIVE + input(sud) AS ( + VALUES('53..7....6..195....98....6.8...6...34..8.3..17...2...6.6....28....419..5....8..79') + ), + digits(z, lp) AS ( + VALUES('1', 1) + UNION ALL SELECT + CAST(lp+1 AS TEXT), lp+1 FROM digits WHERE lp<9 + ), + x(s, ind) AS ( + SELECT sud, instr(sud, '.') FROM input + UNION ALL + SELECT + substr(s, 1, ind-1) || z || substr(s, ind+1), + instr( substr(s, 1, ind-1) || z || substr(s, ind+1), '.' ) + FROM x, digits AS z + WHERE ind>0 + AND NOT EXISTS ( + SELECT 1 + FROM digits AS lp + WHERE z.z = substr(s, ((ind-1)/9)*9 + lp, 1) + OR z.z = substr(s, ((ind-1)%9) + (lp-1)*9 + 1, 1) + OR z.z = substr(s, (((ind-1)/3) % 3) * 3 + + ((ind-1)/27) * 27 + lp + + ((lp-1) / 3) * 6, 1) + ) + ) +SELECT s FROM x WHERE ind=0; diff --git a/ext/wasm/test-opfs-vfs.html b/ext/wasm/test-opfs-vfs.html new file mode 100644 index 0000000000..235ef51e9f --- /dev/null +++ b/ext/wasm/test-opfs-vfs.html @@ -0,0 +1,26 @@ + + + + + + + + + Async-behind-Sync experiment + + +
Async-behind-Sync sqlite3_vfs
+
This performs a sanity test of the "opfs" sqlite3_vfs. + See the dev console for all output. +
+
+ Use this link to delete the persistent OPFS-side db (if any). +
+
+ + + diff --git a/ext/wasm/test-opfs-vfs.js b/ext/wasm/test-opfs-vfs.js new file mode 100644 index 0000000000..bba31bb9c9 --- /dev/null +++ b/ext/wasm/test-opfs-vfs.js @@ -0,0 +1,85 @@ +/* + 2022-09-17 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A testing ground for the OPFS VFS. +*/ +'use strict'; +const tryOpfsVfs = async function(sqlite3){ + const toss = function(...args){throw new Error(args.join(' '))}; + const logPrefix = "OPFS tester:"; + const log = (...args)=>console.log(logPrefix,...args); + const warn = (...args)=>console.warn(logPrefix,...args); + const error = (...args)=>console.error(logPrefix,...args); + const opfs = sqlite3.opfs; + log("tryOpfsVfs()"); + if(!sqlite3.opfs){ + const e = toss("OPFS is not available."); + error(e); + throw e; + } + const capi = sqlite3.capi; + const pVfs = capi.sqlite3_vfs_find("opfs") || toss("Missing 'opfs' VFS."); + const oVfs = capi.sqlite3_vfs.instanceForPointer(pVfs) || toss("Unexpected instanceForPointer() result.");; + log("OPFS VFS:",pVfs, oVfs); + + const wait = async (ms)=>{ + return new Promise((resolve)=>setTimeout(resolve, ms)); + }; + + const urlArgs = new URL(self.location.href).searchParams; + const dbFile = "my-persistent.db"; + if(urlArgs.has('delete')) sqlite3.opfs.unlink(dbFile); + + const db = new opfs.OpfsDb(dbFile,'ct'); + log("db file:",db.filename); + try{ + if(opfs.entryExists(dbFile)){ + let n = db.selectValue("select count(*) from sqlite_schema"); + log("Persistent data found. sqlite_schema entry count =",n); + } + db.transaction((db)=>{ + db.exec({ + sql:[ + "create table if not exists t(a);", + "insert into t(a) values(?),(?),(?);", + ], + bind: [performance.now() | 0, + (performance.now() |0) / 2, + (performance.now() |0) / 4] + }); + }); + log("count(*) from t =",db.selectValue("select count(*) from t")); + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + await opfs.mkdir(aDir) || toss("mkdir failed"); + await opfs.mkdir(aDir) || toss("mkdir must pass if the dir exists"); + await opfs.unlink(testDir+'/test') && toss("delete 1 should have failed (dir not empty)"); + //await opfs.entryExists(testDir) + await opfs.unlink(testDir+'/test/dir') || toss("delete 2 failed"); + await opfs.unlink(testDir+'/test/dir') && toss("delete 2b should have failed (dir already deleted)"); + await opfs.unlink(testDir, true) || toss("delete 3 failed"); + await opfs.entryExists(testDir) && toss("entryExists(",testDir,") should have failed"); + }finally{ + db.close(); + } + + log("Done!"); +}/*tryOpfsVfs()*/; + +importScripts('jswasm/sqlite3.js'); +self.sqlite3InitModule() + .then((sqlite3)=>tryOpfsVfs(sqlite3)) + .catch((e)=>{ + console.error("Error initializing module:",e); + }); diff --git a/ext/wasm/tester1-worker.html b/ext/wasm/tester1-worker.html new file mode 100644 index 0000000000..4d2df0c8d1 --- /dev/null +++ b/ext/wasm/tester1-worker.html @@ -0,0 +1,63 @@ + + + + + + + + + sqlite3 tester #1 (Worker thread) + + + +

sqlite3 WASM/JS tester #1 (Worker thread)

+
See tester1.html + for the UI-thread variant.
+
+ + +
+
+ + + diff --git a/ext/wasm/tester1.html b/ext/wasm/tester1.html new file mode 100644 index 0000000000..f7a2fba4af --- /dev/null +++ b/ext/wasm/tester1.html @@ -0,0 +1,28 @@ + + + + + + + + + sqlite3 tester #1 (UI thread) + + + +

sqlite3 WASM/JS tester #1 (UI thread)

+
See tester1-worker.html + for the Worker-thread variant.
+
+ + +
+
+ + + + diff --git a/ext/wasm/tester1.js b/ext/wasm/tester1.js new file mode 100644 index 0000000000..3fe6a0ac79 --- /dev/null +++ b/ext/wasm/tester1.js @@ -0,0 +1,1852 @@ +/* + 2022-10-12 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + Main functional and regression tests for the sqlite3 WASM API. + + This mini-framework works like so: + + This script adds a series of test groups, each of which contains an + arbitrary number of tests, into a queue. After loading of the + sqlite3 WASM/JS module is complete, that queue is processed. If any + given test fails, the whole thing fails. This script is built such + that it can run from the main UI thread or worker thread. Test + groups and individual tests can be assigned a predicate function + which determines whether to run them or not, and this is + specifically intended to be used to toggle certain tests on or off + for the main/worker threads. + + Each test group defines a state object which gets applied as each + test function's `this`. Test functions can use that to, e.g., set up + a db in an early test and close it in a later test. Each test gets + passed the sqlite3 namespace object as its only argument. +*/ +'use strict'; +(function(){ + /** + Set up our output channel differently depending + on whether we are running in a worker thread or + the main (UI) thread. + */ + let logClass; + /* Predicate for tests/groups. */ + const isUIThread = ()=>(self.window===self && self.document); + /* Predicate for tests/groups. */ + const isWorker = ()=>!isUIThread(); + /* Predicate for tests/groups. */ + const testIsTodo = ()=>false; + const haveWasmCTests = ()=>{ + return !!wasm.exports.sqlite3_wasm_test_intptr; + }; + { + const mapToString = (v)=>{ + switch(typeof v){ + case 'number': case 'string': case 'boolean': + case 'undefined': case 'bigint': + return ''+v; + default: break; + } + if(null===v) return 'null'; + if(v instanceof Error){ + v = { + message: v.message, + stack: v.stack, + errorClass: v.name + }; + } + return JSON.stringify(v,undefined,2); + }; + const normalizeArgs = (args)=>args.map(mapToString); + if( isUIThread() ){ + console.log("Running in the UI thread."); + const logTarget = document.querySelector('#test-output'); + logClass = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass){ + for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){ + ln.classList.add(c); + } + } + ln.append(document.createTextNode(normalizeArgs(args).join(' '))); + logTarget.append(ln); + }; + const cbReverse = document.querySelector('#cb-log-reverse'); + const cbReverseKey = 'tester1:cb-log-reverse'; + const cbReverseIt = ()=>{ + logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); + //localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); + }; + cbReverse.addEventListener('change', cbReverseIt, true); + /*if(localStorage.getItem(cbReverseKey)){ + cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); + }*/ + cbReverseIt(); + }else{ /* Worker thread */ + console.log("Running in a Worker thread."); + logClass = function(cssClass,...args){ + postMessage({ + type:'log', + payload:{cssClass, args: normalizeArgs(args)} + }); + }; + } + } + const reportFinalTestStatus = function(pass){ + if(isUIThread()){ + const e = document.querySelector('#color-target'); + e.classList.add(pass ? 'tests-pass' : 'tests-fail'); + }else{ + postMessage({type:'test-result', payload:{pass}}); + } + }; + const log = (...args)=>{ + //console.log(...args); + logClass('',...args); + } + const warn = (...args)=>{ + console.warn(...args); + logClass('warning',...args); + } + const error = (...args)=>{ + console.error(...args); + logClass('error',...args); + }; + + const toss = (...args)=>{ + error(...args); + throw new Error(args.join(' ')); + }; + const tossQuietly = (...args)=>{ + throw new Error(args.join(' ')); + }; + + const roundMs = (ms)=>Math.round(ms*100)/100; + + /** + Helpers for writing sqlite3-specific tests. + */ + const TestUtil = { + /** Running total of the number of tests run via + this API. */ + counter: 0, + /* Separator line for log messages. */ + separator: '------------------------------------------------------------', + /** + If expr is a function, it is called and its result + is returned, coerced to a bool, else expr, coerced to + a bool, is returned. + */ + toBool: function(expr){ + return (expr instanceof Function) ? !!expr() : !!expr; + }, + /** Throws if expr is false. If expr is a function, it is called + and its result is evaluated. If passed multiple arguments, + those after the first are a message string which get applied + as an exception message if the assertion fails. The message + arguments are concatenated together with a space between each. + */ + assert: function f(expr, ...msg){ + ++this.counter; + if(!this.toBool(expr)){ + throw new Error(msg.length ? msg.join(' ') : "Assertion failed."); + } + return this; + }, + /** Calls f() and squelches any exception it throws. If it + does not throw, this function throws. */ + mustThrow: function(f, msg){ + ++this.counter; + let err; + try{ f(); } catch(e){err=e;} + if(!err) throw new Error(msg || "Expected exception."); + return this; + }, + /** + Works like mustThrow() but expects filter to be a regex, + function, or string to match/filter the resulting exception + against. If f() does not throw, this test fails and an Error is + thrown. If filter is a regex, the test passes if + filter.test(error.message) passes. If it's a function, the test + passes if filter(error) returns truthy. If it's a string, the + test passes if the filter matches the exception message + precisely. In all other cases the test fails, throwing an + Error. + + If it throws, msg is used as the error report unless it's falsy, + in which case a default is used. + */ + mustThrowMatching: function(f, filter, msg){ + ++this.counter; + let err; + try{ f(); } catch(e){err=e;} + if(!err) throw new Error(msg || "Expected exception."); + let pass = false; + if(filter instanceof RegExp) pass = filter.test(err.message); + else if(filter instanceof Function) pass = filter(err); + else if('string' === typeof filter) pass = (err.message === filter); + if(!pass){ + throw new Error(msg || ("Filter rejected this exception: "+err.message)); + } + return this; + }, + /** Throws if expr is truthy or expr is a function and expr() + returns truthy. */ + throwIf: function(expr, msg){ + ++this.counter; + if(this.toBool(expr)) throw new Error(msg || "throwIf() failed"); + return this; + }, + /** Throws if expr is falsy or expr is a function and expr() + returns falsy. */ + throwUnless: function(expr, msg){ + ++this.counter; + if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); + return this; + }, + eqApprox: (v1,v2,factor=0.05)=>(v1>=(v2-factor) && v1<=(v2+factor)), + TestGroup: (function(){ + let groupCounter = 0; + const TestGroup = function(name, predicate){ + this.number = ++groupCounter; + this.name = name; + this.predicate = predicate; + this.tests = []; + }; + TestGroup.prototype = { + addTest: function(testObj){ + this.tests.push(testObj); + return this; + }, + run: async function(sqlite3){ + log(TestUtil.separator); + logClass('group-start',"Group #"+this.number+':',this.name); + const indent = ' '; + if(this.predicate && !this.predicate(sqlite3)){ + logClass('warning',indent, + "SKIPPING group because predicate says to."); + return; + } + const assertCount = TestUtil.counter; + const groupState = Object.create(null); + const skipped = []; + let runtime = 0, i = 0; + for(const t of this.tests){ + ++i; + const n = this.number+"."+i; + log(indent, n+":", t.name); + if(t.predicate && !t.predicate(sqlite3)){ + logClass('warning', indent, indent, + 'SKIPPING because predicate says to'); + skipped.push( n+': '+t.name ); + }else{ + const tc = TestUtil.counter, now = performance.now(); + await t.test.call(groupState, sqlite3); + const then = performance.now(); + runtime += then - now; + logClass('faded',indent, indent, + TestUtil.counter - tc, 'assertion(s) in', + roundMs(then-now),'ms'); + } + } + logClass('green', + "Group #"+this.number+":",(TestUtil.counter - assertCount), + "assertion(s) in",roundMs(runtime),"ms"); + if(skipped.length){ + logClass('warning',"SKIPPED test(s) in group",this.number+":",skipped); + } + } + }; + return TestGroup; + })()/*TestGroup*/, + testGroups: [], + currentTestGroup: undefined, + addGroup: function(name, predicate){ + this.testGroups.push( this.currentTestGroup = + new this.TestGroup(name, predicate) ); + return this; + }, + addTest: function(name, callback){ + let predicate; + if(1===arguments.length){ + const opt = arguments[0]; + predicate = opt.predicate; + name = opt.name; + callback = opt.test; + } + this.currentTestGroup.addTest({ + name, predicate, test: callback + }); + return this; + }, + runTests: async function(sqlite3){ + return new Promise(async function(pok,pnok){ + try { + let runtime = 0; + for(let g of this.testGroups){ + const now = performance.now(); + await g.run(sqlite3); + runtime += performance.now() - now; + } + log(TestUtil.separator); + logClass(['strong','green'], + "Done running tests.",TestUtil.counter,"assertions in", + roundMs(runtime),'ms'); + pok(); + reportFinalTestStatus(true); + }catch(e){ + error(e); + pnok(e); + reportFinalTestStatus(false); + } + }.bind(this)); + } + }/*TestUtil*/; + const T = TestUtil; + T.g = T.addGroup; + T.t = T.addTest; + let capi, wasm/*assigned after module init*/; + //////////////////////////////////////////////////////////////////////// + // End of infrastructure setup. Now define the tests... + //////////////////////////////////////////////////////////////////////// + + //////////////////////////////////////////////////////////////////// + T.g('Basic sanity checks') + .t('Namespace object checks', function(sqlite3){ + const wasmCtypes = wasm.ctype; + T.assert(wasmCtypes.structs[0].name==='sqlite3_vfs'). + assert(wasmCtypes.structs[0].members.szOsFile.sizeof>=4). + assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ + ].members.xFileSize.offset>0); + [ /* Spot-check a handful of constants to make sure they got installed... */ + 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', + 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', + 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' + ].forEach((k)=>T.assert('number' === typeof capi[k])); + [/* Spot-check a few of the WASM API methods. */ + 'alloc', 'dealloc', 'installFunction' + ].forEach((k)=>T.assert(wasm[k] instanceof Function)); + + T.assert(capi.sqlite3_errstr(capi.SQLITE_IOERR_ACCESS).indexOf("I/O")>=0). + assert(capi.sqlite3_errstr(capi.SQLITE_CORRUPT).indexOf('malformed')>0). + assert(capi.sqlite3_errstr(capi.SQLITE_OK) === 'not an error'); + + try { + throw new sqlite3.WasmAllocError; + }catch(e){ + T.assert(e instanceof Error) + .assert(e instanceof sqlite3.WasmAllocError); + } + try{ throw new sqlite3.SQLite3Error(capi.SQLITE_SCHEMA) } + catch(e){ T.assert('SQLITE_SCHEMA' === e.message) } + try{ sqlite3.SQLite3Error.toss(capi.SQLITE_CORRUPT,{cause: true}) } + catch(e){ + T.assert('SQLITE_CORRUPT'===e.message) + .assert(true===e.cause); + } + }) + //////////////////////////////////////////////////////////////////// + .t('strglob/strlike', function(sqlite3){ + T.assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). + assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). + assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). + assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); + }) + //////////////////////////////////////////////////////////////////// + ;/*end of basic sanity checks*/ + + //////////////////////////////////////////////////////////////////// + T.g('C/WASM Utilities') + .t('sqlite3.wasm namespace', function(sqlite3){ + const w = wasm; + const chr = (x)=>x.charCodeAt(0); + //log("heap getters..."); + { + const li = [8, 16, 32]; + if(w.bigIntEnabled) li.push(64); + for(const n of li){ + const bpe = n/8; + const s = w.heapForSize(n,false); + T.assert(bpe===s.BYTES_PER_ELEMENT). + assert(w.heapForSize(s.constructor) === s); + const u = w.heapForSize(n,true); + T.assert(bpe===u.BYTES_PER_ELEMENT). + assert(s!==u). + assert(w.heapForSize(u.constructor) === u); + } + } + + // isPtr32() + { + const ip = w.isPtr32; + T.assert(ip(0)) + .assert(!ip(-1)) + .assert(!ip(1.1)) + .assert(!ip(0xffffffff)) + .assert(ip(0x7fffffff)) + .assert(!ip()) + .assert(!ip(null)/*might change: under consideration*/) + ; + } + + //log("jstrlen()..."); + { + T.assert(3 === w.jstrlen("abc")).assert(4 === w.jstrlen("äbc")); + } + + //log("jstrcpy()..."); + { + const fillChar = 10; + let ua = new Uint8Array(8), rc, + refill = ()=>ua.fill(fillChar); + refill(); + rc = w.jstrcpy("hello", ua); + T.assert(6===rc).assert(0===ua[5]).assert(chr('o')===ua[4]); + refill(); + ua[5] = chr('!'); + rc = w.jstrcpy("HELLO", ua, 0, -1, false); + T.assert(5===rc).assert(chr('!')===ua[5]).assert(chr('O')===ua[4]); + refill(); + rc = w.jstrcpy("the end", ua, 4); + //log("rc,ua",rc,ua); + T.assert(4===rc).assert(0===ua[7]). + assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); + refill(); + rc = w.jstrcpy("the end", ua, 4, -1, false); + T.assert(4===rc).assert(chr(' ')===ua[7]). + assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); + refill(); + rc = w.jstrcpy("", ua, 0, 1, true); + //log("rc,ua",rc,ua); + T.assert(1===rc).assert(0===ua[0]); + refill(); + rc = w.jstrcpy("x", ua, 0, 1, true); + //log("rc,ua",rc,ua); + T.assert(1===rc).assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 1, true); + T.assert(1===rc, 'Must not write partial multi-byte char.') + .assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 2, true); + T.assert(1===rc, 'Must not write partial multi-byte char.') + .assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 2, false); + T.assert(2===rc).assert(fillChar!==ua[1]).assert(fillChar===ua[2]); + }/*jstrcpy()*/ + + //log("cstrncpy()..."); + { + const scope = w.scopedAllocPush(); + try { + let cStr = w.scopedAllocCString("hello"); + const n = w.cstrlen(cStr); + let cpy = w.scopedAlloc(n+10); + let rc = w.cstrncpy(cpy, cStr, n+10); + T.assert(n+1 === rc). + assert("hello" === w.cstringToJs(cpy)). + assert(chr('o') === w.getMemValue(cpy+n-1)). + assert(0 === w.getMemValue(cpy+n)); + let cStr2 = w.scopedAllocCString("HI!!!"); + rc = w.cstrncpy(cpy, cStr2, 3); + T.assert(3===rc). + assert("HI!lo" === w.cstringToJs(cpy)). + assert(chr('!') === w.getMemValue(cpy+2)). + assert(chr('l') === w.getMemValue(cpy+3)); + }finally{ + w.scopedAllocPop(scope); + } + } + + //log("jstrToUintArray()..."); + { + let a = w.jstrToUintArray("hello", false); + T.assert(5===a.byteLength).assert(chr('o')===a[4]); + a = w.jstrToUintArray("hello", true); + T.assert(6===a.byteLength).assert(chr('o')===a[4]).assert(0===a[5]); + a = w.jstrToUintArray("äbä", false); + T.assert(5===a.byteLength).assert(chr('b')===a[2]); + a = w.jstrToUintArray("äbä", true); + T.assert(6===a.byteLength).assert(chr('b')===a[2]).assert(0===a[5]); + } + + //log("allocCString()..."); + { + const cstr = w.allocCString("hällo, world"); + const n = w.cstrlen(cstr); + T.assert(13 === n) + .assert(0===w.getMemValue(cstr+n)) + .assert(chr('d')===w.getMemValue(cstr+n-1)); + } + + //log("scopedAlloc() and friends..."); + { + const alloc = w.alloc, dealloc = w.dealloc; + w.alloc = w.dealloc = null; + T.assert(!w.scopedAlloc.level) + .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) + .mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); + w.alloc = alloc; + T.mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); + w.dealloc = dealloc; + T.mustThrowMatching(()=>w.scopedAllocPop(), /^Invalid state/) + .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) + .mustThrowMatching(()=>w.scopedAlloc.level=0, /read-only/); + const asc = w.scopedAllocPush(); + let asc2; + try { + const p1 = w.scopedAlloc(16), + p2 = w.scopedAlloc(16); + T.assert(1===w.scopedAlloc.level) + .assert(Number.isFinite(p1)) + .assert(Number.isFinite(p2)) + .assert(asc[0] === p1) + .assert(asc[1]===p2); + asc2 = w.scopedAllocPush(); + const p3 = w.scopedAlloc(16); + T.assert(2===w.scopedAlloc.level) + .assert(Number.isFinite(p3)) + .assert(2===asc.length) + .assert(p3===asc2[0]); + + const [z1, z2, z3] = w.scopedAllocPtr(3); + T.assert('number'===typeof z1).assert(z2>z1).assert(z3>z2) + .assert(0===w.getMemValue(z1,'i32'), 'allocPtr() must zero the targets') + .assert(0===w.getMemValue(z3,'i32')); + }finally{ + // Pop them in "incorrect" order to make sure they behave: + w.scopedAllocPop(asc); + T.assert(0===asc.length); + T.mustThrowMatching(()=>w.scopedAllocPop(asc), + /^Invalid state object/); + if(asc2){ + T.assert(2===asc2.length,'Should be p3 and z1'); + w.scopedAllocPop(asc2); + T.assert(0===asc2.length); + T.mustThrowMatching(()=>w.scopedAllocPop(asc2), + /^Invalid state object/); + } + } + T.assert(0===w.scopedAlloc.level); + w.scopedAllocCall(function(){ + T.assert(1===w.scopedAlloc.level); + const [cstr, n] = w.scopedAllocCString("hello, world", true); + T.assert(12 === n) + .assert(0===w.getMemValue(cstr+n)) + .assert(chr('d')===w.getMemValue(cstr+n-1)); + }); + }/*scopedAlloc()*/ + + //log("xCall()..."); + { + const pJson = w.xCall('sqlite3_wasm_enum_json'); + T.assert(Number.isFinite(pJson)).assert(w.cstrlen(pJson)>300); + } + + //log("xWrap()..."); + { + T.mustThrowMatching(()=>w.xWrap('sqlite3_libversion',null,'i32'), + /requires 0 arg/). + assert(w.xWrap.resultAdapter('i32') instanceof Function). + assert(w.xWrap.argAdapter('i32') instanceof Function); + let fw = w.xWrap('sqlite3_libversion','utf8'); + T.mustThrowMatching(()=>fw(1), /requires 0 arg/); + let rc = fw(); + T.assert('string'===typeof rc).assert(rc.length>5); + rc = w.xCallWrapped('sqlite3_wasm_enum_json','*'); + T.assert(rc>0 && Number.isFinite(rc)); + rc = w.xCallWrapped('sqlite3_wasm_enum_json','utf8'); + T.assert('string'===typeof rc).assert(rc.length>300); + if(haveWasmCTests()){ + fw = w.xWrap('sqlite3_wasm_test_str_hello', 'utf8:free',['i32']); + rc = fw(0); + T.assert('hello'===rc); + rc = fw(1); + T.assert(null===rc); + + if(w.bigIntEnabled){ + w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); + w.xWrap.argAdapter('twice', (v)=>2n*BigInt(v)); + fw = w.xWrap('sqlite3_wasm_test_int64_times2','thrice','twice'); + rc = fw(1); + T.assert(12n===rc); + + w.scopedAllocCall(function(){ + let pI1 = w.scopedAlloc(8), pI2 = pI1+4; + w.setMemValue(pI1, 0,'*')(pI2, 0, '*'); + let f = w.xWrap('sqlite3_wasm_test_int64_minmax',undefined,['i64*','i64*']); + let r1 = w.getMemValue(pI1, 'i64'), r2 = w.getMemValue(pI2, 'i64'); + T.assert(!Number.isSafeInteger(r1)).assert(!Number.isSafeInteger(r2)); + }); + } + } + } + }/*WhWasmUtil*/) + + //////////////////////////////////////////////////////////////////// + .t('sqlite3.StructBinder (jaccwabyt)', function(sqlite3){ + const S = sqlite3, W = S.wasm; + const MyStructDef = { + sizeof: 16, + members: { + p4: {offset: 0, sizeof: 4, signature: "i"}, + pP: {offset: 4, sizeof: 4, signature: "P"}, + ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, + cstr: {offset: 12, sizeof: 4, signature: "s"} + } + }; + if(W.bigIntEnabled){ + const m = MyStructDef; + m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; + m.sizeof += m.members.p8.sizeof; + } + const StructType = S.StructBinder.StructType; + const K = S.StructBinder('my_struct',MyStructDef); + T.mustThrowMatching(()=>K(), /via 'new'/). + mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); + const k1 = new K(), k2 = new K(); + try { + T.assert(k1.constructor === K). + assert(K.isA(k1)). + assert(k1 instanceof K). + assert(K.prototype.lookupMember('p4').key === '$p4'). + assert(K.prototype.lookupMember('$p4').name === 'p4'). + mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). + assert(undefined === K.prototype.lookupMember('nope',false)). + assert(k1 instanceof StructType). + assert(StructType.isA(k1)). + assert(K.resolveToInstance(k1.pointer)===k1). + mustThrowMatching(()=>K.resolveToInstance(null,true), /is-not-a my_struct/). + assert(k1 === StructType.instanceForPointer(k1.pointer)). + mustThrowMatching(()=>k1.$ro = 1, /read-only/); + Object.keys(MyStructDef.members).forEach(function(key){ + key = K.memberKey(key); + T.assert(0 == k1[key], + "Expecting allocation to zero the memory "+ + "for "+key+" but got: "+k1[key]+ + " from "+k1.memoryDump()); + }); + T.assert('number' === typeof k1.pointer). + mustThrowMatching(()=>k1.pointer = 1, /pointer/). + assert(K.instanceForPointer(k1.pointer) === k1); + k1.$p4 = 1; k1.$pP = 2; + T.assert(1 === k1.$p4).assert(2 === k1.$pP); + if(MyStructDef.members.$p8){ + k1.$p8 = 1/*must not throw despite not being a BigInt*/; + k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); + } + T.assert(!k1.ondispose); + k1.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(k1.ondispose)). + assert(k1.ondispose[0] === k1.$cstr). + assert('number' === typeof k1.$cstr). + assert('A C-string.' === k1.memberToJsString('cstr')); + k1.$pP = k2; + T.assert(k1.$pP === k2); + k1.$pP = null/*null is special-cased to 0.*/; + T.assert(0===k1.$pP); + let ptr = k1.pointer; + k1.dispose(); + T.assert(undefined === k1.pointer). + assert(undefined === K.instanceForPointer(ptr)). + mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); + const k3 = new K(); + ptr = k3.pointer; + T.assert(k3 === K.instanceForPointer(ptr)); + K.disposeAll(); + T.assert(ptr). + assert(undefined === k2.pointer). + assert(undefined === k3.pointer). + assert(undefined === K.instanceForPointer(ptr)); + }finally{ + k1.dispose(); + k2.dispose(); + } + + if(!W.bigIntEnabled){ + log("Skipping WasmTestStruct tests: BigInt not enabled."); + return; + } + + const WTStructDesc = + W.ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; + const autoResolvePtr = true /* EXPERIMENTAL */; + if(autoResolvePtr){ + WTStructDesc.members.ppV.signature = 'P'; + } + const WTStruct = S.StructBinder(WTStructDesc); + //log(WTStruct.structName, WTStruct.structInfo); + const wts = new WTStruct(); + //log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); + try{ + T.assert(wts.constructor === WTStruct). + assert(WTStruct.memberKeys().indexOf('$ppV')>=0). + assert(wts.memberKeys().indexOf('$v8')>=0). + assert(!K.isA(wts)). + assert(WTStruct.isA(wts)). + assert(wts instanceof WTStruct). + assert(wts instanceof StructType). + assert(StructType.isA(wts)). + assert(wts === StructType.instanceForPointer(wts.pointer)); + T.assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). + assert(0===wts.$ppV).assert(0===wts.$xFunc). + assert(WTStruct.instanceForPointer(wts.pointer) === wts); + const testFunc = + W.xGet('sqlite3_wasm_test_struct'/*name gets mangled in -O3 builds!*/); + let counter = 0; + //log("wts.pointer =",wts.pointer); + const wtsFunc = function(arg){ + /*log("This from a JS function called from C, "+ + "which itself was called from JS. arg =",arg);*/ + ++counter; + T.assert(WTStruct.instanceForPointer(arg) === wts); + if(3===counter){ + tossQuietly("Testing exception propagation."); + } + } + wts.$v4 = 10; wts.$v8 = 20; + wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) + T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) + .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) + .assert(0 === wts.$cstr) + .assert(wts.memberIsString('$cstr')) + .assert(!wts.memberIsString('$v4')) + .assert(null === wts.memberToJsString('$cstr')) + .assert(W.functionEntry(wts.$xFunc) instanceof Function); + /* It might seem silly to assert that the values match + what we just set, but recall that all of those property + reads and writes are, via property interceptors, + actually marshaling their data to/from a raw memory + buffer, so merely reading them back is actually part of + testing the struct-wrapping API. */ + + testFunc(wts.pointer); + //log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); + T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) + .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)) + .assert('string' === typeof wts.memberToJsString('cstr')) + .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) + .mustThrowMatching(()=>wts.memberToJsString('xFunc'), + /Invalid member type signature for C-string/) + ; + testFunc(wts.pointer); + T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) + .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)); + /** The 3rd call to wtsFunc throw from JS, which is called + from C, which is called from JS. Let's ensure that + that exception propagates back here... */ + T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); + W.uninstallFunction(wts.$xFunc); + wts.$xFunc = 0; + if(autoResolvePtr){ + wts.$ppV = 0; + T.assert(!wts.$ppV); + //WTStruct.debugFlags(0x03); + wts.$ppV = wts; + T.assert(wts === wts.$ppV) + //WTStruct.debugFlags(0); + } + wts.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(wts.ondispose)). + assert(wts.ondispose[0] === wts.$cstr). + assert('A C-string.' === wts.memberToJsString('cstr')); + const ptr = wts.pointer; + wts.dispose(); + T.assert(ptr).assert(undefined === wts.pointer). + assert(undefined === WTStruct.instanceForPointer(ptr)) + }finally{ + wts.dispose(); + } + }/*StructBinder*/) + + //////////////////////////////////////////////////////////////////// + .t('sqlite3.StructBinder part 2', function(sqlite3){ + // https://www.sqlite.org/c3ref/vfs.html + // https://www.sqlite.org/c3ref/io_methods.html + const sqlite3_io_methods = capi.sqlite3_io_methods, + sqlite3_vfs = capi.sqlite3_vfs, + sqlite3_file = capi.sqlite3_file; + //log("struct sqlite3_file", sqlite3_file.memberKeys()); + //log("struct sqlite3_vfs", sqlite3_vfs.memberKeys()); + //log("struct sqlite3_io_methods", sqlite3_io_methods.memberKeys()); + const installMethod = function callee(tgt, name, func){ + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.ondisposeRemoveFunc = function(){ + if(this.__ondispose){ + const who = this; + this.__ondispose.forEach( + (v)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + }else{/*wasm function wrapper property*/ + delete who[v]; + } + } + ); + delete this.__ondispose; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name), + memKey = tgt.memberKey(name); + //log("installMethod",tgt, name, sigN); + if(!tgt.__ondispose){ + T.assert(undefined === tgt.ondispose); + tgt.ondispose = [callee.ondisposeRemoveFunc]; + tgt.__ondispose = []; + } + const fProxy = callee.argcProxy(func, sigN); + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + /** + ACHTUNG: function pointer IDs are from a different pool than + allocation IDs, starting at 1 and incrementing in steps of 1, + so if we set tgt[memKey] to those values, we'd very likely + later misinterpret them as plain old pointer addresses unless + unless we use some silly heuristic like "all values <5k are + presumably function pointers," or actually perform a function + lookup on every pointer to first see if it's a function. That + would likely work just fine, but would be kludgy. + + It turns out that "all values less than X are functions" is + essentially how it works in wasm: a function pointer is + reported to the client as its index into the + __indirect_function_table. + + So... once jaccwabyt can be told how to access the + function table, it could consider all pointer values less + than that table's size to be functions. As "real" pointer + values start much, much higher than the function table size, + that would likely work reasonably well. e.g. the object + pointer address for sqlite3's default VFS is (in this local + setup) 65104, whereas the function table has fewer than 600 + entries. + */ + const wrapperKey = '$'+memKey; + tgt[wrapperKey] = fProxy; + tgt.__ondispose.push(pFunc, wrapperKey); + //log("tgt.__ondispose =",tgt.__ondispose); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + const installIOMethods = function instm(iom){ + (iom instanceof capi.sqlite3_io_methods) || toss("Invalid argument type."); + if(!instm._requireFileArg){ + instm._requireFileArg = function(arg,methodName){ + arg = capi.sqlite3_file.resolveToInstance(arg); + if(!arg){ + err("sqlite3_io_methods::xClose() was passed a non-sqlite3_file."); + } + return arg; + }; + instm._methods = { + // https://sqlite.org/c3ref/io_methods.html + xClose: /*i(P)*/function(f){ + /* int (*xClose)(sqlite3_file*) */ + log("xClose(",f,")"); + if(!(f = instm._requireFileArg(f,'xClose'))) return capi.SQLITE_MISUSE; + f.dispose(/*noting that f has externally-owned memory*/); + return 0; + }, + xRead: /*i(Ppij)*/function(f,dest,n,offset){ + /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ + log("xRead(",arguments,")"); + if(!(f = instm._requireFileArg(f))) return capi.SQLITE_MISUSE; + wasm.heap8().fill(0, dest + offset, n); + return 0; + }, + xWrite: /*i(Ppij)*/function(f,dest,n,offset){ + /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ + log("xWrite(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xWrite'))) return capi.SQLITE_MISUSE; + return 0; + }, + xTruncate: /*i(Pj)*/function(f){ + /* int (*xTruncate)(sqlite3_file*, sqlite3_int64 size) */ + log("xTruncate(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xTruncate'))) return capi.SQLITE_MISUSE; + return 0; + }, + xSync: /*i(Pi)*/function(f){ + /* int (*xSync)(sqlite3_file*, int flags) */ + log("xSync(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xSync'))) return capi.SQLITE_MISUSE; + return 0; + }, + xFileSize: /*i(Pp)*/function(f,pSz){ + /* int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize) */ + log("xFileSize(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xFileSize'))) return capi.SQLITE_MISUSE; + wasm.setMemValue(pSz, 0/*file size*/); + return 0; + }, + xLock: /*i(Pi)*/function(f){ + /* int (*xLock)(sqlite3_file*, int) */ + log("xLock(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xLock'))) return capi.SQLITE_MISUSE; + return 0; + }, + xUnlock: /*i(Pi)*/function(f){ + /* int (*xUnlock)(sqlite3_file*, int) */ + log("xUnlock(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xUnlock'))) return capi.SQLITE_MISUSE; + return 0; + }, + xCheckReservedLock: /*i(Pp)*/function(){ + /* int (*xCheckReservedLock)(sqlite3_file*, int *pResOut) */ + log("xCheckReservedLock(",arguments,")"); + return 0; + }, + xFileControl: /*i(Pip)*/function(){ + /* int (*xFileControl)(sqlite3_file*, int op, void *pArg) */ + log("xFileControl(",arguments,")"); + return capi.SQLITE_NOTFOUND; + }, + xSectorSize: /*i(P)*/function(){ + /* int (*xSectorSize)(sqlite3_file*) */ + log("xSectorSize(",arguments,")"); + return 0/*???*/; + }, + xDeviceCharacteristics:/*i(P)*/function(){ + /* int (*xDeviceCharacteristics)(sqlite3_file*) */ + log("xDeviceCharacteristics(",arguments,")"); + return 0; + } + }; + }/*static init*/ + iom.$iVersion = 1; + Object.keys(instm._methods).forEach( + (k)=>installMethod(iom, k, instm._methods[k]) + ); + }/*installIOMethods()*/; + + const iom = new sqlite3_io_methods, sfile = new sqlite3_file; + const err = console.error.bind(console); + try { + const IOM = sqlite3_io_methods, S3F = sqlite3_file; + //log("iom proto",iom,iom.constructor.prototype); + //log("sfile",sfile,sfile.constructor.prototype); + T.assert(0===sfile.$pMethods).assert(iom.pointer > 0); + //log("iom",iom); + sfile.$pMethods = iom.pointer; + T.assert(iom.pointer === sfile.$pMethods) + .assert(IOM.resolveToInstance(iom)) + .assert(undefined ===IOM.resolveToInstance(sfile)) + .mustThrow(()=>IOM.resolveToInstance(0,true)) + .assert(S3F.resolveToInstance(sfile.pointer)) + .assert(undefined===S3F.resolveToInstance(iom)) + .assert(iom===IOM.resolveToInstance(sfile.$pMethods)); + T.assert(0===iom.$iVersion); + installIOMethods(iom); + T.assert(1===iom.$iVersion); + //log("iom.__ondispose",iom.__ondispose); + T.assert(Array.isArray(iom.__ondispose)).assert(iom.__ondispose.length>10); + }finally{ + iom.dispose(); + T.assert(undefined === iom.__ondispose); + } + + const dVfs = new sqlite3_vfs(capi.sqlite3_vfs_find(null)); + try { + const SB = sqlite3.StructBinder; + T.assert(dVfs instanceof SB.StructType) + .assert(dVfs.pointer) + .assert('sqlite3_vfs' === dVfs.structName) + .assert(!!dVfs.structInfo) + .assert(SB.StructType.hasExternalPointer(dVfs)) + .assert(dVfs.$iVersion>0) + .assert('number'===typeof dVfs.$zName) + .assert('number'===typeof dVfs.$xSleep) + .assert(wasm.functionEntry(dVfs.$xOpen)) + .assert(dVfs.memberIsString('zName')) + .assert(dVfs.memberIsString('$zName')) + .assert(!dVfs.memberIsString('pAppData')) + .mustThrowMatching(()=>dVfs.memberToJsString('xSleep'), + /Invalid member type signature for C-string/) + .mustThrowMatching(()=>dVfs.memberSignature('nope'), /nope is not a mapped/) + .assert('string' === typeof dVfs.memberToJsString('zName')) + .assert(dVfs.memberToJsString('zName')===dVfs.memberToJsString('$zName')) + ; + //log("Default VFS: @",dVfs.pointer); + Object.keys(sqlite3_vfs.structInfo.members).forEach(function(mname){ + const mk = sqlite3_vfs.memberKey(mname), mbr = sqlite3_vfs.structInfo.members[mname], + addr = dVfs[mk], prefix = 'defaultVfs.'+mname; + if(1===mbr.signature.length){ + let sep = '?', val = undefined; + switch(mbr.signature[0]){ + // TODO: move this into an accessor, e.g. getPreferredValue(member) + case 'i': case 'j': case 'f': case 'd': sep = '='; val = dVfs[mk]; break + case 'p': case 'P': sep = '@'; val = dVfs[mk]; break; + case 's': sep = '='; + val = dVfs.memberToJsString(mname); + break; + } + //log(prefix, sep, val); + }else{ + //log(prefix," = funcptr @",addr, wasm.functionEntry(addr)); + } + }); + }finally{ + dVfs.dispose(); + T.assert(undefined===dVfs.pointer); + } + }/*StructBinder part 2*/) + + //////////////////////////////////////////////////////////////////// + .t('sqlite3.wasm.pstack', function(sqlite3){ + const P = wasm.pstack; + const isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; + const stack = P.pointer; + T.assert(0===stack % 8 /* must be 8-byte aligned */); + try{ + const remaining = P.remaining; + T.assert(P.quota >= 4096) + .assert(remaining === P.quota) + .mustThrowMatching(()=>P.alloc(0), isAllocErr) + .mustThrowMatching(()=>P.alloc(-1), isAllocErr); + let p1 = P.alloc(12); + T.assert(p1 === stack - 16/*8-byte aligned*/) + .assert(P.pointer === p1); + let p2 = P.alloc(7); + T.assert(p2 === p1-8/*8-byte aligned, stack grows downwards*/) + .mustThrowMatching(()=>P.alloc(remaining), isAllocErr) + .assert(24 === stack - p2) + .assert(P.pointer === p2); + let n = remaining - (stack - p2); + let p3 = P.alloc(n); + T.assert(p3 === stack-remaining) + .mustThrowMatching(()=>P.alloc(1), isAllocErr); + }finally{ + P.restore(stack); + } + + T.assert(P.pointer === stack); + try { + const [p1, p2, p3] = P.allocChunks(3,4); + T.assert(P.pointer === stack-16/*always rounded to multiple of 8*/) + .assert(p2 === p1 + 4) + .assert(p3 === p2 + 4); + T.mustThrowMatching(()=>P.allocChunks(1024, 1024 * 16), + (e)=>e instanceof sqlite3.WasmAllocError) + }finally{ + P.restore(stack); + } + + T.assert(P.pointer === stack); + try { + let [p1, p2, p3] = P.allocPtr(3,false); + let sPos = stack-16/*always rounded to multiple of 8*/; + T.assert(P.pointer === sPos) + .assert(p2 === p1 + 4) + .assert(p3 === p2 + 4); + [p1, p2, p3] = P.allocPtr(3); + T.assert(P.pointer === sPos-24/*3 x 8 bytes*/) + .assert(p2 === p1 + 8) + .assert(p3 === p2 + 8); + p1 = P.allocPtr(); + T.assert('number'===typeof p1); + }finally{ + P.restore(stack); + } + }/*pstack tests*/) + + //////////////////////////////////////////////////////////////////// + ;/*end of C/WASM utils checks*/ + + T.g('sqlite3_randomness()') + .t('To memory buffer', function(sqlite3){ + const stack = wasm.pstack.pointer; + try{ + const n = 520; + const p = wasm.pstack.alloc(n); + T.assert(0===wasm.getMemValue(p)) + .assert(0===wasm.getMemValue(p+n-1)); + T.assert(undefined === capi.sqlite3_randomness(n - 10, p)); + let j, check = 0; + const heap = wasm.heap8u(); + for(j = 0; j < 10 && 0===check; ++j){ + check += heap[p + j]; + } + T.assert(check > 0); + check = 0; + // Ensure that the trailing bytes were not modified... + for(j = n - 10; j < n && 0===check; ++j){ + check += heap[p + j]; + } + T.assert(0===check); + }finally{ + wasm.pstack.restore(stack); + } + }) + .t('To byte array', function(sqlite3){ + const ta = new Uint8Array(117); + let i, n = 0; + for(i=0; i0); + const t0 = new Uint8Array(0); + T.assert(t0 === capi.sqlite3_randomness(t0), + "0-length array is a special case"); + }) + ;/*end sqlite3_randomness() checks*/ + + //////////////////////////////////////////////////////////////////////// + T.g('sqlite3.oo1') + .t('Create db', function(sqlite3){ + const dbFile = '/tester1.db'; + wasm.sqlite3_wasm_vfs_unlink(0, dbFile); + const db = this.db = new sqlite3.oo1.DB(dbFile); + T.assert(Number.isInteger(db.pointer)) + .mustThrowMatching(()=>db.pointer=1, /read-only/) + .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) + .assert('main'===db.dbName(0)) + .assert('string' === typeof db.dbVfsName()); + // Custom db error message handling via sqlite3_prepare_v2/v3() + let rc = capi.sqlite3_prepare_v3(db.pointer, {/*invalid*/}, -1, 0, null, null); + T.assert(capi.SQLITE_MISUSE === rc) + .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) + .assert(dbFile === db.dbFilename()) + .assert(!db.dbFilename('nope')); + }) + + //////////////////////////////////////////////////////////////////// + .t('DB.Stmt', function(S){ + let st = this.db.prepare( + new TextEncoder('utf-8').encode("select 3 as a") + ); + //debug("statement =",st); + try { + T.assert(Number.isInteger(st.pointer)) + .mustThrowMatching(()=>st.pointer=1, /read-only/) + .assert(1===this.db.openStatementCount()) + .assert(!st._mayGet) + .assert('a' === st.getColumnName(0)) + .assert(1===st.columnCount) + .assert(0===st.parameterCount) + .mustThrow(()=>st.bind(1,null)) + .assert(true===st.step()) + .assert(3 === st.get(0)) + .mustThrow(()=>st.get(1)) + .mustThrow(()=>st.get(0,~capi.SQLITE_INTEGER)) + .assert(3 === st.get(0,capi.SQLITE_INTEGER)) + .assert(3 === st.getInt(0)) + .assert('3' === st.get(0,capi.SQLITE_TEXT)) + .assert('3' === st.getString(0)) + .assert(3.0 === st.get(0,capi.SQLITE_FLOAT)) + .assert(3.0 === st.getFloat(0)) + .assert(3 === st.get({}).a) + .assert(3 === st.get([])[0]) + .assert(3 === st.getJSON(0)) + .assert(st.get(0,capi.SQLITE_BLOB) instanceof Uint8Array) + .assert(1===st.get(0,capi.SQLITE_BLOB).length) + .assert(st.getBlob(0) instanceof Uint8Array) + .assert('3'.charCodeAt(0) === st.getBlob(0)[0]) + .assert(st._mayGet) + .assert(false===st.step()) + .assert(!st._mayGet) + ; + T.assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). + assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). + assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). + assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); + }finally{ + st.finalize(); + } + T.assert(!st.pointer) + .assert(0===this.db.openStatementCount()); + }) + + //////////////////////////////////////////////////////////////////////// + .t('sqlite3_js_...()', function(){ + const db = this.db; + if(1){ + const vfsList = capi.sqlite3_js_vfs_list(); + T.assert(vfsList.length>1); + T.assert('string'===typeof vfsList[0]); + //log("vfsList =",vfsList); + for(const v of vfsList){ + T.assert('string' === typeof v) + .assert(capi.sqlite3_vfs_find(v) > 0); + } + } + /** + Trivia: the magic db name ":memory:" does not actually use the + "memdb" VFS unless "memdb" is _explicitly_ provided as the VFS + name. Instead, it uses the default VFS with an in-memory btree. + Thus this.db's VFS may not be memdb even though it's an in-memory + db. + */ + const pVfsMem = capi.sqlite3_vfs_find('memdb'), + pVfsDflt = capi.sqlite3_vfs_find(0), + pVfsDb = capi.sqlite3_js_db_vfs(db.pointer); + T.assert(pVfsMem > 0) + .assert(pVfsDflt > 0) + .assert(pVfsDb > 0) + .assert(pVfsMem !== pVfsDflt + /* memdb lives on top of the default vfs */) + .assert(pVfsDb === pVfsDflt || pVfsdb === pVfsMem) + ; + /*const vMem = new capi.sqlite3_vfs(pVfsMem), + vDflt = new capi.sqlite3_vfs(pVfsDflt), + vDb = new capi.sqlite3_vfs(pVfsDb);*/ + const duv = capi.sqlite3_js_db_uses_vfs; + T.assert(pVfsDflt === duv(db.pointer, 0) + || pVfsMem === duv(db.pointer,0)) + .assert(!duv(db.pointer, "foo")) + ; + }/*sqlite3_js_...()*/) + + //////////////////////////////////////////////////////////////////// + .t('Table t', function(sqlite3){ + const db = this.db; + let list = []; + let rc = db.exec({ + sql:['CREATE TABLE t(a,b);', + // ^^^ using TEMP TABLE breaks the db export test + "INSERT INTO t(a,b) VALUES(1,2),(3,4),", + "(?,?),('blob',X'6869')"/*intentionally missing semicolon to test for + off-by-one bug in string-to-WASM conversion*/], + saveSql: list, + bind: [5,6] + }); + //debug("Exec'd SQL:", list); + T.assert(rc === db) + .assert(2 === list.length) + .assert('string'===typeof list[1]) + .assert(4===db.changes()); + if(wasm.bigIntEnabled){ + T.assert(4n===db.changes(false,true)); + } + let blob = db.selectValue("select b from t where a='blob'"); + T.assert(blob instanceof Uint8Array). + assert(0x68===blob[0] && 0x69===blob[1]); + blob = null; + let counter = 0, colNames = []; + list.length = 0; + db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ + rowMode: 'object', + resultRows: list, + columnNames: colNames, + callback: function(row,stmt){ + ++counter; + T.assert((row.a%2 && row.a<6) || 'blob'===row.a); + } + }); + T.assert(2 === colNames.length) + .assert('a' === colNames[0]) + .assert(4 === counter) + .assert(4 === list.length); + list.length = 0; + db.exec("SELECT a a, b b FROM t",{ + rowMode: 'array', + callback: function(row,stmt){ + ++counter; + T.assert(Array.isArray(row)) + .assert((0===row[1]%2 && row[1]<7) + || (row[1] instanceof Uint8Array)); + } + }); + T.assert(8 === counter); + T.assert(Number.MIN_SAFE_INTEGER === + db.selectValue("SELECT "+Number.MIN_SAFE_INTEGER)). + assert(Number.MAX_SAFE_INTEGER === + db.selectValue("SELECT "+Number.MAX_SAFE_INTEGER)); + if(wasm.bigIntEnabled && haveWasmCTests()){ + const mI = wasm.xCall('sqlite3_wasm_test_int64_max'); + const b = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(b === db.selectValue("SELECT "+b)). + assert(b === db.selectValue("SELECT ?", b)). + assert(mI == db.selectValue("SELECT $x", {$x:mI})); + }else{ + /* Curiously, the JS spec seems to be off by one with the definitions + of MIN/MAX_SAFE_INTEGER: + + https://github.com/emscripten-core/emscripten/issues/17391 */ + T.mustThrow(()=>db.selectValue("SELECT "+(Number.MAX_SAFE_INTEGER+1))). + mustThrow(()=>db.selectValue("SELECT "+(Number.MIN_SAFE_INTEGER-1))); + } + + let st = db.prepare("update t set b=:b where a='blob'"); + try { + const ndx = st.getParamIndex(':b'); + T.assert(1===ndx); + st.bindAsBlob(ndx, "ima blob").reset(true); + } finally { + st.finalize(); + } + + try { + db.prepare("/*empty SQL*/"); + toss("Must not be reached."); + }catch(e){ + T.assert(e instanceof sqlite3.SQLite3Error) + .assert(0==e.message.indexOf('Cannot prepare empty')); + } + }) + + //////////////////////////////////////////////////////////////////////// + .t('selectArray/Object()', function(sqlite3){ + const db = this.db; + let rc = db.selectArray('select a, b from t where a=?', 5); + T.assert(Array.isArray(rc)) + .assert(2===rc.length) + .assert(5===rc[0] && 6===rc[1]); + rc = db.selectArray('select a, b from t where b=-1'); + T.assert(undefined === rc); + rc = db.selectObject('select a A, b b from t where b=?', 6); + T.assert(rc && 'object'===typeof rc) + .assert(5===rc.A) + .assert(6===rc.b); + rc = db.selectArray('select a, b from t where b=-1'); + T.assert(undefined === rc); + }) + + //////////////////////////////////////////////////////////////////////// + .t('sqlite3_js_db_export()', function(){ + const db = this.db; + const xp = capi.sqlite3_js_db_export(db.pointer); + T.assert(xp instanceof Uint8Array) + .assert(xp.byteLength>0) + .assert(0 === xp.byteLength % 512); + }/*sqlite3_js_db_export()*/) + + //////////////////////////////////////////////////////////////////// + .t('Scalar UDFs', function(sqlite3){ + const db = this.db; + db.createFunction("foo",(pCx,a,b)=>a+b); + T.assert(7===db.selectValue("select foo(3,4)")). + assert(5===db.selectValue("select foo(3,?)",2)). + assert(5===db.selectValue("select foo(?,?2)",[1,4])). + assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); + db.createFunction("bar", { + arity: -1, + xFunc: (pCx,...args)=>{ + let rc = 0; + for(const v of args) rc += v; + return rc; + } + }).createFunction({ + name: "asis", + xFunc: (pCx,arg)=>arg + }); + T.assert(0===db.selectValue("select bar()")). + assert(1===db.selectValue("select bar(1)")). + assert(3===db.selectValue("select bar(1,2)")). + assert(-1===db.selectValue("select bar(1,2,-4)")). + assert('hi' === db.selectValue("select asis('hi')")). + assert('hi' === db.selectValue("select ?",'hi')). + assert(null === db.selectValue("select null")). + assert(null === db.selectValue("select asis(null)")). + assert(1 === db.selectValue("select ?",1)). + assert(2 === db.selectValue("select ?",[2])). + assert(3 === db.selectValue("select $a",{$a:3})). + assert(T.eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). + assert(T.eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))); + + let blobArg = new Uint8Array(2); + blobArg.set([0x68, 0x69], 0); + let blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + blobRc = db.selectValue("select asis(X'6869')"); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + + blobArg = new Int8Array(2); + blobArg.set([0x68, 0x69]); + //debug("blobArg=",blobArg); + blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length); + //debug("blobRc=",blobRc); + T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); + }) + + //////////////////////////////////////////////////////////////////// + .t({ + name: 'Aggregate UDFs', + test: function(sqlite3){ + const db = this.db; + const sjac = capi.sqlite3_js_aggregate_context; + db.createFunction({ + name: 'summer', + xStep: (pCtx, n)=>{ + const ac = sjac(pCtx, 4); + wasm.setMemValue(ac, wasm.getMemValue(ac,'i32') + Number(n), 'i32'); + }, + xFinal: (pCtx)=>{ + const ac = sjac(pCtx, 0); + return ac ? wasm.getMemValue(ac,'i32') : 0; + } + }); + let v = db.selectValue([ + "with cte(v) as (", + "select 3 union all select 5 union all select 7", + ") select summer(v), summer(v+1) from cte" + /* ------------------^^^^^^^^^^^ ensures that we're handling + sqlite3_aggregate_context() properly. */ + ]); + T.assert(15===v); + T.mustThrowMatching(()=>db.selectValue("select summer(1,2)"), + /wrong number of arguments/); + + db.createFunction({ + name: 'summerN', + arity: -1, + xStep: (pCtx, ...args)=>{ + const ac = sjac(pCtx, 4); + let sum = wasm.getMemValue(ac, 'i32'); + for(const v of args) sum += Number(v); + wasm.setMemValue(ac, sum, 'i32'); + }, + xFinal: (pCtx)=>{ + const ac = sjac(pCtx, 0); + capi.sqlite3_result_int( pCtx, ac ? wasm.getMemValue(ac,'i32') : 0 ); + // xFinal() may either return its value directly or call + // sqlite3_result_xyz() and return undefined. Both are + // functionally equivalent. + } + }); + T.assert(18===db.selectValue('select summerN(1,8,9), summerN(2,3,4)')); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{ + xFunc: ()=>{}, xStep: ()=>{} + }); + }, /scalar or aggregate\?/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{xStep: ()=>{}}); + }, /Missing xFinal/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{xFinal: ()=>{}}); + }, /Missing xStep/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{}); + }, /Missing function-type properties/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{xFunc:()=>{}, xDestroy:'nope'}); + }, /xDestroy property must be a function/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{xFunc:()=>{}, pApp:'nope'}); + }, /Invalid value for pApp/); + } + }/*aggregate UDFs*/) + + //////////////////////////////////////////////////////////////////////// + .t({ + name: 'Aggregate UDFs (64-bit)', + predicate: ()=>wasm.bigIntEnabled, + test: function(sqlite3){ + const db = this.db; + const sjac = capi.sqlite3_js_aggregate_context; + db.createFunction({ + name: 'summer64', + xStep: (pCtx, n)=>{ + const ac = sjac(pCtx, 8); + wasm.setMemValue(ac, wasm.getMemValue(ac,'i64') + BigInt(n), 'i64'); + }, + xFinal: (pCtx)=>{ + const ac = sjac(pCtx, 0); + return ac ? wasm.getMemValue(ac,'i64') : 0n; + } + }); + let v = db.selectValue([ + "with cte(v) as (", + "select 9007199254740991 union all select 1 union all select 2", + ") select summer64(v), summer64(v+1) from cte" + ]); + T.assert(9007199254740994n===v); + } + }/*aggregate UDFs*/) + + //////////////////////////////////////////////////////////////////// + .t({ + name: 'Window UDFs', + test: function(){ + /* Example window function, table, and results taken from: + https://sqlite.org/windowfunctions.html#udfwinfunc */ + const db = this.db; + const sjac = (cx,n=4)=>capi.sqlite3_js_aggregate_context(cx,n); + const xValueFinal = (pCtx)=>{ + const ac = sjac(pCtx, 0); + return ac ? wasm.getMemValue(ac,'i32') : 0; + }; + const xStepInverse = (pCtx, n)=>{ + const ac = sjac(pCtx); + wasm.setMemValue(ac, wasm.getMemValue(ac,'i32') + Number(n), 'i32'); + }; + db.createFunction({ + name: 'winsumint', + xStep: (pCtx, n)=>xStepInverse(pCtx, n), + xInverse: (pCtx, n)=>xStepInverse(pCtx, -n), + xFinal: xValueFinal, + xValue: xValueFinal + }); + db.exec([ + "CREATE TEMP TABLE twin(x, y); INSERT INTO twin VALUES", + "('a', 4),('b', 5),('c', 3),('d', 8),('e', 1)" + ]); + let rc = db.exec({ + returnValue: 'resultRows', + sql:[ + "SELECT x, winsumint(y) OVER (", + "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING", + ") AS sum_y ", + "FROM twin ORDER BY x;" + ] + }); + T.assert(Array.isArray(rc)) + .assert(5 === rc.length); + let count = 0; + for(const row of rc){ + switch(++count){ + case 1: T.assert('a'===row[0] && 9===row[1]); break; + case 2: T.assert('b'===row[0] && 12===row[1]); break; + case 3: T.assert('c'===row[0] && 16===row[1]); break; + case 4: T.assert('d'===row[0] && 12===row[1]); break; + case 5: T.assert('e'===row[0] && 9===row[1]); break; + default: toss("Too many rows to window function."); + } + } + const resultRows = []; + rc = db.exec({ + resultRows, + returnValue: 'resultRows', + sql:[ + "SELECT x, winsumint(y) OVER (", + "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING", + ") AS sum_y ", + "FROM twin ORDER BY x;" + ] + }); + T.assert(rc === resultRows) + .assert(5 === rc.length); + + rc = db.exec({ + returnValue: 'saveSql', + sql: "select 1; select 2; -- empty\n; select 3" + }); + T.assert(Array.isArray(rc)) + .assert(3===rc.length) + .assert('select 1;' === rc[0]) + .assert('select 2;' === rc[1]) + .assert('-- empty\n; select 3' === rc[2] + /* Strange but true. */); + + T.mustThrowMatching(()=>{ + db.exec({sql:'', returnValue: 'nope'}); + }, /^Invalid returnValue/); + + db.exec("DROP TABLE twin"); + } + }/*window UDFs*/) + + //////////////////////////////////////////////////////////////////// + .t("ATTACH", function(){ + const db = this.db; + const resultRows = []; + db.exec({ + sql:new TextEncoder('utf-8').encode([ + // ^^^ testing string-vs-typedarray handling in exec() + "attach 'session' as foo;", + "create table foo.bar(a);", + "insert into foo.bar(a) values(1),(2),(3);", + "select a from foo.bar order by a;" + ].join('')), + rowMode: 0, + resultRows + }); + T.assert(3===resultRows.length) + .assert(2===resultRows[1]); + T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a')); + let colCount = 0, rowCount = 0; + const execCallback = function(pVoid, nCols, aVals, aNames){ + colCount = nCols; + ++rowCount; + T.assert(2===aVals.length) + .assert(2===aNames.length) + .assert(+(aVals[1]) === 2 * +(aVals[0])); + }; + let rc = capi.sqlite3_exec( + db.pointer, "select a, a*2 from foo.bar", execCallback, + 0, 0 + ); + T.assert(0===rc).assert(3===rowCount).assert(2===colCount); + rc = capi.sqlite3_exec( + db.pointer, "select a from foo.bar", ()=>{ + tossQuietly("Testing throwing from exec() callback."); + }, 0, 0 + ); + T.assert(capi.SQLITE_ABORT === rc); + db.exec("detach foo"); + T.mustThrow(()=>db.exec("select * from foo.bar")); + }) + + //////////////////////////////////////////////////////////////////// + .t({ + name: 'C-side WASM tests (if compiled in)', + predicate: haveWasmCTests, + test: function(){ + const w = wasm, db = this.db; + const stack = w.scopedAllocPush(); + let ptrInt; + const origValue = 512; + const ptrValType = 'i32'; + try{ + ptrInt = w.scopedAlloc(4); + w.setMemValue(ptrInt,origValue, ptrValType); + const cf = w.xGet('sqlite3_wasm_test_intptr'); + const oldPtrInt = ptrInt; + //log('ptrInt',ptrInt); + //log('getMemValue(ptrInt)',w.getMemValue(ptrInt)); + T.assert(origValue === w.getMemValue(ptrInt, ptrValType)); + const rc = cf(ptrInt); + //log('cf(ptrInt)',rc); + //log('ptrInt',ptrInt); + //log('getMemValue(ptrInt)',w.getMemValue(ptrInt,ptrValType)); + T.assert(2*origValue === rc). + assert(rc === w.getMemValue(ptrInt,ptrValType)). + assert(oldPtrInt === ptrInt); + const pi64 = w.scopedAlloc(8)/*ptr to 64-bit integer*/; + const o64 = 0x010203040506/*>32-bit integer*/; + const ptrType64 = 'i64'; + if(w.bigIntEnabled){ + w.setMemValue(pi64, o64, ptrType64); + //log("pi64 =",pi64, "o64 = 0x",o64.toString(16), o64); + const v64 = ()=>w.getMemValue(pi64,ptrType64) + //log("getMemValue(pi64)",v64()); + T.assert(v64() == o64); + //T.assert(o64 === w.getMemValue(pi64, ptrType64)); + const cf64w = w.xGet('sqlite3_wasm_test_int64ptr'); + cf64w(pi64); + //log("getMemValue(pi64)",v64()); + T.assert(v64() == BigInt(2 * o64)); + cf64w(pi64); + T.assert(v64() == BigInt(4 * o64)); + + const biTimes2 = w.xGet('sqlite3_wasm_test_int64_times2'); + T.assert(BigInt(2 * o64) === + biTimes2(BigInt(o64)/*explicit conv. required to avoid TypeError + in the call :/ */)); + + const pMin = w.scopedAlloc(16); + const pMax = pMin + 8; + const g64 = (p)=>w.getMemValue(p,ptrType64); + w.setMemValue(pMin, 0, ptrType64); + w.setMemValue(pMax, 0, ptrType64); + const minMaxI64 = [ + w.xCall('sqlite3_wasm_test_int64_min'), + w.xCall('sqlite3_wasm_test_int64_max') + ]; + T.assert(minMaxI64[0] < BigInt(Number.MIN_SAFE_INTEGER)). + assert(minMaxI64[1] > BigInt(Number.MAX_SAFE_INTEGER)); + //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); + w.xCall('sqlite3_wasm_test_int64_minmax', pMin, pMax); + T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). + assert(g64(pMax) === minMaxI64[1], "int64 mismatch"); + //log("pMin",g64(pMin), "pMax",g64(pMax)); + w.setMemValue(pMin, minMaxI64[0], ptrType64); + T.assert(g64(pMin) === minMaxI64[0]). + assert(minMaxI64[0] === db.selectValue("select ?",g64(pMin))). + assert(minMaxI64[1] === db.selectValue("select ?",g64(pMax))); + const rxRange = /too big/; + T.mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[0] - BigInt(1))}, + rxRange). + mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[1] + BigInt(1))}, + (e)=>rxRange.test(e.message)); + }else{ + log("No BigInt support. Skipping related tests."); + log("\"The problem\" here is that we can manipulate, at the byte level,", + "heap memory to set 64-bit values, but we can't get those values", + "back into JS because of the lack of 64-bit integer support."); + } + }finally{ + const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); + //log("x=",x,"y=",y,"z=",z); // just looking at the alignment + w.scopedAllocPop(stack); + } + } + }/* jaccwabyt-specific tests */) + + .t('Close db', function(){ + T.assert(this.db).assert(Number.isInteger(this.db.pointer)); + wasm.exports.sqlite3_wasm_db_reset(this.db.pointer); + this.db.close(); + T.assert(!this.db.pointer); + }) + ;/* end of oo1 checks */ + + //////////////////////////////////////////////////////////////////////// + T.g('kvvfs') + .t('kvvfs sanity checks', function(sqlite3){ + if(isWorker()){ + T.assert( + !capi.sqlite3_vfs_find('kvvfs'), + "Expecting kvvfs to be unregistered." + ); + log("kvvfs is (correctly) unavailable in a Worker."); + return; + } + const filename = 'session'; + const pVfs = capi.sqlite3_vfs_find('kvvfs'); + T.assert(pVfs); + const JDb = sqlite3.oo1.JsStorageDb; + const unlink = ()=>JDb.clearStorage(filename); + unlink(); + let db = new JDb(filename); + try { + db.exec([ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]); + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + db.close(); + db = new JDb(filename); + db.exec('insert into kvvfs(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from kvvfs')); + }finally{ + db.close(); + unlink(); + } + }/*kvvfs sanity checks*/) + ;/* end kvvfs tests */ + + //////////////////////////////////////////////////////////////////////// + T.g('OPFS (Worker thread only and only in supported browsers)', + (sqlite3)=>{return !!sqlite3.opfs}) + .t({ + name: 'OPFS sanity checks', + test: async function(sqlite3){ + const opfs = sqlite3.opfs; + const filename = 'sqlite3-tester1.db'; + const pVfs = capi.sqlite3_vfs_find('opfs'); + T.assert(pVfs); + const unlink = (fn=filename)=>wasm.sqlite3_wasm_vfs_unlink(pVfs,fn); + unlink(); + let db = new opfs.OpfsDb(filename); + try { + db.exec([ + 'create table p(a);', + 'insert into p(a) values(1),(2),(3)' + ]); + T.assert(3 === db.selectValue('select count(*) from p')); + db.close(); + db = new opfs.OpfsDb(filename); + db.exec('insert into p(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from p')); + }finally{ + db.close(); + unlink(); + } + + if(1){ + // Sanity-test sqlite3_wasm_vfs_create_file()... + const fSize = 1379; + let sh; + try{ + T.assert(!(await opfs.entryExists(filename))); + let rc = wasm.sqlite3_wasm_vfs_create_file( + pVfs, filename, null, fSize + ); + T.assert(0===rc) + .assert(await opfs.entryExists(filename)); + const fh = await opfs.rootDirectory.getFileHandle(filename); + sh = await fh.createSyncAccessHandle(); + T.assert(fSize === await sh.getSize()); + }finally{ + if(sh) await sh.close(); + unlink(); + } + } + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } + }/*OPFS sanity checks*/) + ;/* end OPFS tests */ + + //////////////////////////////////////////////////////////////////////// + log("Loading and initializing sqlite3 WASM module..."); + if(!isUIThread()){ + /* + If sqlite3.js is in a directory other than this script, in order + to get sqlite3.js to resolve sqlite3.wasm properly, we have to + explicitly tell it where sqlite3.js is being loaded from. We do + that by passing the `sqlite3.dir=theDirName` URL argument to + _this_ script. That URL argument will be seen by the JS/WASM + loader and it will adjust the sqlite3.wasm path accordingly. If + sqlite3.js/.wasm are in the same directory as this script then + that's not needed. + + URL arguments passed as part of the filename via importScripts() + are simply lost, and such scripts see the self.location of + _this_ script. + */ + let sqlite3Js = 'sqlite3.js'; + const urlParams = new URL(self.location.href).searchParams; + if(urlParams.has('sqlite3.dir')){ + sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js; + } + importScripts(sqlite3Js); + } + self.sqlite3InitModule({ + print: log, + printErr: error + }).then(function(sqlite3){ + //console.log('sqlite3 =',sqlite3); + log("Done initializing WASM/JS bits. Running tests..."); + capi = sqlite3.capi; + wasm = sqlite3.wasm; + log("sqlite3 version:",capi.sqlite3_libversion(), + capi.sqlite3_sourceid()); + if(wasm.bigIntEnabled){ + log("BigInt/int64 support is enabled."); + }else{ + logClass('warning',"BigInt/int64 support is disabled."); + } + if(haveWasmCTests()){ + log("sqlite3_wasm_test_...() APIs are available."); + }else{ + logClass('warning',"sqlite3_wasm_test_...() APIs unavailable."); + } + TestUtil.runTests(sqlite3); + }); +})(); diff --git a/ext/wasm/testing1.js b/ext/wasm/testing1.js deleted file mode 100644 index a733156e7a..0000000000 --- a/ext/wasm/testing1.js +++ /dev/null @@ -1,1080 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - A basic test script for sqlite3-api.js. This file must be run in - main JS thread and sqlite3.js must have been loaded before it. -*/ -'use strict'; -(function(){ - const T = self.SqliteTestUtil; - const toss = function(...args){throw new Error(args.join(' '))}; - const debug = console.debug.bind(console); - const eOutput = document.querySelector('#test-output'); - const log = console.log.bind(console) - const logHtml = function(...args){ - log.apply(this, args); - const ln = document.createElement('div'); - ln.append(document.createTextNode(args.join(' '))); - eOutput.append(ln); - }; - - const eqApprox = function(v1,v2,factor=0.05){ - //debug('eqApprox',v1, v2); - return v1>=(v2-factor) && v1<=(v2+factor); - }; - - const testBasicSanity = function(db,sqlite3){ - const capi = sqlite3.capi; - log("Basic sanity tests..."); - T.assert(Number.isInteger(db.pointer)). - mustThrowMatching(()=>db.pointer=1, /read-only/). - assert(0===capi.sqlite3_extended_result_codes(db.pointer,1)). - assert('main'===db.dbName(0)); - let pId; - let st = db.prepare( - new TextEncoder('utf-8').encode("select 3 as a") - /* Testing handling of Uint8Array input */ - ); - //debug("statement =",st); - try { - T.assert(Number.isInteger(st.pointer)) - .mustThrowMatching(()=>st.pointer=1, /read-only/) - .assert(1===db.openStatementCount()) - .assert(!st._mayGet) - .assert('a' === st.getColumnName(0)) - .assert(1===st.columnCount) - .assert(0===st.parameterCount) - .mustThrow(()=>st.bind(1,null)) - .assert(true===st.step()) - .assert(3 === st.get(0)) - .mustThrow(()=>st.get(1)) - .mustThrow(()=>st.get(0,~capi.SQLITE_INTEGER)) - .assert(3 === st.get(0,capi.SQLITE_INTEGER)) - .assert(3 === st.getInt(0)) - .assert('3' === st.get(0,capi.SQLITE_TEXT)) - .assert('3' === st.getString(0)) - .assert(3.0 === st.get(0,capi.SQLITE_FLOAT)) - .assert(3.0 === st.getFloat(0)) - .assert(3 === st.get({}).a) - .assert(3 === st.get([])[0]) - .assert(3 === st.getJSON(0)) - .assert(st.get(0,capi.SQLITE_BLOB) instanceof Uint8Array) - .assert(1===st.get(0,capi.SQLITE_BLOB).length) - .assert(st.getBlob(0) instanceof Uint8Array) - .assert('3'.charCodeAt(0) === st.getBlob(0)[0]) - .assert(st._mayGet) - .assert(false===st.step()) - .assert(!st._mayGet) - ; - pId = st.pointer; - T.assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). - assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). - assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). - assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); - }finally{ - st.finalize(); - } - T.assert(!st.pointer) - .assert(0===db.openStatementCount()); - let list = []; - db.exec({ - sql:['CREATE TABLE t(a,b);', - "INSERT INTO t(a,b) VALUES(1,2),(3,4),", - "(?,?),('blob',X'6869')"/*intentionally missing semicolon to test for - off-by-one bug in string-to-WASM conversion*/], - multi: true, - saveSql: list, - bind: [5,6] - }); - //debug("Exec'd SQL:", list); - T.assert(2 === list.length) - .assert('string'===typeof list[1]) - .assert(4===db.changes()); - if(capi.wasm.bigIntEnabled){ - T.assert(4n===db.changes(false,true)); - } - let blob = db.selectValue("select b from t where a='blob'"); - T.assert(blob instanceof Uint8Array). - assert(0x68===blob[0] && 0x69===blob[1]); - blob = null; - - let counter = 0, colNames = []; - list.length = 0; - db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ - rowMode: 'object', - resultRows: list, - columnNames: colNames, - callback: function(row,stmt){ - ++counter; - T.assert((row.a%2 && row.a<6) || 'blob'===row.a); - } - }); - T.assert(2 === colNames.length) - .assert('a' === colNames[0]) - .assert(4 === counter) - .assert(4 === list.length); - list.length = 0; - db.exec("SELECT a a, b b FROM t",{ - rowMode: 'array', - callback: function(row,stmt){ - ++counter; - T.assert(Array.isArray(row)) - .assert((0===row[1]%2 && row[1]<7) - || (row[1] instanceof Uint8Array)); - } - }); - T.assert(8 === counter); - T.assert(Number.MIN_SAFE_INTEGER === - db.selectValue("SELECT "+Number.MIN_SAFE_INTEGER)). - assert(Number.MAX_SAFE_INTEGER === - db.selectValue("SELECT "+Number.MAX_SAFE_INTEGER)); - if(capi.wasm.bigIntEnabled){ - const mI = capi.wasm.xCall('jaccwabyt_test_int64_max'); - const b = BigInt(Number.MAX_SAFE_INTEGER * 2); - T.assert(b === db.selectValue("SELECT "+b)). - assert(b === db.selectValue("SELECT ?", b)). - assert(mI == db.selectValue("SELECT $x", {$x:mI})); - }else{ - /* Curiously, the JS spec seems to be off by one with the definitions - of MIN/MAX_SAFE_INTEGER: - - https://github.com/emscripten-core/emscripten/issues/17391 */ - T.mustThrow(()=>db.selectValue("SELECT "+(Number.MAX_SAFE_INTEGER+1))). - mustThrow(()=>db.selectValue("SELECT "+(Number.MIN_SAFE_INTEGER-1))); - } - - st = db.prepare("update t set b=:b where a='blob'"); - try { - const ndx = st.getParamIndex(':b'); - T.assert(1===ndx); - st.bindAsBlob(ndx, "ima blob").reset(true); - } finally { - st.finalize(); - } - - try { - throw new capi.WasmAllocError; - }catch(e){ - T.assert(e instanceof Error) - .assert(e instanceof capi.WasmAllocError); - } - - try { - db.prepare("/*empty SQL*/"); - toss("Must not be reached."); - }catch(e){ - T.assert(e instanceof sqlite3.SQLite3Error) - .assert(0==e.message.indexOf('Cannot prepare empty')); - } - - T.assert(capi.sqlite3_errstr(capi.SQLITE_IOERR_ACCESS).indexOf("I/O")>=0). - assert(capi.sqlite3_errstr(capi.SQLITE_CORRUPT).indexOf('malformed')>0). - assert(capi.sqlite3_errstr(capi.SQLITE_OK) === 'not an error'); - - // Custom db error message handling via sqlite3_prepare_v2/v3() - if(capi.wasm.exports.sqlite3_wasm_db_error){ - log("Testing custom error message via prepare_v3()..."); - let rc = capi.sqlite3_prepare_v3(db.pointer, [/*invalid*/], -1, 0, null, null); - T.assert(capi.SQLITE_MISUSE === rc) - .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")); - log("errmsg =",capi.sqlite3_errmsg(db.pointer)); - } - }/*testBasicSanity()*/; - - const testUDF = function(db){ - db.createFunction("foo",function(a,b){return a+b}); - T.assert(7===db.selectValue("select foo(3,4)")). - assert(5===db.selectValue("select foo(3,?)",2)). - assert(5===db.selectValue("select foo(?,?2)",[1,4])). - assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); - db.createFunction("bar", { - arity: -1, - callback: function(){ - var rc = 0; - for(let i = 0; i < arguments.length; ++i) rc += arguments[i]; - return rc; - } - }).createFunction({ - name: "asis", - callback: (arg)=>arg - }); - - //log("Testing DB::selectValue() w/ UDF..."); - T.assert(0===db.selectValue("select bar()")). - assert(1===db.selectValue("select bar(1)")). - assert(3===db.selectValue("select bar(1,2)")). - assert(-1===db.selectValue("select bar(1,2,-4)")). - assert('hi'===db.selectValue("select asis('hi')")); - - T.assert('hi' === db.selectValue("select ?",'hi')). - assert(null===db.selectValue("select null")). - assert(null === db.selectValue("select ?",null)). - assert(null === db.selectValue("select ?",[null])). - assert(null === db.selectValue("select $a",{$a:null})). - assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). - assert(eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))) - ; - - //log("Testing binding and UDF propagation of blobs..."); - let blobArg = new Uint8Array(2); - blobArg.set([0x68, 0x69], 0); - let blobRc = db.selectValue("select asis(?1)", blobArg); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length). - assert(0x68==blobRc[0] && 0x69==blobRc[1]); - blobRc = db.selectValue("select asis(X'6869')"); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length). - assert(0x68==blobRc[0] && 0x69==blobRc[1]); - - blobArg = new Int8Array(2); - blobArg.set([0x68, 0x69]); - //debug("blobArg=",blobArg); - blobRc = db.selectValue("select asis(?1)", blobArg); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length); - //debug("blobRc=",blobRc); - T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); - }; - - const testAttach = function(db){ - const resultRows = []; - db.exec({ - sql:new TextEncoder('utf-8').encode([ - // ^^^ testing string-vs-typedarray handling in execMulti() - "attach 'foo.db' as foo;", - "create table foo.bar(a);", - "insert into foo.bar(a) values(1),(2),(3);", - "select a from foo.bar order by a;" - ].join('')), - multi: true, - rowMode: 0, - resultRows - }); - T.assert(3===resultRows.length) - .assert(2===resultRows[1]); - T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a')); - db.exec("detach foo"); - T.mustThrow(()=>db.exec("select * from foo.bar")); - }; - - const testIntPtr = function(db,S,Module){ - const w = S.capi.wasm; - const stack = w.scopedAllocPush(); - let ptrInt; - const origValue = 512; - const ptrValType = 'i32'; - try{ - ptrInt = w.scopedAlloc(4); - w.setMemValue(ptrInt,origValue, ptrValType); - const cf = w.xGet('jaccwabyt_test_intptr'); - const oldPtrInt = ptrInt; - //log('ptrInt',ptrInt); - //log('getMemValue(ptrInt)',w.getMemValue(ptrInt)); - T.assert(origValue === w.getMemValue(ptrInt, ptrValType)); - const rc = cf(ptrInt); - //log('cf(ptrInt)',rc); - //log('ptrInt',ptrInt); - //log('getMemValue(ptrInt)',w.getMemValue(ptrInt,ptrValType)); - T.assert(2*origValue === rc). - assert(rc === w.getMemValue(ptrInt,ptrValType)). - assert(oldPtrInt === ptrInt); - const pi64 = w.scopedAlloc(8)/*ptr to 64-bit integer*/; - const o64 = 0x010203040506/*>32-bit integer*/; - const ptrType64 = 'i64'; - if(w.bigIntEnabled){ - log("BigInt support is enabled..."); - w.setMemValue(pi64, o64, ptrType64); - //log("pi64 =",pi64, "o64 = 0x",o64.toString(16), o64); - const v64 = ()=>w.getMemValue(pi64,ptrType64) - //log("getMemValue(pi64)",v64()); - T.assert(v64() == o64); - //T.assert(o64 === w.getMemValue(pi64, ptrType64)); - const cf64w = w.xGet('jaccwabyt_test_int64ptr'); - cf64w(pi64); - //log("getMemValue(pi64)",v64()); - T.assert(v64() == BigInt(2 * o64)); - cf64w(pi64); - T.assert(v64() == BigInt(4 * o64)); - - const biTimes2 = w.xGet('jaccwabyt_test_int64_times2'); - T.assert(BigInt(2 * o64) === - biTimes2(BigInt(o64)/*explicit conv. required to avoid TypeError - in the call :/ */)); - - const pMin = w.scopedAlloc(16); - const pMax = pMin + 8; - const g64 = (p)=>w.getMemValue(p,ptrType64); - w.setMemValue(pMin, 0, ptrType64); - w.setMemValue(pMax, 0, ptrType64); - const minMaxI64 = [ - w.xCall('jaccwabyt_test_int64_min'), - w.xCall('jaccwabyt_test_int64_max') - ]; - T.assert(minMaxI64[0] < BigInt(Number.MIN_SAFE_INTEGER)). - assert(minMaxI64[1] > BigInt(Number.MAX_SAFE_INTEGER)); - //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); - w.xCall('jaccwabyt_test_int64_minmax', pMin, pMax); - T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). - assert(g64(pMax) === minMaxI64[1], "int64 mismatch"); - //log("pMin",g64(pMin), "pMax",g64(pMax)); - w.setMemValue(pMin, minMaxI64[0], ptrType64); - T.assert(g64(pMin) === minMaxI64[0]). - assert(minMaxI64[0] === db.selectValue("select ?",g64(pMin))). - assert(minMaxI64[1] === db.selectValue("select ?",g64(pMax))); - const rxRange = /out of range for int64/; - T.mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[0] - BigInt(1))}, - rxRange). - mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[1] + BigInt(1))}, - (e)=>rxRange.test(e.message)); - }else{ - log("No BigInt support. Skipping related tests."); - log("\"The problem\" here is that we can manipulate, at the byte level,", - "heap memory to set 64-bit values, but we can't get those values", - "back into JS because of the lack of 64-bit integer support."); - } - }finally{ - const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); - //log("x=",x,"y=",y,"z=",z); // just looking at the alignment - w.scopedAllocPop(stack); - } - }/*testIntPtr()*/; - - const testStructStuff = function(db,S,M){ - const W = S.capi.wasm, C = S; - /** Maintenance reminder: the rest of this function is copy/pasted - from the upstream jaccwabyt tests. */ - log("Jaccwabyt tests..."); - const MyStructDef = { - sizeof: 16, - members: { - p4: {offset: 0, sizeof: 4, signature: "i"}, - pP: {offset: 4, sizeof: 4, signature: "P"}, - ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, - cstr: {offset: 12, sizeof: 4, signature: "s"} - } - }; - if(W.bigIntEnabled){ - const m = MyStructDef; - m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; - m.sizeof += m.members.p8.sizeof; - } - const StructType = C.StructBinder.StructType; - const K = C.StructBinder('my_struct',MyStructDef); - T.mustThrowMatching(()=>K(), /via 'new'/). - mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); - const k1 = new K(), k2 = new K(); - try { - T.assert(k1.constructor === K). - assert(K.isA(k1)). - assert(k1 instanceof K). - assert(K.prototype.lookupMember('p4').key === '$p4'). - assert(K.prototype.lookupMember('$p4').name === 'p4'). - mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). - assert(undefined === K.prototype.lookupMember('nope',false)). - assert(k1 instanceof StructType). - assert(StructType.isA(k1)). - assert(K.resolveToInstance(k1.pointer)===k1). - mustThrowMatching(()=>K.resolveToInstance(null,true), /is-not-a my_struct/). - assert(k1 === StructType.instanceForPointer(k1.pointer)). - mustThrowMatching(()=>k1.$ro = 1, /read-only/); - Object.keys(MyStructDef.members).forEach(function(key){ - key = K.memberKey(key); - T.assert(0 == k1[key], - "Expecting allocation to zero the memory "+ - "for "+key+" but got: "+k1[key]+ - " from "+k1.memoryDump()); - }); - T.assert('number' === typeof k1.pointer). - mustThrowMatching(()=>k1.pointer = 1, /pointer/). - assert(K.instanceForPointer(k1.pointer) === k1); - k1.$p4 = 1; k1.$pP = 2; - T.assert(1 === k1.$p4).assert(2 === k1.$pP); - if(MyStructDef.members.$p8){ - k1.$p8 = 1/*must not throw despite not being a BigInt*/; - k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); - T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); - } - T.assert(!k1.ondispose); - k1.setMemberCString('cstr', "A C-string."); - T.assert(Array.isArray(k1.ondispose)). - assert(k1.ondispose[0] === k1.$cstr). - assert('number' === typeof k1.$cstr). - assert('A C-string.' === k1.memberToJsString('cstr')); - k1.$pP = k2; - T.assert(k1.$pP === k2); - k1.$pP = null/*null is special-cased to 0.*/; - T.assert(0===k1.$pP); - let ptr = k1.pointer; - k1.dispose(); - T.assert(undefined === k1.pointer). - assert(undefined === K.instanceForPointer(ptr)). - mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); - const k3 = new K(); - ptr = k3.pointer; - T.assert(k3 === K.instanceForPointer(ptr)); - K.disposeAll(); - T.assert(ptr). - assert(undefined === k2.pointer). - assert(undefined === k3.pointer). - assert(undefined === K.instanceForPointer(ptr)); - }finally{ - k1.dispose(); - k2.dispose(); - } - - if(!W.bigIntEnabled){ - log("Skipping WasmTestStruct tests: BigInt not enabled."); - return; - } - - const ctype = W.xCallWrapped('jaccwabyt_test_ctype_json', 'json'); - log("Struct descriptions:",ctype.structs); - const WTStructDesc = - ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; - const autoResolvePtr = true /* EXPERIMENTAL */; - if(autoResolvePtr){ - WTStructDesc.members.ppV.signature = 'P'; - } - const WTStruct = C.StructBinder(WTStructDesc); - log(WTStruct.structName, WTStruct.structInfo); - const wts = new WTStruct(); - log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); - try{ - T.assert(wts.constructor === WTStruct). - assert(WTStruct.memberKeys().indexOf('$ppV')>=0). - assert(wts.memberKeys().indexOf('$v8')>=0). - assert(!K.isA(wts)). - assert(WTStruct.isA(wts)). - assert(wts instanceof WTStruct). - assert(wts instanceof StructType). - assert(StructType.isA(wts)). - assert(wts === StructType.instanceForPointer(wts.pointer)); - T.assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). - assert(0===wts.$ppV).assert(0===wts.$xFunc). - assert(WTStruct.instanceForPointer(wts.pointer) === wts); - const testFunc = - W.xGet('jaccwabyt_test_struct'/*name gets mangled in -O3 builds!*/); - let counter = 0; - log("wts.pointer =",wts.pointer); - const wtsFunc = function(arg){ - log("This from a JS function called from C, "+ - "which itself was called from JS. arg =",arg); - ++counter; - T.assert(WTStruct.instanceForPointer(arg) === wts); - if(3===counter){ - toss("Testing exception propagation."); - } - } - wts.$v4 = 10; wts.$v8 = 20; - wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) - /* ^^^ compiles wtsFunc to WASM and returns its new function pointer */; - T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) - .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) - .assert(0 === wts.$cstr) - .assert(wts.memberIsString('$cstr')) - .assert(!wts.memberIsString('$v4')) - .assert(null === wts.memberToJsString('$cstr')) - .assert(W.functionEntry(wts.$xFunc) instanceof Function); - /* It might seem silly to assert that the values match - what we just set, but recall that all of those property - reads and writes are, via property interceptors, - actually marshaling their data to/from a raw memory - buffer, so merely reading them back is actually part of - testing the struct-wrapping API. */ - - testFunc(wts.pointer); - log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); - T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) - .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)) - .assert('string' === typeof wts.memberToJsString('cstr')) - .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) - .mustThrowMatching(()=>wts.memberToJsString('xFunc'), - /Invalid member type signature for C-string/) - ; - testFunc(wts.pointer); - T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) - .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)); - /** The 3rd call to wtsFunc throw from JS, which is called - from C, which is called from JS. Let's ensure that - that exception propagates back here... */ - T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); - W.uninstallFunction(wts.$xFunc); - wts.$xFunc = 0; - if(autoResolvePtr){ - wts.$ppV = 0; - T.assert(!wts.$ppV); - WTStruct.debugFlags(0x03); - wts.$ppV = wts; - T.assert(wts === wts.$ppV) - WTStruct.debugFlags(0); - } - wts.setMemberCString('cstr', "A C-string."); - T.assert(Array.isArray(wts.ondispose)). - assert(wts.ondispose[0] === wts.$cstr). - assert('A C-string.' === wts.memberToJsString('cstr')); - const ptr = wts.pointer; - wts.dispose(); - T.assert(ptr).assert(undefined === wts.pointer). - assert(undefined === WTStruct.instanceForPointer(ptr)) - }finally{ - wts.dispose(); - } - }/*testStructStuff()*/; - - const testSqliteStructs = function(db,sqlite3,M){ - log("Tinkering with sqlite3_io_methods..."); - // https://www.sqlite.org/c3ref/vfs.html - // https://www.sqlite.org/c3ref/io_methods.html - const capi = sqlite3.capi, W = capi.wasm; - const sqlite3_io_methods = capi.sqlite3_io_methods, - sqlite3_vfs = capi.sqlite3_vfs, - sqlite3_file = capi.sqlite3_file; - log("struct sqlite3_file", sqlite3_file.memberKeys()); - log("struct sqlite3_vfs", sqlite3_vfs.memberKeys()); - log("struct sqlite3_io_methods", sqlite3_io_methods.memberKeys()); - - const installMethod = function callee(tgt, name, func){ - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - callee.ondisposeRemoveFunc = function(){ - if(this.__ondispose){ - const who = this; - this.__ondispose.forEach( - (v)=>{ - if('number'===typeof v){ - try{capi.wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - }else{/*wasm function wrapper property*/ - delete who[v]; - } - } - ); - delete this.__ondispose; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name), - memKey = tgt.memberKey(name); - //log("installMethod",tgt, name, sigN); - if(!tgt.__ondispose){ - T.assert(undefined === tgt.ondispose); - tgt.ondispose = [callee.ondisposeRemoveFunc]; - tgt.__ondispose = []; - } - const fProxy = callee.argcProxy(func, sigN); - const pFunc = capi.wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - /** - ACHTUNG: function pointer IDs are from a different pool than - allocation IDs, starting at 1 and incrementing in steps of 1, - so if we set tgt[memKey] to those values, we'd very likely - later misinterpret them as plain old pointer addresses unless - unless we use some silly heuristic like "all values <5k are - presumably function pointers," or actually perform a function - lookup on every pointer to first see if it's a function. That - would likely work just fine, but would be kludgy. - - It turns out that "all values less than X are functions" is - essentially how it works in wasm: a function pointer is - reported to the client as its index into the - __indirect_function_table. - - So... once jaccwabyt can be told how to access the - function table, it could consider all pointer values less - than that table's size to be functions. As "real" pointer - values start much, much higher than the function table size, - that would likely work reasonably well. e.g. the object - pointer address for sqlite3's default VFS is (in this local - setup) 65104, whereas the function table has fewer than 600 - entries. - */ - const wrapperKey = '$'+memKey; - tgt[wrapperKey] = fProxy; - tgt.__ondispose.push(pFunc, wrapperKey); - //log("tgt.__ondispose =",tgt.__ondispose); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; - - const installIOMethods = function instm(iom){ - (iom instanceof capi.sqlite3_io_methods) || toss("Invalid argument type."); - if(!instm._requireFileArg){ - instm._requireFileArg = function(arg,methodName){ - arg = capi.sqlite3_file.resolveToInstance(arg); - if(!arg){ - err("sqlite3_io_methods::xClose() was passed a non-sqlite3_file."); - } - return arg; - }; - instm._methods = { - // https://sqlite.org/c3ref/io_methods.html - xClose: /*i(P)*/function(f){ - /* int (*xClose)(sqlite3_file*) */ - log("xClose(",f,")"); - if(!(f = instm._requireFileArg(f,'xClose'))) return capi.SQLITE_MISUSE; - f.dispose(/*noting that f has externally-owned memory*/); - return 0; - }, - xRead: /*i(Ppij)*/function(f,dest,n,offset){ - /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ - log("xRead(",arguments,")"); - if(!(f = instm._requireFileArg(f))) return capi.SQLITE_MISUSE; - capi.wasm.heap8().fill(0, dest + offset, n); - return 0; - }, - xWrite: /*i(Ppij)*/function(f,dest,n,offset){ - /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ - log("xWrite(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xWrite'))) return capi.SQLITE_MISUSE; - return 0; - }, - xTruncate: /*i(Pj)*/function(f){ - /* int (*xTruncate)(sqlite3_file*, sqlite3_int64 size) */ - log("xTruncate(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xTruncate'))) return capi.SQLITE_MISUSE; - return 0; - }, - xSync: /*i(Pi)*/function(f){ - /* int (*xSync)(sqlite3_file*, int flags) */ - log("xSync(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xSync'))) return capi.SQLITE_MISUSE; - return 0; - }, - xFileSize: /*i(Pp)*/function(f,pSz){ - /* int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize) */ - log("xFileSize(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xFileSize'))) return capi.SQLITE_MISUSE; - capi.wasm.setMemValue(pSz, 0/*file size*/); - return 0; - }, - xLock: /*i(Pi)*/function(f){ - /* int (*xLock)(sqlite3_file*, int) */ - log("xLock(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xLock'))) return capi.SQLITE_MISUSE; - return 0; - }, - xUnlock: /*i(Pi)*/function(f){ - /* int (*xUnlock)(sqlite3_file*, int) */ - log("xUnlock(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xUnlock'))) return capi.SQLITE_MISUSE; - return 0; - }, - xCheckReservedLock: /*i(Pp)*/function(){ - /* int (*xCheckReservedLock)(sqlite3_file*, int *pResOut) */ - log("xCheckReservedLock(",arguments,")"); - return 0; - }, - xFileControl: /*i(Pip)*/function(){ - /* int (*xFileControl)(sqlite3_file*, int op, void *pArg) */ - log("xFileControl(",arguments,")"); - return capi.SQLITE_NOTFOUND; - }, - xSectorSize: /*i(P)*/function(){ - /* int (*xSectorSize)(sqlite3_file*) */ - log("xSectorSize(",arguments,")"); - return 0/*???*/; - }, - xDeviceCharacteristics:/*i(P)*/function(){ - /* int (*xDeviceCharacteristics)(sqlite3_file*) */ - log("xDeviceCharacteristics(",arguments,")"); - return 0; - } - }; - }/*static init*/ - iom.$iVersion = 1; - Object.keys(instm._methods).forEach( - (k)=>installMethod(iom, k, instm._methods[k]) - ); - }/*installIOMethods()*/; - - const iom = new sqlite3_io_methods, sfile = new sqlite3_file; - const err = console.error.bind(console); - try { - const IOM = sqlite3_io_methods, S3F = sqlite3_file; - //log("iom proto",iom,iom.constructor.prototype); - //log("sfile",sfile,sfile.constructor.prototype); - T.assert(0===sfile.$pMethods).assert(iom.pointer > 0); - //log("iom",iom); - /** Some of the following tests require that pMethods has a - signature of "P", as opposed to "p". */ - sfile.$pMethods = iom; - T.assert(iom === sfile.$pMethods); - sfile.$pMethods = iom.pointer; - T.assert(iom === sfile.$pMethods) - .assert(IOM.resolveToInstance(iom)) - .assert(undefined ===IOM.resolveToInstance(sfile)) - .mustThrow(()=>IOM.resolveToInstance(0,true)) - .assert(S3F.resolveToInstance(sfile.pointer)) - .assert(undefined===S3F.resolveToInstance(iom)); - T.assert(0===iom.$iVersion); - installIOMethods(iom); - T.assert(1===iom.$iVersion); - //log("iom.__ondispose",iom.__ondispose); - T.assert(Array.isArray(iom.__ondispose)).assert(iom.__ondispose.length>10); - }finally{ - iom.dispose(); - T.assert(undefined === iom.__ondispose); - } - - const dVfs = new sqlite3_vfs(capi.sqlite3_vfs_find(null)); - try { - const SB = sqlite3.StructBinder; - T.assert(dVfs instanceof SB.StructType) - .assert(dVfs.pointer) - .assert('sqlite3_vfs' === dVfs.structName) - .assert(!!dVfs.structInfo) - .assert(SB.StructType.hasExternalPointer(dVfs)) - .assert(3===dVfs.$iVersion) - .assert('number'===typeof dVfs.$zName) - .assert('number'===typeof dVfs.$xSleep) - .assert(capi.wasm.functionEntry(dVfs.$xOpen)) - .assert(dVfs.memberIsString('zName')) - .assert(dVfs.memberIsString('$zName')) - .assert(!dVfs.memberIsString('pAppData')) - .mustThrowMatching(()=>dVfs.memberToJsString('xSleep'), - /Invalid member type signature for C-string/) - .mustThrowMatching(()=>dVfs.memberSignature('nope'), /nope is not a mapped/) - .assert('string' === typeof dVfs.memberToJsString('zName')) - .assert(dVfs.memberToJsString('zName')===dVfs.memberToJsString('$zName')) - ; - log("Default VFS: @",dVfs.pointer); - Object.keys(sqlite3_vfs.structInfo.members).forEach(function(mname){ - const mk = sqlite3_vfs.memberKey(mname), mbr = sqlite3_vfs.structInfo.members[mname], - addr = dVfs[mk], prefix = 'defaultVfs.'+mname; - if(1===mbr.signature.length){ - let sep = '?', val = undefined; - switch(mbr.signature[0]){ - // TODO: move this into an accessor, e.g. getPreferredValue(member) - case 'i': case 'j': case 'f': case 'd': sep = '='; val = dVfs[mk]; break - case 'p': case 'P': sep = '@'; val = dVfs[mk]; break; - case 's': sep = '='; - //val = capi.wasm.UTF8ToString(addr); - val = dVfs.memberToJsString(mname); - break; - } - log(prefix, sep, val); - } - else{ - log(prefix," = funcptr @",addr, capi.wasm.functionEntry(addr)); - } - }); - }finally{ - dVfs.dispose(); - T.assert(undefined===dVfs.pointer); - } - }/*testSqliteStructs()*/; - - const testWasmUtil = function(DB,S){ - const w = S.capi.wasm; - /** - Maintenance reminder: the rest of this function is part of the - upstream Jaccwabyt tree. - */ - const chr = (x)=>x.charCodeAt(0); - log("heap getters..."); - { - const li = [8, 16, 32]; - if(w.bigIntEnabled) li.push(64); - for(const n of li){ - const bpe = n/8; - const s = w.heapForSize(n,false); - T.assert(bpe===s.BYTES_PER_ELEMENT). - assert(w.heapForSize(s.constructor) === s); - const u = w.heapForSize(n,true); - T.assert(bpe===u.BYTES_PER_ELEMENT). - assert(s!==u). - assert(w.heapForSize(u.constructor) === u); - } - } - - log("jstrlen()..."); - { - T.assert(3 === w.jstrlen("abc")).assert(4 === w.jstrlen("äbc")); - } - - log("jstrcpy()..."); - { - const fillChar = 10; - let ua = new Uint8Array(8), rc, - refill = ()=>ua.fill(fillChar); - refill(); - rc = w.jstrcpy("hello", ua); - T.assert(6===rc).assert(0===ua[5]).assert(chr('o')===ua[4]); - refill(); - ua[5] = chr('!'); - rc = w.jstrcpy("HELLO", ua, 0, -1, false); - T.assert(5===rc).assert(chr('!')===ua[5]).assert(chr('O')===ua[4]); - refill(); - rc = w.jstrcpy("the end", ua, 4); - //log("rc,ua",rc,ua); - T.assert(4===rc).assert(0===ua[7]). - assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); - refill(); - rc = w.jstrcpy("the end", ua, 4, -1, false); - T.assert(4===rc).assert(chr(' ')===ua[7]). - assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); - refill(); - rc = w.jstrcpy("", ua, 0, 1, true); - //log("rc,ua",rc,ua); - T.assert(1===rc).assert(0===ua[0]); - refill(); - rc = w.jstrcpy("x", ua, 0, 1, true); - //log("rc,ua",rc,ua); - T.assert(1===rc).assert(0===ua[0]); - refill(); - rc = w.jstrcpy('äbä', ua, 0, 1, true); - T.assert(1===rc, 'Must not write partial multi-byte char.') - .assert(0===ua[0]); - refill(); - rc = w.jstrcpy('äbä', ua, 0, 2, true); - T.assert(1===rc, 'Must not write partial multi-byte char.') - .assert(0===ua[0]); - refill(); - rc = w.jstrcpy('äbä', ua, 0, 2, false); - T.assert(2===rc).assert(fillChar!==ua[1]).assert(fillChar===ua[2]); - }/*jstrcpy()*/ - - log("cstrncpy()..."); - { - w.scopedAllocPush(); - try { - let cStr = w.scopedAllocCString("hello"); - const n = w.cstrlen(cStr); - let cpy = w.scopedAlloc(n+10); - let rc = w.cstrncpy(cpy, cStr, n+10); - T.assert(n+1 === rc). - assert("hello" === w.cstringToJs(cpy)). - assert(chr('o') === w.getMemValue(cpy+n-1)). - assert(0 === w.getMemValue(cpy+n)); - let cStr2 = w.scopedAllocCString("HI!!!"); - rc = w.cstrncpy(cpy, cStr2, 3); - T.assert(3===rc). - assert("HI!lo" === w.cstringToJs(cpy)). - assert(chr('!') === w.getMemValue(cpy+2)). - assert(chr('l') === w.getMemValue(cpy+3)); - }finally{ - w.scopedAllocPop(); - } - } - - log("jstrToUintArray()..."); - { - let a = w.jstrToUintArray("hello", false); - T.assert(5===a.byteLength).assert(chr('o')===a[4]); - a = w.jstrToUintArray("hello", true); - T.assert(6===a.byteLength).assert(chr('o')===a[4]).assert(0===a[5]); - a = w.jstrToUintArray("äbä", false); - T.assert(5===a.byteLength).assert(chr('b')===a[2]); - a = w.jstrToUintArray("äbä", true); - T.assert(6===a.byteLength).assert(chr('b')===a[2]).assert(0===a[5]); - } - - log("allocCString()..."); - { - const cstr = w.allocCString("hällo, world"); - const n = w.cstrlen(cstr); - T.assert(13 === n) - .assert(0===w.getMemValue(cstr+n)) - .assert(chr('d')===w.getMemValue(cstr+n-1)); - } - - log("scopedAlloc() and friends..."); - { - const alloc = w.alloc, dealloc = w.dealloc; - w.alloc = w.dealloc = null; - T.assert(!w.scopedAlloc.level) - .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) - .mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); - w.alloc = alloc; - T.mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); - w.dealloc = dealloc; - T.mustThrowMatching(()=>w.scopedAllocPop(), /^Invalid state/) - .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) - .mustThrowMatching(()=>w.scopedAlloc.level=0, /read-only/); - const asc = w.scopedAllocPush(); - let asc2; - try { - const p1 = w.scopedAlloc(16), - p2 = w.scopedAlloc(16); - T.assert(1===w.scopedAlloc.level) - .assert(Number.isFinite(p1)) - .assert(Number.isFinite(p2)) - .assert(asc[0] === p1) - .assert(asc[1]===p2); - asc2 = w.scopedAllocPush(); - const p3 = w.scopedAlloc(16); - T.assert(2===w.scopedAlloc.level) - .assert(Number.isFinite(p3)) - .assert(2===asc.length) - .assert(p3===asc2[0]); - - const [z1, z2, z3] = w.scopedAllocPtr(3); - T.assert('number'===typeof z1).assert(z2>z1).assert(z3>z2) - .assert(0===w.getMemValue(z1,'i32'), 'allocPtr() must zero the targets') - .assert(0===w.getMemValue(z3,'i32')); - }finally{ - // Pop them in "incorrect" order to make sure they behave: - w.scopedAllocPop(asc); - T.assert(0===asc.length); - T.mustThrowMatching(()=>w.scopedAllocPop(asc), - /^Invalid state object/); - if(asc2){ - T.assert(2===asc2.length,'Should be p3 and z1'); - w.scopedAllocPop(asc2); - T.assert(0===asc2.length); - T.mustThrowMatching(()=>w.scopedAllocPop(asc2), - /^Invalid state object/); - } - } - T.assert(0===w.scopedAlloc.level); - w.scopedAllocCall(function(){ - T.assert(1===w.scopedAlloc.level); - const [cstr, n] = w.scopedAllocCString("hello, world", true); - T.assert(12 === n) - .assert(0===w.getMemValue(cstr+n)) - .assert(chr('d')===w.getMemValue(cstr+n-1)); - }); - }/*scopedAlloc()*/ - - log("xCall()..."); - { - const pJson = w.xCall('jaccwabyt_test_ctype_json'); - T.assert(Number.isFinite(pJson)).assert(w.cstrlen(pJson)>300); - } - - log("xWrap()..."); - { - //int jaccwabyt_test_intptr(int * p); - //int64_t jaccwabyt_test_int64_max(void) - //int64_t jaccwabyt_test_int64_min(void) - //int64_t jaccwabyt_test_int64_times2(int64_t x) - //void jaccwabyt_test_int64_minmax(int64_t * min, int64_t *max) - //int64_t jaccwabyt_test_int64ptr(int64_t * p) - //const char * jaccwabyt_test_ctype_json(void) - T.mustThrowMatching(()=>w.xWrap('jaccwabyt_test_ctype_json',null,'i32'), - /requires 0 arg/). - assert(w.xWrap.resultAdapter('i32') instanceof Function). - assert(w.xWrap.argAdapter('i32') instanceof Function); - let fw = w.xWrap('jaccwabyt_test_ctype_json','string'); - T.mustThrowMatching(()=>fw(1), /requires 0 arg/); - let rc = fw(); - T.assert('string'===typeof rc).assert(rc.length>300); - rc = w.xCallWrapped('jaccwabyt_test_ctype_json','*'); - T.assert(rc>0 && Number.isFinite(rc)); - rc = w.xCallWrapped('jaccwabyt_test_ctype_json','string'); - T.assert('string'===typeof rc).assert(rc.length>300); - fw = w.xWrap('jaccwabyt_test_str_hello', 'string:free',['i32']); - rc = fw(0); - T.assert('hello'===rc); - rc = fw(1); - T.assert(null===rc); - - w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); - w.xWrap.argAdapter('twice', (v)=>2n*BigInt(v)); - fw = w.xWrap('jaccwabyt_test_int64_times2','thrice','twice'); - rc = fw(1); - T.assert(12n===rc); - - w.scopedAllocCall(function(){ - let pI1 = w.scopedAlloc(8), pI2 = pI1+4; - w.setMemValue(pI1, 0,'*')(pI2, 0, '*'); - let f = w.xWrap('jaccwabyt_test_int64_minmax',undefined,['i64*','i64*']); - let r1 = w.getMemValue(pI1, 'i64'), r2 = w.getMemValue(pI2, 'i64'); - T.assert(!Number.isSafeInteger(r1)).assert(!Number.isSafeInteger(r2)); - }); - } - }/*testWasmUtil()*/; - - const runTests = function(Module){ - //log("Module",Module); - const sqlite3 = Module.sqlite3, - capi = sqlite3.capi, - oo = sqlite3.oo1, - wasm = capi.wasm; - log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); - log("Build options:",wasm.compileOptionUsed()); - - if(1){ - /* Let's grab those last few lines of test coverage for - sqlite3-api.js... */ - const rc = wasm.compileOptionUsed(['COMPILER']); - T.assert(1 === rc.COMPILER); - const obj = {COMPILER:undefined}; - wasm.compileOptionUsed(obj); - T.assert(1 === obj.COMPILER); - } - log("WASM heap size =",wasm.heap8().length); - //log("capi.wasm.exports.__indirect_function_table",capi.wasm.exports.__indirect_function_table); - - const wasmCtypes = wasm.ctype; - //log("wasmCtypes",wasmCtypes); - T.assert(wasmCtypes.structs[0].name==='sqlite3_vfs'). - assert(wasmCtypes.structs[0].members.szOsFile.sizeof>=4). - assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ - ].members.xFileSize.offset>0); - //log(wasmCtypes.structs[0].name,"members",wasmCtypes.structs[0].members); - [ /* Spot-check a handful of constants to make sure they got installed... */ - 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', - 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', - 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' - ].forEach(function(k){ - T.assert('number' === typeof capi[k]); - }); - [/* Spot-check a few of the WASM API methods. */ - 'alloc', 'dealloc', 'installFunction' - ].forEach(function(k){ - T.assert(capi.wasm[k] instanceof Function); - }); - - const db = new oo.DB(':memory:'), startTime = performance.now(); - try { - log("DB filename:",db.filename,db.fileName()); - const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', - banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; - [ - testWasmUtil, testBasicSanity, testUDF, - testAttach, testIntPtr, testStructStuff, - testSqliteStructs - ].forEach((f)=>{ - const t = T.counter, n = performance.now(); - logHtml(banner1,"Running",f.name+"()..."); - f(db, sqlite3, Module); - logHtml(banner2,f.name+"():",T.counter - t,'tests in',(performance.now() - n),"ms"); - }); - }finally{ - db.close(); - } - logHtml("Total Test count:",T.counter,"in",(performance.now() - startTime),"ms"); - log('capi.wasm.exports',capi.wasm.exports); - }; - - sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ - /** Use a timeout so that we are (hopefully) out from under - the module init stack when our setup gets run. Just on - principle, not because we _need_ to be. */ - //console.debug("theModule =",theModule); - //setTimeout(()=>runTests(theModule), 0); - // ^^^ Chrome warns: "VIOLATION: setTimeout() handler took A WHOLE 50ms!" - self._MODULE = theModule /* this is only to facilitate testing from the console */ - runTests(theModule); - }); -})(); diff --git a/ext/wasm/version-info.c b/ext/wasm/version-info.c new file mode 100644 index 0000000000..62fcd633c8 --- /dev/null +++ b/ext/wasm/version-info.c @@ -0,0 +1,106 @@ +/* +** 2022-10-16 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************* +** This file simply outputs sqlite3 version information in JSON form, +** intended for embedding in the sqlite3 JS API build. +*/ +#ifdef TEST_VERSION +/*3029003 3039012*/ +#define SQLITE_VERSION "X.Y.Z" +#define SQLITE_VERSION_NUMBER TEST_VERSION +#define SQLITE_SOURCE_ID "dummy" +#else +#include "sqlite3.h" +#endif +#include +#include +static void usage(const char *zAppName){ + puts("Emits version info about the sqlite3 it is built against."); + printf("Usage: %s [--quote] --INFO-FLAG:\n\n", zAppName); + puts(" --version Emit SQLITE_VERSION (3.X.Y)"); + puts(" --version-number Emit SQLITE_VERSION_NUMBER (30XXYYZZ)"); + puts(" --download-version Emit /download.html version number (3XXYYZZ)"); + puts(" --source-id Emit SQLITE_SOURCE_ID"); + puts(" --json Emit all info in JSON form"); + puts("\nThe non-JSON formats may be modified by:\n"); + puts(" --quote Add double quotes around output."); +} + +int main(int argc, char const * const * argv){ + int fJson = 0; + int fVersion = 0; + int fVersionNumber = 0; + int fDlVersion = 0; + int dlVersion = 0; + int fSourceInfo = 0; + int fQuote = 0; + int nFlags = 0; + int i; + + for( i = 1; i < argc; ++i ){ + const char * zArg = argv[i]; + while('-'==*zArg) ++zArg; + if( 0==strcmp("version", zArg) ){ + fVersion = 1; + }else if( 0==strcmp("version-number", zArg) ){ + fVersionNumber = 1; + }else if( 0==strcmp("download-version", zArg) ){ + fDlVersion = 1; + }else if( 0==strcmp("source-id", zArg) ){ + fSourceInfo = 1; + }else if( 0==strcmp("json", zArg) ){ + fJson = 1; + }else if( 0==strcmp("quote", zArg) ){ + fQuote = 1; + --nFlags; + }else{ + printf("Unhandled flag: %s\n", argv[i]); + usage(argv[0]); + return 1; + } + ++nFlags; + } + + if( 0==nFlags ) fJson = 1; + + { + const int v = SQLITE_VERSION_NUMBER; + int ver[4] = {0,0,0,0}; + ver[0] = (v / 1000000) * 1000000; + ver[1] = v % 1000000 / 100 * 1000; + ver[2] = v % 100 * 100; + dlVersion = ver[0] + ver[1] + ver[2] + ver[3]; + } + if( fJson ){ + printf("{\"libVersion\": \"%s\", " + "\"libVersionNumber\": %d, " + "\"sourceId\": \"%s\"," + "\"downloadVersion\": %d}"/*missing newline is intentional*/, + SQLITE_VERSION, + SQLITE_VERSION_NUMBER, + SQLITE_SOURCE_ID, + dlVersion); + }else{ + if(fQuote) printf("%c", '"'); + if( fVersion ){ + printf("%s", SQLITE_VERSION); + }else if( fVersionNumber ){ + printf("%d", SQLITE_VERSION_NUMBER); + }else if( fSourceInfo ){ + printf("%s", SQLITE_SOURCE_ID); + }else if( fDlVersion ){ + printf("%d", dlVersion); + } + if(fQuote) printf("%c", '"'); + puts(""); + } + return 0; +} diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make new file mode 100644 index 0000000000..81b41870c8 --- /dev/null +++ b/ext/wasm/wasmfs.make @@ -0,0 +1,113 @@ +#!/usr/bin/make +#^^^^ help emacs select makefile mode +# +# This is a sub-make for building a standalone wasmfs-based +# sqlite3.wasm. It is intended to be "include"d from the main +# GNUMakefile. +######################################################################## +MAKEFILE.wasmfs := $(lastword $(MAKEFILE_LIST)) + +# Maintenance reminder: these particular files cannot be built into a +# subdirectory because loading of the auxiliary +# sqlite3-wasmfs.worker.js file it creates fails if sqlite3-wasmfs.js +# is loaded from any directory other than the one in which the +# containing HTML lives. Similarly, they cannot be loaded from a +# Worker to an Emscripten quirk regarding loading nested Workers. +dir.wasmfs := $(dir.wasm) +sqlite3-wasmfs.js := $(dir.wasmfs)/sqlite3-wasmfs.js +sqlite3-wasmfs.wasm := $(dir.wasmfs)/sqlite3-wasmfs.wasm + +CLEAN_FILES += $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.wasm) \ + $(subst .js,.worker.js,$(sqlite3-wasmfs.js)) + +######################################################################## +# emcc flags for .c/.o. +sqlite3-wasmfs.cflags := +sqlite3-wasmfs.cflags += -std=c99 -fPIC +sqlite3-wasmfs.cflags += -pthread +sqlite3-wasmfs.cflags += $(cflags.common) +sqlite3-wasmfs.cflags += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS + +######################################################################## +# emcc flags specific to building the final .js/.wasm file... +sqlite3-wasmfs.jsflags := -fPIC +sqlite3-wasmfs.jsflags += --no-entry +sqlite3-wasmfs.jsflags += --minify 0 +sqlite3-wasmfs.jsflags += -sMODULARIZE +sqlite3-wasmfs.jsflags += -sSTRICT_JS +sqlite3-wasmfs.jsflags += -sDYNAMIC_EXECUTION=0 +sqlite3-wasmfs.jsflags += -sNO_POLYFILL +sqlite3-wasmfs.jsflags += -sEXPORTED_FUNCTIONS=@$(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +sqlite3-wasmfs.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack + # wasmMemory ==> for -sIMPORTED_MEMORY + # allocateUTF8OnStack ==> wasmfs internals +sqlite3-wasmfs.jsflags += -sUSE_CLOSURE_COMPILER=0 +sqlite3-wasmfs.jsflags += -sIMPORTED_MEMORY +#sqlite3-wasmfs.jsflags += -sINITIAL_MEMORY=13107200 +#sqlite3-wasmfs.jsflags += -sTOTAL_STACK=4194304 +sqlite3-wasmfs.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) +sqlite3-wasmfs.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. +#sqlite3-wasmfs.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API +# Perhaps the wasmfs build doesn't? +#sqlite3-wasmfs.jsflags += -sABORTING_MALLOC +sqlite3-wasmfs.jsflags += -sALLOW_TABLE_GROWTH +sqlite3-wasmfs.jsflags += -Wno-limited-postlink-optimizations +# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. +sqlite3-wasmfs.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 +sqlite3-wasmfs.jsflags += -sLLD_REPORT_UNDEFINED +#sqlite3-wasmfs.jsflags += --import-undefined +sqlite3-wasmfs.jsflags += -sMEMORY64=0 +sqlite3-wasmfs.jsflags += -sINITIAL_MEMORY=128450560 +# ^^^^ 64MB is not enough for WASMFS/OPFS test runs using batch-runner.js +sqlite3-wasmfs.fsflags := -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2 -sENVIRONMENT=web,worker +# -sPTHREAD_POOL_SIZE values of 2 or higher trigger that bug. +sqlite3-wasmfs.jsflags += $(sqlite3-wasmfs.fsflags) +#sqlite3-wasmfs.jsflags += -sALLOW_MEMORY_GROWTH +#^^^ using ALLOW_MEMORY_GROWTH produces a warning from emcc: +# USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, +# see https://github.com/WebAssembly/design/issues/1271 [-Wpthreads-mem-growth] +sqlite3-wasmfs.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) +$(eval $(call call-make-pre-js,sqlite3-wasmfs)) +sqlite3-wasmfs.jsflags += $(pre-post-common.flags) $(pre-post-sqlite3-wasmfs.flags) +$(sqlite3-wasmfs.js): $(sqlite3-wasm.c) \ + $(EXPORTED_FUNCTIONS.api) $(MAKEFILE) $(MAKEFILE.wasmfs) \ + $(pre-post-sqlite3-wasmfs.deps) + @echo "Building $@ ..." + $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ + $(sqlite3-wasmfs.cflags) $(sqlite3-wasmfs.jsflags) \ + $(sqlite3-wasm.c) + chmod -x $(sqlite3-wasmfs.wasm) + $(maybe-wasm-strip) $(sqlite3-wasmfs.wasm) + @ls -la $@ $(sqlite3-wasmfs.wasm) +$(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.js) +wasmfs: $(sqlite3-wasmfs.js) +all: wasmfs + +######################################################################## +# speedtest1 for wasmfs. +speedtest1-wasmfs.js := $(dir.wasmfs)/speedtest1-wasmfs.js +speedtest1-wasmfs.wasm := $(subst .js,.wasm,$(speedtest1-wasmfs.js)) +speedtest1-wasmfs.eflags := $(sqlite3-wasmfs.fsflags) +speedtest1-wasmfs.eflags += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS +speedtest1-wasmfs.eflags += -sALLOW_MEMORY_GROWTH=0 +speedtest1-wasmfs.eflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) +$(eval $(call call-make-pre-js,speedtest1-wasmfs)) +$(speedtest1-wasmfs.js): $(speedtest1.cses) $(sqlite3-wasmfs.js) \ + $(MAKEFILE) $(MAKEFILE.wasmfs) \ + $(pre-post-speedtest1-wasmfs.deps) \ + $(EXPORTED_FUNCTIONS.speedtest1) + @echo "Building $@ ..." + $(emcc.bin) \ + $(speedtest1-wasmfs.eflags) $(speedtest1-common.eflags) \ + $(pre-post-speedtest1-wasmfs.flags) \ + $(speedtest1.cflags) \ + $(sqlite3-wasmfs.cflags) \ + -o $@ $(speedtest1.cses) -lm + $(maybe-wasm-strip) $(speedtest1-wasmfs.wasm) + ls -la $@ $(speedtest1-wasmfs.wasm) + +speedtest1: $(speedtest1-wasmfs.js) +CLEAN_FILES += $(speedtest1-wasmfs.js) $(speedtest1-wasmfs.wasm) \ + $(subst .js,.worker.js,$(speedtest1-wasmfs.js)) +# end speedtest1.js +######################################################################## diff --git a/magic.txt b/magic.txt index 65622687cc..f52a7a90ed 100644 --- a/magic.txt +++ b/magic.txt @@ -9,18 +9,18 @@ # PRAGMA application_id = INTEGER; # # INTEGER can be any signed 32-bit integer. That integer is written as -# a 4-byte big-endian integer into offset 68 of the database header. +# a 4-byte big-endian integer into offset 68 of the database header. # # The Monotone application used "PRAGMA user_version=1598903374;" to set # its identifier long before "PRAGMA application_id" became available. # The user_version is very similar to application_id except that it is -# stored at offset 68 instead of offset 60. The application_id pragma +# stored at offset 60 instead of offset 68. The application_id pragma # is preferred. The rule using offset 60 for Monotone is for historical # compatibility only. # 0 string =SQLite\ format\ 3 ->68 belong =0x0f055112 Fossil checkout - ->68 belong =0x0f055113 Fossil global configuration - +>68 belong =0x0f055112 Fossil checkout - +>68 belong =0x0f055113 Fossil global configuration - >68 belong =0x0f055111 Fossil repository - >68 belong =0x42654462 Bentley Systems BeSQLite Database - >68 belong =0x42654c6e Bentley Systems Localization File - @@ -29,4 +29,5 @@ >68 belong =0x47503130 OGC GeoPackage version 1.0 file - >68 belong =0x45737269 Esri Spatially-Enabled Database - >68 belong =0x4d504258 MBTiles tileset - +>68 belong =0x6a035744 TeXnicard card database >0 string =SQLite SQLite3 database diff --git a/main.mk b/main.mk index 69a073a549..52fcdcd2ed 100644 --- a/main.mk +++ b/main.mk @@ -68,7 +68,7 @@ LIBOBJ+= vdbe.o parse.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memdb.o memjournal.o \ mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ - notify.o opcodes.o os.o os_unix.o os_win.o \ + notify.o opcodes.o os.o os_kv.o os_unix.o os_win.o \ pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \ random.o resolve.o rowset.o rtree.o \ select.o sqlite3rbu.o status.o stmt.o \ @@ -134,6 +134,7 @@ SRC = \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ $(TOP)/src/os_setup.h \ + $(TOP)/src/os_kv.c \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ $(TOP)/src/os_win.h \ @@ -412,6 +413,7 @@ TESTSRC2 = \ $(TOP)/src/main.c \ $(TOP)/src/mem5.c \ $(TOP)/src/os.c \ + $(TOP)/src/os_kv.c \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ $(TOP)/src/pager.c \ @@ -446,6 +448,9 @@ TESTSRC2 = \ $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/session/sqlite3changebatch.c \ $(TOP)/ext/session/test_session.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/test_recover.c \ fts5.c # Header files used by all library source files. @@ -541,7 +546,9 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC -FUZZCHECK_OPT = -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPT += -I$(TOP)/test +FUZZCHECK_OPT += -I$(TOP)/ext/recover +FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4 @@ -551,7 +558,10 @@ FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB FUZZSRC += $(TOP)/test/fuzzcheck.c FUZZSRC += $(TOP)/test/ossfuzz.c +FUZZSRC += $(TOP)/test/vt02.c FUZZSRC += $(TOP)/test/fuzzinvariants.c +FUZZSRC += $(TOP)/ext/recover/dbdata.c +FUZZSRC += $(TOP)/ext/recover/sqlite3recover.c DBFUZZ_OPT = KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ ST_OPT = -DSQLITE_THREADSAFE=0 @@ -610,7 +620,7 @@ dbfuzz2$(EXE): $(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h $(TCCX) -I. -g -O0 -DSTANDALONE -o dbfuzz2$(EXE) \ $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS) $(THREADLIB) -fuzzcheck$(EXE): $(FUZZSRC) sqlite3.c sqlite3.h +fuzzcheck$(EXE): $(FUZZSRC) sqlite3.c sqlite3.h $(FUZZDEP) $(TCCX) -o fuzzcheck$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_ENABLE_MEMSYS5 $(FUZZCHECK_OPT) -DSQLITE_OSS_FUZZ \ $(FUZZSRC) sqlite3.c $(TLIBS) $(THREADLIB) @@ -761,7 +771,9 @@ SHELL_SRC = \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ - $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)/src/test_windirent.c shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl diff --git a/manifest b/manifest index c6fa5feea5..3dad987da3 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ C Merge\sthe\slatest\strunk\senhancements\sinto\sthe\sbegin-concurrent-pnu-wal2\sbranch. -D 2022-09-30T14:04:15.072 +D 2022-11-04T19:09:41.774 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 6b71e6e41d5a56742309afdb69f2408d9827744123112f2649420b4beb055dc6 +F Makefile.in 55e1cb08fc07a83b5e57d204f2e491dc8d596664e252b728f28065ee9ac2f9f8 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc e27b351d669b070d1c217df18118dcb3a41f7ca6bf18fa3b47e36f93792823cc +F Makefile.msc 8f55fd960e3c3418c1fbd8b04c491e97fa8fa65b137979a4a5c150fc7e6a536a F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 8868ddfa6e1eee218286021a94b3e22d13e550c76c72d878857547ca001de24a F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -32,10 +32,9 @@ F autoconf/tea/win/makefile.vc 2c478a9a962e48b2bf9062734e04d7c63c556e21709541917 F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea00342d1a1a7eaa19cb F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 -F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure f959db96f314b3b91b3d658eebbc0a96b9542f1265c4de97e885aedb6bdcead5 x -F configure.ac 3ef6eeff4387585bfcab76b0c3f6e15a0618587bb90245dd5d44e4378141bb35 +F configure b93755fe94b3e9b2015f3c2d6c63928e8899b0a7c54e042deb843498094179da x +F configure.ac 48dc6bfee293eef05910faf085760f2fd79b680aa47b50e8e6a22ca40bb026bb F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/begin_concurrent.md 4bee2c3990d1eb800f1ce3726a911292a8e4b889300b2ffd4b08d357370db299 @@ -298,12 +297,11 @@ F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0e F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 F ext/misc/carray.h d2b1b12486d531367c37832d3d0dad34eea4bdd83ed839d445521ef01f0bc4e3 -F ext/misc/cksumvfs.c b42ef52eaaa510d54ec320c87bea149e934a3b06cd232be2093562bf669bd572 +F ext/misc/cksumvfs.c 9224e33cc0cb6aa61ff1d7d7b8fd6fe56beca9f9c47954fa4ae0a69bef608f69 F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c85c243 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 -F ext/misc/dbdata.c e316fba936571584e55abd5b974a32a191727a6b746053a0c9d439bd2cf93940 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 @@ -391,6 +389,22 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a +F ext/recover/dbdata.c 1d5353d3af247c4e0656f8f88a80564aa840644c1177212dd11a186dce4ab213 w ext/misc/dbdata.c +F ext/recover/recover1.test 02004eb8f9ec2825ba77e24742c18e45162cb21d27e76a3a435b83a759a1131a +F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a +F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e +F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49 +F ext/recover/recovercorrupt2.test 74bef7dd2d7dd4856f3da21be6e213d27da44827e0f5f0946ca0325b46d163ed +F ext/recover/recoverfault.test 9d9f88eeb222615a25e7514f234c950d46bee20d24cd8db49d8fff8d650dcfe1 +F ext/recover/recoverfault2.test 730e7371bcda769554d15460cb23126abba1be8eca9539ccabf63623e7bb7e09 +F ext/recover/recoverold.test 68db3d6f85dd2b98e785b6c4da4f5eea4bbe52ccf6674d9a94c7506dc92596aa +F ext/recover/recoverpgsz.test 3658ab8e68475b1bb87d6af88baa04551c84b73280a566a1be847182410ffc58 +F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f +F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411 +F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 +F ext/recover/sqlite3recover.c 3e38f2bd607f6ecd8dc10ed419363448c206791c7ce344e3a2a6848731b9f37c +F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 +F ext/recover/test_recover.c 1a34e2d04533d919a30ae4d5caeb1643f6684e9ccd7597ca27721d8af81f4ade F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -402,7 +416,7 @@ F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c3350 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/geopoly.c bb0dcd013dbb3fe0335b6bbae0d9c29fcfffda93d5ef6c31daa47e802132e435 -F ext/rtree/rtree.c d7b4b8b81d8d54376a7f81de5be85ec58b37c11604bcf42984a8418b34158d93 +F ext/rtree/rtree.c b963ebced19f249d8523396a779351481d13a67c36c3c1f034b982886cd39a42 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test d47f58832145fcfed9067bc457ca8664962196c4566c17a1ebd679367db55d11 F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d @@ -481,43 +495,75 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 -F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 -F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3b1ae2b1 -F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 +F ext/wasm/GNUmakefile 3aa8c160705ab9d9d4d552a1f4e630925a65a27df216befe2e9a956904434c1d +F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 +F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md b6d0fb64bfdf7bf9ce6938ea4104228f6f5bbef600f5d910b2f8c8694195988c -F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c -F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b -F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 -F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b -F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 -F ext/wasm/api/sqlite3-api-opfs.js c93cdd14f81a26b3a64990515ee05c7e29827fbc8fba4e4c2fef3a37a984db89 -F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb -F ext/wasm/api/sqlite3-api-worker.js 1124f404ecdf3c14d9f829425cef778cd683911a9883f0809a463c3c7773c9fd +F ext/wasm/api/README.md 1350088aee90e959ad9a94fab1bb6bcb5e99d4d27f976db389050f54f2640c78 +F ext/wasm/api/extern-post-js.js f3dc4219a2a1f183d98452dcbd55a0c5351b759eccca75480a92473974d8b047 +F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 +F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 +F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 +F ext/wasm/api/pre-js.js 287e462f969342b032c03900e668099fa1471d852df7a472de5bc349161d9c04 +F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 +F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 +F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed +F ext/wasm/api/sqlite3-api-opfs.js cdcbb57acc66f4569ac9e18f9d13d5a3657d8aae195725c6324943da56c1005d +F ext/wasm/api/sqlite3-api-prologue.js 952ba908cc5ee42728c5c09dd549af32ef0c3cc15ab7b919a8007c5684f69320 +F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f w ext/wasm/api/sqlite3-api-worker.js +F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 +F ext/wasm/api/sqlite3-opfs-async-proxy.js ab7d2888ad9b3dd24bb782bd882fcada2a20cb88eb78c8f36e7bfe708857dbd1 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 8585793ca8311c7a0618b7e00ed2b3729799c20664a51f196258576e3d475c9e -F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f -F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 +F ext/wasm/api/sqlite3-wasm.c 778409e00fb25a4d6989be17fc856c84460198fd3b05ba2ef8289e60c50157ca +F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b +F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 w ext/wasm/api/sqlite3-worker.js +F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 +F ext/wasm/batch-runner.js 49609e89aaac9989d6c1ad3fae268e4878e1ad7bc5fd3e5c2f44959660780b2e +F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 -F ext/wasm/common/whwasmutil.js 3d9deda1be718e2b10e2b6b474ba6ba857d905be314201ae5b3df5eef79f66aa +F ext/wasm/common/testing.css 35889709547d89a6109ff83b25c11bbc91d8dd43aab8722e428655ca98880a06 +F ext/wasm/common/whwasmutil.js 16a592d5c304a2d268ca1c28e08a5b029a2f3cbe10af78dbc3456cfc9e3559d1 +F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed +F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 +F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 +F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e +F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 +F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f +F ext/wasm/demo-worker1-promiser.js b85a2bb1b918db4f09dfa24419241cb3edad7791389425c2505092e9b715017d +F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d w ext/wasm/testing2.html +F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa w ext/wasm/testing2.js +F ext/wasm/dist.make 481289899a07958439d07ee4302ff86235fa0fbb72f17ea05db2be90a94abf90 +F ext/wasm/fiddle.make e570ec1bfc7d803507a2e514fe32f673fe001b2114b85c73c3964a462ba8bcfc F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae +F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 -F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 -F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9de2a985f3c21e7082b2 -F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 -F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f -F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 -F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 -F ext/wasm/testing1.js cba7134901a965743fa9289d82447ab71de4690b1ee5d06f6cb83e8b569d7943 -F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 -F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc +F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 +F ext/wasm/index-dist.html cb0da16cba0f21cda2c25724c5869102d48eb0af04446acd3cd0ca031f80ed19 +F ext/wasm/index.html ce6a68a75532b47e3c0adb83381a06d15de8c0ac0331fb7bf31d33f8e7c77dc4 +F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 +F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e +F ext/wasm/module-symbols.html eca884ef4380612145ee550213be57478ee2b9cd9a9c2b27530cc23359c99682 +F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 +F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 +F ext/wasm/speedtest1-wasmfs.html bc28eb29b69a73864b8d7aae428448f8b7e1de81d8bfb9bba99541322054dbd0 +F ext/wasm/speedtest1-worker.html 94246488e10af9daa1ebd0979b1b8d7a22a579e5a983fa2a5ad94591ecf55f2c +F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1 +F ext/wasm/speedtest1.html e4c4e5c1c8ec1ad13c995e346e4216a1df152fd2c5cd17e0793b865b2f3c5000 +F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x +F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 +F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 +F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f +F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac +F ext/wasm/tester1-worker.html 51bf39e2b87f974ae3d5bc3086e2fb36d258f3698c54f6e21ba4b3b99636fa27 +F ext/wasm/tester1.html 624ec41cd9f78a1f2b6d7df70aaa7a6394396b1f2455ecbd6de5775c1275b121 +F ext/wasm/tester1.js 3a5558201359ff8a1c3051ab24fcc8bed5a4e99ae5e086e5184cbfc915d08724 +F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 +F ext/wasm/wasmfs.make edfd60691d10fd19ada4c061280fd7fbe4cf5f6bf6b913268e8ebedfccea6ab5 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk c7406b33f2184b5678adb9e5582761d013ef14989aa7cb7bf5425120bcf1d892 +F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 +F main.mk e458e3deb04302c8c90f4fddcb3a6dbdb35cf39b406760093fda75f625bb2e99 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -529,38 +575,39 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a +F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 w config.h.in F src/alter.c 0390ca1d69ec3626cfa9f153114b7ab233e6b2bada6a9eb91361ed385fe90deb -F src/analyze.c aabdf3769c7fd9954a8ec508eb7041ae174b66f88d12c47199fabbea9a646467 +F src/analyze.c d2fce73f6a024897593012c6ca25368629fa4aeb49960d88a52fac664582e483 F src/attach.c 4431f82f0247bf3aaf91589acafdff77d1882235c95407b36da1585c765fbbc8 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 3907fcbe8a0c8c2db58d97087d15cdabbf2842adb9125df9ab9ff87d3db16775 F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 1cc413c83cf2dfd7b33d82e74e518f57c186d4eb19492a8c3e5f1658088eb04f +F src/btree.c 1ec9a2e72c4de9e4f8ae1e1c742159d217fab16d693edb35cc9f48541c2c2b5c F src/btree.h 900067641b64d619e6e2a93bd115c952a52f41d3bee32e551e2a4ceee05fc431 F src/btreeInt.h 650add92a0ffc8c315406f140325c5f41f0e386848dafbb1e27a72fe7cf6f179 -F src/build.c c3cfa409c354a291acdb9d40199fa30e71020c30fdc6dee2d936ca15deb9046a +F src/build.c 73d472e3de6fb356bcfbb062814c95e6d5f6f986cbfc771fa3d906762faecaf7 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c 93e4b5f4faf6d3f688988a116773259a4fbfb4ddac0e9bf9d0ae0429390c2543 +F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 -F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec +F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 1cbdd76eeedb729ea9060df03e3e6b74a302784a13bfa38794a8194f894641ea +F src/expr.c 847f87d9df3ede2b2b0a8db088af0b9c1923b21009f8ea1b9b7b28cb0a383170 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 -F src/func.c c4b2229aac24b3d7989ef51abae0e5e87e58e8bda3c7b6c55f76d231dbedfb8d -F src/global.c e83ee571b79ee3adc32e380cf554cf1254bc43763d23786c71721fbcdfbbb965 +F src/func.c 2c186ba4d58b61380b31f01cebeff6d30d93f9ac1bf7d09533926ca284ddac1d +F src/global.c e06ff8e0acd85aec13563c9ecb44fbbf38232ccf73594998fd880b92d619594b F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c aea5361767817f917b0f0f647a1f0b1621bd858938ae6ae545c3b6b9814b798f +F src/insert.c 90a32bc7faa755cd5292ade21d2b3c6edba8fd1d70754a364caccabfde2c3bb2 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b -F src/main.c db487c05a732709ae4c94f1592ceaa49e05d0f56257223f2ae7e0dec54a858c3 +F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9 +F src/main.c 568be0895002517946bfb56444ec19c4ee9edd0d4091550597ec81f5f43195c3 F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -576,11 +623,12 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c dd2b3f1cc1863079bc1349ac0fec395a500090c4fe4e11ab775310a49f2f956d F src/mutex_w32.c caa50e1c0258ac4443f52e00fe8aaea73b6d0728bd8856bedfff822cae418541 F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 -F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9 +F src/os.c 81c9c1c52eab711e27e33fd51fe5788488d3a02bc1a71439857abbee5d0d2c97 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 -F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 52ac6823d3895c6a3a5cf4b07c05052f7ee8bd100a076bf0cd672ec83c996246 +F src/os_kv.c 0e59600d25b72034c7666b8b7dcc527f039b5d9c16f24a7eca4c08c66f63c364 +F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 +F src/os_unix.c 6a1e13c207b146cf4b5d82d359cf80e38acd112e56e32b4403a76b98fb7a8ec7 F src/os_win.c 8d129ae3e59e0fa900e20d0ad789e96f2e08177f0b00b53cdda65c40331e0902 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 6d3a93a7abfcb17e69ceb8a5e78ab74a0234ebf2a87819cc63d5c6044e78834c @@ -589,24 +637,24 @@ F src/parse.y 17c50d262d92083badeb60b3ebe4725e19c76548f90aea898ab07d4f2940a7d8 F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc -F src/pragma.c c471a8752cc37919a213d860a1550c3e4b5bc1a416dff2aa72212e73c6982230 +F src/pragma.c 49a34aba78f9ad034d46b894c158e0c4199d6eb07cfa8263e75fe374ec223950 F src/pragma.h 1f421360eed1a7721e8c521463df8519a7c8d0d5893ebd9dbfe0dba8de996f8c -F src/prepare.c 971d5819a4bda88038c2283d71fc0a2974ffc3dd480f9bd941341017abacfd1b +F src/prepare.c 1b02be0441eda4579471fea097f678effcbb77ef0c39ab3f703c837822bcd674 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c f767e3c0048b408aa14bcdc084fdb9520b88bfdb7cd6d7e356f70c7ee26bcb45 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 321a529476188dbd8ff1e795e1f78287c6ea0e895523081d3c3954f416b9766f -F src/shell.c.in e7e7c2c69ae86c5ee9e8ad66227203d46ff6dce8700a1b1dababff01c71d33df -F src/sqlite.h.in 8134578d7027812e7c3cfdff69117b08f5a69592c5168dfc2f545368a284c9d8 +F src/select.c 8f376b788138a6e70a00a466d62ae01382284811f45ac5ec6385ec7f4bcb5eee +F src/shell.c.in 84bb08d8762920285f08f1c0993f1b3992ac43af5d72445cb8a973fc50c71923 +F src/sqlite.h.in 0a3272d06780400fddec94ca2d0fbd6402e0d913905844096c1cbcba4a5b0a59 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d -F src/sqliteInt.h c9b07e3a5542b935ac160c71705c345d3e113438bdb67986ef09ed36efd16b8f +F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f +F src/sqliteInt.h db846794194ce108b4dd343a516910b07cf0f7d8e3497b9a6028e792af1f5047 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 4e64ba300a5a26e0f1170e09032429faeb65e45e8f3d1a7833e8edb69fc2979e -F src/test1.c 95c688261edf59ff77676aca426b343beb9f90d6548ecb16189ce614e5a63620 +F src/test1.c 917aafccba9b2061cc52471fefad26dfce4a490df4b04d763f28baa8e0b148ca F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -623,7 +671,7 @@ F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 F src/test_config.c c7a93ef3c0c881e3c46ff14163af886a602317206d82a7dc4ebbf400e661a6ff F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f -F src/test_demovfs.c 86142ba864d4297d54c5b2e972e74f3141ae4b30f05b3a95824184ed2d3d7f91 +F src/test_demovfs.c 7cc7623d1025d1e92c51da20fd25060759733b7a356a121545a3b7d2faa8a0f1 F src/test_devsym.c aff2255ea290d7718da08af30cdf18e470ff7325a5eff63e0057b1496ed66593 F src/test_fs.c ba1e1dc18fd3159fdba0b9c4256f14032159785320dfbd6776eb9973cb75d480 F src/test_func.c 24df3a346c012b1fc9e1001d346db6054deb426db0a7437e92490630e71c9b0a @@ -635,7 +683,7 @@ F src/test_journal.c a0b9709b2f12b1ec819eea8a1176f283bca6d688a6d4a502bd6fd79786f F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd F src/test_malloc.c 21121ea85b49ec0bdb69995847cef9036ef9beca3ce63bbb776e4ea2ecc44b97 F src/test_md5.c 7268e1e8c399d4a5e181b64ac20e1e6f3bc4dd9fc87abac02db145a3d951fa8c -F src/test_multiplex.c 1b23782212a01349fac382913ef82b8de4ae8a4cb46556602c2ee733edabbbc9 +F src/test_multiplex.c d8bc260a57c2028946a9bce9918d24e69e44cebd71ac6be240aa5b6b75e2b369 F src/test_multiplex.h 5436d03f2d0501d04f3ed50a75819e190495b635 F src/test_mutex.c abf486e91bd65e2448027d4bb505e7cce6ba110e1afb9bd348d1996961cadf0d F src/test_onefile.c f31e52e891c5fef6709b9fcef54ce660648a34172423a9cbdf4cbce3ba0049f4 @@ -649,7 +697,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a F src/test_superlock.c f4d4cc7319a608a54b7608158e8c7135fac19b88d6179e5bf17080e89d1f0278 F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 -F src/test_tclsh.c dec6a760161492d51580e3b3702c6d527ae03aff238cc52fd15234085157a154 +F src/test_tclsh.c 0b15685c31189b38b0a773bbbf8b231976db0f001c2aad6524488b5eb9de2007 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 269ea9e1fa5828dba550eb26f619aa18aedbc29fd92f8a5f6b93521fbb74a61c F src/test_vdbecov.c f60c6f135ec42c0de013a1d5136777aa328a776d33277f92abac648930453d43 @@ -662,31 +710,31 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 -F src/trigger.c bc70c58e713dcfb6cabe5cc0bed71aedb02c3e9e128c6089a78aca945ba4d720 -F src/update.c c6fbfa86c7dbff057bb62b29e00846dec1468f12c62e1a575f745f87c5a55818 -F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 +F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 +F src/update.c 40918db2449022ee11c7df852df8d4d8e3e77ca4386a9399bbd562c4e702098e +F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 0be191521ff6d2805995f4910f0b6231b42843678b2efdc1abecaf39929a673f -F src/vacuum.c 5b7888f917936dda09f98b1fda164cff1dad44fb5e143436bdbb9dc3191ce2df -F src/vdbe.c be71256da51f60d13e0f53850bae9902d7c9926c71e5e50d4c686bd92e458574 -F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 +F src/vacuum.c f6e47729554e0d2c576bb710c415d5bc414935be0d7a70f38d1f58ffa7a6d8c0 +F src/vdbe.c 687a642e6cb46fa3b582a5c2ce227f8b49f70ad18cca6ae3b1b36f5a6b559f14 +F src/vdbe.h 58675f47dcf3105bab182c3ad3726efd60ffd003e954386904ac9107d0d2b743 F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f -F src/vdbeapi.c fc3183daf72808b4311b228989120fdbc2dc44972fb0d77d5c453460cc0e5b2c -F src/vdbeaux.c df2bbf6b5d6e45a7ab4061fb11f788fd1e4f91ae95ef0309cf8b9465f94eec83 +F src/vdbeapi.c 1e8713d0b653acb43cd1bdf579c40e005c4844ea90f414f065946a83db3c27fb +F src/vdbeaux.c fc2a532acdc75265f4f4e236e004ef22dba4a6e1aababb31c504a4b954aa9819 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd -F src/vdbemem.c c3ce80af15e2ff5c2824a8db881681cbf511376f13613da020bac6d320c535b1 +F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c -F src/vtab.c bb53f9e2eaeecca07158643dd3d5039cf13b525fe2d267e113b39a36f374556c +F src/vtab.c b2f993aa954078985bc40317bb2140fe0880a08a7440f3a428b60fce74636808 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 393ffbef8381b50265a3f0de5cf3c68d0df6c6867df429c5719c2446e9254126 F src/wal.h 7a733af13b966ecb81872ce397e862116b3575ea53245b90b139a2873ee87825 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 63e712bcad47f70e94c2150976cd7da5040933699e3938d4189d064acbe40891 -F src/whereInt.h 70cd30de9ed784aa33fa6bd1245f060617de7a00d992469b6d8e419eed915743 -F src/wherecode.c 6bb1cf9d0a4e3e04dab0bf0ea4a8d936a0dcc05a7e2207beeda6c61aea6dd341 -F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 +F src/where.c 1ef5aae7fac877057b9f360f06b26d4275888460d8fb6e92bbb9e70e07afe946 +F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f +F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 +F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -790,6 +838,7 @@ F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aa F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 +F test/bloom1.test 5318eb0648dff073ca9b9a54387ec2c0a7a07ed3490461fe2db0d074b2eb0e7f F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -813,7 +862,7 @@ F test/capi3c.test 54e2dc0c8fd7c34ad1590d1be6864397da2438c95a9f5aee2f8fbc60c112e F test/capi3d.test 8b778794af891b0dca3d900bd345fbc8ebd2aa2aae425a9dccdd10d5233dfbde F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/carray01.test d55d57bf66b1af1c7ac55fae66ff4910884a8f5d21a90a18797ce386212a2634 -F test/cast.test 336fa21989b5170ebcaf90c24266be22dd97b3e23d1fad5ecf6ad4efb04c4423 +F test/cast.test 6064022ba9af31a8a2ff7bb345e5bd0e74172ffad85bdab5898a42d8227c7585 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/changes.test 9dd8e597d84072122fc8a4fcdea837f4a54a461e6e536053ea984303e8ca937b F test/changes2.test d222c0cbf5ab0ac4d7c180594e486c1bf20b2098d33e56ce33b8e12eba6823b9 @@ -827,7 +876,7 @@ F test/collate1.test 71a6f27fdc93a92f14d8ab80c05e1937656a5a03197e1a10157314554d6 F test/collate2.test 471c6f74573382b89b0f8b88a05256faa52f7964f9e4799e76708a3b1ece6ba4 F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5 F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd -F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6 +F test/collate5.test b1dfeff239ea69ee9225832553f423d37a6184eb730cee06f6846ab4e3c6dbef F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 @@ -871,7 +920,7 @@ F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test a17bbf54fdde78d43cf3cc34b0057719fd4a173a3d824285b67dc5257c064c7b F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af -F test/corruptL.test ecce40d7b9b909a670a42a45d86e30d927735d7e7f09041af438b19529d35532 +F test/corruptL.test 9d1a0055c8db19baccd12f22ac36a33ec7d63afb59e82eb30835aea8f89b94df F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 F test/corruptN.test 57985a0737f5e008283a91c24630cd3c7003d3c7b62824edaa21258e46da9455 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 @@ -942,7 +991,7 @@ F test/e_totalchanges.test c927f7499dc3aa28b9b556b7d6d115a2f0fe41f012b128d16bf1f F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 F test/e_uri.test 86564382132d9c453845eeb5293c7e375487b625900ab56c181a0464908417d8 F test/e_vacuum.test 89fc48e8beee2f9dfd6de1fbb2edea6542dae9121dc0fbe6313764169e742104 -F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 +F test/e_wal.test db7c33642711cf3c7959714b5f012aca08cacfa78da0382f95e849eb3ba66aa4 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b344cb66 @@ -1123,7 +1172,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 7b501d55631c2d759e0bed02ed329904a35690fc6563d7b6cc69b7788a024f26 +F test/fuzzcheck.c 30475c820dc5ab8a87fa3be1fe8ba8199ebfe2544508a759d653688d8d168766 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1131,11 +1180,11 @@ F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e4 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5 F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7 F test/fuzzdata7.db 0166b56fd7a6b9636a1d60ef0a060f86ddaecf99400a666bb6e5bbd7199ad1f2 -F test/fuzzdata8.db ca9a97f401b06b0d5376139ec7e1f9e773e13345a9a2d9ccc0032cdbfedea230 +F test/fuzzdata8.db 653423800b7671e67caa740e977d80e1360f0d69e9992851f3ea5c4a69a2724a F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c d7bb4a0fcc0ac344bcb72f1b86e4ae0acba5ea26dddde8160ee3db6520f10c64 +F test/fuzzinvariants.c 7877178eaa10eb3ea986f81a7010efc371ccd3e13ee5b14fa290b0459002a36a F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/gencol1.test cc0dbb0ee116e5602e18ea7d47f2a0f76b26e09a823b7c36ef254370c2b0f3c1 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -1214,7 +1263,7 @@ F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f2 F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 -F test/joinH.test e67d1d6a8c7141caf981a07386caa7fda0362baa09e03669f9a4fbee812806d0 +F test/joinH.test 15f501b33d848521964afde9865a92aeca79c8c41fa84dc4dc3f865c9ed8c868 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e @@ -1232,7 +1281,7 @@ F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7 F test/like.test 5fe0bc37f307aef0a453ce2de4632bdfc0759448f0421c39f6d53caefe905fac -F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da +F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3d3 F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e @@ -1280,7 +1329,7 @@ F test/memdb1.test 2c4e9cc10d21c6bf4e217d72b7f6b8ba9b2605971bb2c5e6df76018e189f9 F test/memjournal.test 70f3a00c7f84ee2978ad14e831231caa1e7f23915a2c54b4f775a021d5740c6c F test/memjournal2.test 6b9083cfaab9a3281ec545c3da2487999e8025fb7501bbae10f713f80c56454c F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 -F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e +F test/memsubsys1.test 86b8158752af9188ed5b32a30674a1ef71183e6bc4e6808e815cd658ca9058a6 F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 F test/merge1.test 2de6d6ef8d25402764b1aab49d8f9d7f89208c89a6674e437f76de4c812157b8 F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a71a @@ -1354,7 +1403,7 @@ F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test a2af0e20f6c6b1c759e9d6e408bf3c956af11052ada41bdc707e3ecb53452dba +F test/permutations.test cbe27f4ddf1911fa33dc1d2dc728349291d3c10dde43c79296834c456dda4380 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test cae534c12a033a5c319ccc94f50b32811acdef9f67bf19a82ff42697caccd69f F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -1378,7 +1427,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 21e99ee237265d0cf95a0c84b50c784e834acaa4ef05d92a27b262626a656682 -F test/recover.test ccb8c2623902a92ebb76770edd075cb4f75a4760bb7afde38026572c6e79070d +F test/recover.test fd5199f928757cb308661b5fdca1abc19398a798ff7f24b57c3071e9f8e0471e F test/regexp1.test 83c631617357150f8054ca1d1fed40a552b0d0f8eb7a7f090c3be02cee9f9913 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d @@ -1408,7 +1457,7 @@ F test/rowvaluefault.test 963ae9cdaed30a85a29668dd514e639f3556cae903ee9f172ea972 F test/rowvaluevtab.test cd9747bb3f308086944c07968f547ad6b05022e698d80b9ffbdfe09ce0b8da6f F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test ea1a6c08454e1a4edd973589bcf1fca83928ed09c48858fde1e8653b6daab278 +F test/savepoint.test 63a120ec4fbbd5025b238c259d12ed0516fbf4bca6384041cb995ade9a5f00d2 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd @@ -1425,6 +1474,7 @@ F test/schema6.test e4bd1f23d368695eb9e7b51ef6e02ca0642ea2ab4a52579959826b5e7dce F test/schemafault.test 1936bceca55ac82c5efbcc9fc91a1933e45c8d1e1d106b9a7e56c972a5a2a51e F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2ff09384 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 +F test/seekscan1.test d79c97de5bb1dd1fd466687f3014add514fddf8248c57baf51d749c7dfd573d8 F test/select1.test 692e84cfa29c405854c69e8a4027183d64c22952866a123fabbce741a379e889 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 8d04b66df7475275a65f7e4a786d6a724c30bd9929f8ae5bd59c8d3d6e75e6cd @@ -1434,7 +1484,7 @@ F test/select6.test 9b2fb4ffedf52e1b5703cfcae1212e7a4a063f014c0458d78d29aca3db76 F test/select7.test f659f231489349e8c5734e610803d7654207318f F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 -F test/selectA.test 985df11dfc58dedd6f8124dea32754566346b93e294f26291b5c96d10c8cc889 +F test/selectA.test 6aef8b2136a4ac7a3e2e4161d2b8ca7bc6ebe2779de084f9bb66ca9e2323a937 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test fec14c9015ed4ec941508bbc144f30b42e40ac34a4bb33001450369865dd0b75 F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d32189bfb @@ -1498,7 +1548,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest1.c 8bf7ebac9ac316feed6656951249db531dc380c73fb3e3b22e224ffda96beff6 +F test/speedtest1.c 645dbf022337116fcef60ac2579e8d25c94e4475109e2c6f48005f302efe7b09 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1542,7 +1592,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl c2c5095594f8b6d9bcb30a835f872241faf9adf8c26f613b7c5cb672202ab2fd +F test/tester.tcl 37d077304c67e35a5496c9f0362f539bd0180f94c2fdeb3a8d93b3c04f7a048f F test/testrunner.tcl 86b57135754ab2160aeb04b4829d321fb285a5cfa7a505fe61d69aed605854cc F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -1739,7 +1789,7 @@ F test/tt3_vacuum.c 71b254cde1fc49d6c8c44efd54f4668f3e57d7b3a8f4601ade069f75a999 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 1aeb81976841a91eef292723649b5c4fe3bc3cac F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a -F test/unionall.test fee6e6adebe5446938475432ae8b042835a286c7e2efdc1b186d2182dfb93ffb +F test/unionall.test bfeeea6c18c09a46f7e8bed69173e31c71336152e6253fb37c7257c76509f0e4 F test/unionall2.test 71e8fa08d5699d50dc9f9dc0c9799c2e7a6bb7931a330d369307a4df7f157fa1 F test/unionallfault.test 652bfbb630e6c43135965dc1e8f0a9a791da83aec885d626a632fe1909c56f73 F test/unionvtab.test e1704ab1b4c1bb3ffc9da4681f8e85a0b909fd80b937984fc94b27415ac8e5a4 @@ -1766,7 +1816,7 @@ F test/uri.test 8f27eaa41804099fca15101e30fd1b29aebbebf32d4e3c24614fa6319216936d F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 9fde0bb5d3a821594aa68c6829ab9c5453a084384137ebb9f6153e2d678039da -F test/vacuum-into.test f0b8c091df5305728b6973e9cce4166c861955b650dd3c599cb045d7160d3971 +F test/vacuum-into.test e0e3406845be4cf1b44db354179e5d9437e38bc267e4ac8e8dc617f9c3c903ab F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8af520 F test/vacuum3.test d9d9a04ee58c485b94694fd4f68cffaba49c32234fdefe1ac1a622c5e17d4ce3 @@ -1779,6 +1829,7 @@ F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test d16e49e89ada6137d1447777ef2a74574526a3b024e6733bf53ae2960da8c17c F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 +F test/vt02.c 33ecddc0832d4cd24e9e9fa83d868981b1e049462f4ec9080710353f6479a534 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1854,7 +1905,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec -F test/wapptest.tcl e3b6d5afa021c39a0f459ea9fbd1077459c1d81fca98eb40af8404ad3fc2360f x +F test/wapptest.tcl 1bea58a6a8e68a73f542ee4fca28b771b84ed803bd0c9e385087070b3d747b3c x F test/where.test d13cd7c24e80009d2b54e2f7a8893c457afa49c64f99359c9eb3fe668ba1d9d4 F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753 @@ -1881,6 +1932,7 @@ F test/wherefault.test 6cf2a9c5712952d463d3f45ebee7f6caf400984df51a195d884cfb7eb F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3 F test/wherelimit.test afb46397c6d7e964e6e294ba3569864a0c570fe3807afc634236c2b752372f31 F test/wherelimit2.test 657a3f24aadee62d058c5091ea682dc4af4b95ffe32f137155be49799a58e721 +F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 @@ -1903,6 +1955,7 @@ F test/windowA.test 6d63dc1260daa17141a55007600581778523a8b420629f1282d2acfc36af F test/windowB.test f2fb42b864b0cf431c956407583e9478a74c3642bdf8737fdcb6ff4a40298b07 F test/windowC.test 6fd75f5bb2f1343d34e470e36e68f0ff638d8a42f6aa7d99471261b31a0d42f2 F test/windowD.test 65cf5a765fb8072450e8a0de2979ce7f09a38d87724fe1280c6444073e3da49b +F test/windowE.test 6ba0c8048e4cc02b942e56640f8fcd50fd7ca72c876656c40f6baf42e316684c F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387019e0 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c @@ -1930,9 +1983,9 @@ F test/zipfile.test 0d8758d8c0d63f16644f959689f78969d223789d998964276554039f067b F test/zipfile2.test 9903388a602a3834189857a985106ff95c3bba6a3969e0134127df991889db5d F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44 -F tool/GetTclKit.bat e95747c0f7a9fe279a9979178b71f6431a21f945b390fc3120244897ff3f5135 +F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x +F tool/build-all-msvc.bat c817b716e0edeecaf265a6775b63e5f45c34a6544f1d4114a222701ed5ac79ab x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 @@ -1959,7 +2012,7 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x -F tool/mkctimec.tcl 8f472681f6041cf0ec26a1799feda39e55370614a6090d66fa5044f9f52d22c6 x +F tool/mkctimec.tcl c185cf1bdcd3d9bd3c06f77a2fd2df8a4a0d07266f992ecda75286965ba3574c x F tool/mkkeywordhash.c 35bfc41adacc4aa6ef6fca7fd0c63e0ec0534b78daf4d0cfdebe398216bbffc3 F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a895ab33 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef @@ -1970,7 +2023,7 @@ F tool/mkshellc.tcl df5d249617f9cc94d5c48eb0401673eb3f31f383ecbc54e8a13ca3dd97e8 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f -F tool/mksqlite3c.tcl eee7e9d9c58abb1045f6ed74ad95ad26e8d22be29fdc431deea5267fb3fa049c +F tool/mksqlite3c.tcl 4fc26a9bfa0c4505b203d7ca0cf086e75ebcd4d63eb719c82f37e3fecdf23d37 F tool/mksqlite3h.tcl 1f5e4a1dbbbc43c83cc6e74fe32c6c620502240b66c7c0f33a51378e78fc4edf F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 @@ -2004,6 +2057,7 @@ F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaa F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 +F tool/stripccomments.c 20b8aabc4694d0d4af5566e42da1f1a03aff057689370326e9269a9ddcffdc37 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh 1612bd947750e21e7b47befad5f6b3825b06cce0705441f903bf35ced65ae9b9 F tool/tserver.c 17b7f0b06f4e776e26220889941a86936b3c56ad18608baadc8faa00b7bd46ee @@ -2035,8 +2089,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dde76e91d3cb5aa66fcfab91ab02adfcfcf1eafc1e7e9d3520f4f396d27655d7 c22c7c879846b1357c00741700b7f6d1111c4b94895b4e7f552c92299d35712e -R e466d0b3e01cc2259e6859e5a0f9267d +P 68a61513f9e064ed9e79638ec40f8bff1a8ee678793683e2725a4ce63563db6a ca63a1bee16d30677c20c7576361dfb9a359e6e1b2b2b58a574da0059d3a8822 +R 900ac2cb8b24aa88575d880a45e5a4fa U drh -Z caa9cc539d5620a96e294ec646f6ad74 +Z 0745285bfbc3b753b7aba0d7f64f67d0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fe26a19e6d..537de37107 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68a61513f9e064ed9e79638ec40f8bff1a8ee678793683e2725a4ce63563db6a \ No newline at end of file +aa2e247b58dea930c4f0af3566d287f196ea178be8e994fc9eec685a89bebac8 \ No newline at end of file diff --git a/config.h.in b/sqlite_cfg.h.in similarity index 82% rename from config.h.in rename to sqlite_cfg.h.in index f2ba7d4fc4..3adea09936 100644 --- a/config.h.in +++ b/sqlite_cfg.h.in @@ -1,4 +1,4 @@ -/* config.h.in. Generated from configure.ac by autoheader. */ +/* sqlite_cfg.h.in. Generated from configure.ac by autoheader. */ /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H @@ -45,16 +45,16 @@ /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H -/* Define to 1 if you have the pread() function. */ +/* Define to 1 if you have the `pread' function. */ #undef HAVE_PREAD -/* Define to 1 if you have the pread64() function. */ +/* Define to 1 if you have the `pread64' function. */ #undef HAVE_PREAD64 -/* Define to 1 if you have the pwrite() function. */ +/* Define to 1 if you have the `pwrite' function. */ #undef HAVE_PWRITE -/* Define to 1 if you have the pwrite64() function. */ +/* Define to 1 if you have the `pwrite64' function. */ #undef HAVE_PWRITE64 /* Define to 1 if you have the header file. */ @@ -63,7 +63,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_STDLIB_H -/* Define to 1 if you have the strchrnul() function */ +/* Define to 1 if you have the `strchrnul' function. */ #undef HAVE_STRCHRNUL /* Define to 1 if you have the header file. */ @@ -99,9 +99,12 @@ /* Define to 1 if you have the `usleep' function. */ #undef HAVE_USLEEP -/* Define to 1 if you have the utime() library function. */ +/* Define to 1 if you have the `utime' function. */ #undef HAVE_UTIME +/* Define to 1 if you have the header file. */ +#undef HAVE_ZLIB_H + /* Define to the sub-directory in which libtool stores uninstalled libraries. */ #undef LT_OBJDIR @@ -118,12 +121,20 @@ /* Define to the one symbol short name of this package. */ #undef PACKAGE_TARNAME +/* Define to the home page for this package. */ +#undef PACKAGE_URL + /* Define to the version of this package. */ #undef PACKAGE_VERSION /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS +/* Enable large inode numbers on Mac OS X 10.5. */ +#ifndef _DARWIN_USE_64_BIT_INODE +# define _DARWIN_USE_64_BIT_INODE 1 +#endif + /* Number of bits in a file offset, on hosts where this is settable. */ #undef _FILE_OFFSET_BITS diff --git a/src/analyze.c b/src/analyze.c index 39009899ab..8562b9d7f1 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -953,6 +953,7 @@ static void analyzeVdbeCommentIndexWithColumnName( if( NEVER(i==XN_ROWID) ){ VdbeComment((v,"%s.rowid",pIdx->zName)); }else if( i==XN_EXPR ){ + assert( pIdx->bHasExpr ); VdbeComment((v,"%s.expr(%d)",pIdx->zName, k)); }else{ VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName)); diff --git a/src/btree.c b/src/btree.c index b507a703ec..1f2c9a401d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1895,7 +1895,6 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; - testcase( pc+x>maxPC ); return &aData[pc]; }else if( x+pc > maxPC ){ /* This slot extends off the end of the usable part of the page */ @@ -4009,6 +4008,9 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ } } }else{ + if( pCell+4 > pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } if( get4byte(pCell)==iFrom ){ put4byte(pCell, iTo); break; @@ -6536,14 +6538,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - if( !pPage->isInit || sqlite3FaultSim(412) ){ - /* The only known way for this to happen is for there to be a - ** recursive SQL function that does a DELETE operation as part of a - ** SELECT which deletes content out from under an active cursor - ** in a corrupt database file where the table being DELETE-ed from - ** has pages in common with the table being queried. See TH3 - ** module cov1/btree78.test testcase 220 (2018-06-08) for an - ** example. */ + if( NEVER(!pPage->isInit) || sqlite3FaultSim(412) ){ return SQLITE_CORRUPT_BKPT; } @@ -9243,6 +9238,11 @@ static int balance(BtCursor *pCur){ }else{ break; } + }else if( sqlite3PagerPageRefcount(pPage->pDbPage)>1 ){ + /* The page being written is not a root page, and there is currently + ** more than one reference to it. This only happens if the page is one + ** of its own ancestor pages. Corruption. */ + rc = SQLITE_CORRUPT_BKPT; }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; diff --git a/src/build.c b/src/build.c index fd03af8d93..24c98ae691 100644 --- a/src/build.c +++ b/src/build.c @@ -453,7 +453,7 @@ Table *sqlite3LocateTable( /* If zName is the not the name of a table in the schema created using ** CREATE, then check to see if it is the name of an virtual table that ** can be an eponymous virtual table. */ - if( pParse->disableVtab==0 && db->init.busy==0 ){ + if( (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)==0 && db->init.busy==0 ){ Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName); if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ pMod = sqlite3PragmaVtabRegister(db, zName); @@ -466,7 +466,7 @@ Table *sqlite3LocateTable( #endif if( flags & LOCATE_NOERR ) return 0; pParse->checkSchema = 1; - }else if( IsVirtual(p) && pParse->disableVtab ){ + }else if( IsVirtual(p) && (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)!=0 ){ p = 0; } @@ -2276,7 +2276,8 @@ static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ /* Recompute the colNotIdxed field of the Index. ** ** colNotIdxed is a bitmask that has a 0 bit representing each indexed -** columns that are within the first 63 columns of the table. The +** columns that are within the first 63 columns of the table and a 1 for +** all other bits (all columns that are not in the index). The ** high-order bit of colNotIdxed is always 1. All unindexed columns ** of the table have a 1. ** @@ -2304,7 +2305,7 @@ static void recomputeColumnsNotIndexed(Index *pIdx){ } } pIdx->colNotIdxed = ~m; - assert( (pIdx->colNotIdxed>>63)==1 ); + assert( (pIdx->colNotIdxed>>63)==1 ); /* See note-20221022-a */ } /* @@ -4192,6 +4193,7 @@ void sqlite3CreateIndex( j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; + pIndex->bHasExpr = 1; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); @@ -4203,6 +4205,7 @@ void sqlite3CreateIndex( } if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ pIndex->bHasVCol = 1; + pIndex->bHasExpr = 1; } } pIndex->aiColumn[i] = (i16)j; diff --git a/src/ctime.c b/src/ctime.c index 7eb43f20a7..c853e5ba46 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -28,7 +28,7 @@ ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -#include "config.h" +#include "sqlite_cfg.h" #define SQLITECONFIG_H 1 #endif @@ -193,6 +193,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT "DISABLE_SKIPAHEAD_DISTINCT", #endif +#ifdef SQLITE_DQS + "DQS=" CTIMEOPT_VAL(SQLITE_DQS), +#endif #ifdef SQLITE_ENABLE_8_3_NAMES "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), #endif diff --git a/src/dbpage.c b/src/dbpage.c index 9b565177c5..458f2bd501 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -274,12 +274,18 @@ static int dbpageColumn( } case 1: { /* data */ DbPage *pDbPage = 0; - rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); - if( rc==SQLITE_OK ){ - sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, - SQLITE_TRANSIENT); + if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + /* The pending byte page. Assume it is zeroed out. Attempting to + ** request this page from the page is an SQLITE_CORRUPT error. */ + sqlite3_result_zeroblob(ctx, pCsr->szPage); + }else{ + rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); } - sqlite3PagerUnref(pDbPage); break; } default: { /* schema */ @@ -288,7 +294,7 @@ static int dbpageColumn( break; } } - return SQLITE_OK; + return rc; } static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ @@ -348,11 +354,12 @@ static int dbpageUpdate( pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pDbPage); - if( rc==SQLITE_OK ){ - memcpy(sqlite3PagerGetData(pDbPage), - sqlite3_value_blob(argv[3]), - szPage); + const void *pData = sqlite3_value_blob(argv[3]); + assert( pData!=0 || pTab->db->mallocFailed ); + if( pData + && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK + ){ + memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); } } sqlite3PagerUnref(pDbPage); diff --git a/src/expr.c b/src/expr.c index baa0fe6476..7a4e59f28d 100644 --- a/src/expr.c +++ b/src/expr.c @@ -55,9 +55,8 @@ char sqlite3ExprAffinity(const Expr *pExpr){ if( op==TK_REGISTER ) op = pExpr->op2; if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ assert( ExprUseYTab(pExpr) ); - if( pExpr->y.pTab ){ - return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); - } + assert( pExpr->y.pTab!=0 ); + return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); } if( op==TK_SELECT ){ assert( ExprUseXSelect(pExpr) ); @@ -175,17 +174,14 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + int j; assert( ExprUseYTab(p) ); - if( p->y.pTab!=0 ){ - /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = p->iColumn; - if( j>=0 ){ - const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - } - break; + assert( p->y.pTab!=0 ); + if( (j = p->iColumn)>=0 ){ + const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); } + break; } if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; @@ -3790,10 +3786,7 @@ void sqlite3ExprCodeGetColumnOfTable( ){ Column *pCol; assert( v!=0 ); - if( pTab==0 ){ - sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); - return; - } + assert( pTab!=0 ); if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); VdbeComment((v, "%s.rowid", pTab->zName)); @@ -4043,6 +4036,53 @@ static int exprCodeInlineFunction( return target; } +/* +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxExpr. +** If it is, then resolve the expression by reading from the index and +** return the register into which the value has been read. If pExpr is +** not an indexed expression, then return negative. +*/ +static SQLITE_NOINLINE int sqlite3IndexedExprLookup( + Parse *pParse, /* The parsing context */ + Expr *pExpr, /* The expression to potentially bypass */ + int target /* Where to store the result of the expression */ +){ + IndexedExpr *p; + Vdbe *v; + for(p=pParse->pIdxExpr; p; p=p->pIENext){ + int iDataCur = p->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( p->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue; + v = pParse->pVdbe; + assert( v!=0 ); + if( p->bMaybeNullRow ){ + /* If the index is on a NULL row due to an outer join, then we + ** cannot extract the value from the index. The value must be + ** computed using the original expression. */ + int addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + sqlite3VdbeGoto(v, 0); + p = pParse->pIdxExpr; + pParse->pIdxExpr = 0; + sqlite3ExprCode(pParse, pExpr, target); + pParse->pIdxExpr = p; + sqlite3VdbeJumpHere(v, addr+2); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + } + return target; + } + return -1; /* Not found */ +} + /* ** Generate code into the current Vdbe to evaluate the given @@ -4071,6 +4111,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ expr_code_doover: if( pExpr==0 ){ op = TK_NULL; + }else if( pParse->pIdxExpr!=0 + && !ExprHasProperty(pExpr, EP_Leaf) + && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 + ){ + return r1; }else{ assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); op = pExpr->op; @@ -4116,11 +4161,8 @@ expr_code_doover: int aff; iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); assert( ExprUseYTab(pExpr) ); - if( pExpr->y.pTab ){ - aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); - }else{ - aff = pExpr->affExpr; - } + assert( pExpr->y.pTab!=0 ); + aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); if( aff>SQLITE_AFF_BLOB ){ static const char zAff[] = "B\000C\000D\000E"; assert( SQLITE_AFF_BLOB=='A' ); @@ -4182,12 +4224,10 @@ expr_code_doover: } } assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, pExpr->iColumn, iTab, target, pExpr->op2); - if( pExpr->y.pTab==0 && pExpr->affExpr==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); - } return iReg; } case TK_INTEGER: { @@ -5241,6 +5281,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -5415,6 +5456,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -5568,7 +5610,13 @@ int sqlite3ExprCompare( if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ return 1; } - return 2; + if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN + && pB->iTable<0 && pA->iTable==iTab + ){ + /* fall through */ + }else{ + return 2; + } } assert( !ExprHasProperty(pA, EP_IntValue) ); assert( !ExprHasProperty(pB, EP_IntValue) ); @@ -5870,10 +5918,10 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); assert( pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); if( (pLeft->op==TK_COLUMN - && pLeft->y.pTab!=0 + && ALWAYS(pLeft->y.pTab!=0) && IsVirtual(pLeft->y.pTab)) || (pRight->op==TK_COLUMN - && pRight->y.pTab!=0 + && ALWAYS(pRight->y.pTab!=0) && IsVirtual(pRight->y.pTab)) ){ return WRC_Prune; diff --git a/src/func.c b/src/func.c index 2f46fb423d..4f04c29ffb 100644 --- a/src/func.c +++ b/src/func.c @@ -742,7 +742,7 @@ static int patternCompare( ** c but in the other case and search the input string for either ** c or cx. */ - if( c<=0x80 ){ + if( c<0x80 ){ char zStop[3]; int bMatch; if( noCase ){ @@ -825,7 +825,13 @@ static int patternCompare( ** non-zero if there is no match. */ int sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + if( zString==0 ){ + return zGlobPattern!=0; + }else if( zGlobPattern==0 ){ + return 1; + }else { + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + } } /* @@ -833,7 +839,13 @@ int sqlite3_strglob(const char *zGlobPattern, const char *zString){ ** a miss - like strcmp(). */ int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ - return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + if( zStr==0 ){ + return zPattern!=0; + }else if( zPattern==0 ){ + return 1; + }else{ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + } } /* diff --git a/src/global.c b/src/global.c index 9e3c07a515..799b1e47c4 100644 --- a/src/global.c +++ b/src/global.c @@ -375,10 +375,6 @@ const char sqlite3StrBINARY[] = "BINARY"; ** ** sqlite3StdTypeAffinity[] The affinity associated with each entry ** in sqlite3StdType[]. -** -** sqlite3StdTypeMap[] The type value (as returned from -** sqlite3_column_type() or sqlite3_value_type()) -** for each entry in sqlite3StdType[]. */ const unsigned char sqlite3StdTypeLen[] = { 3, 4, 3, 7, 4, 4 }; const char sqlite3StdTypeAffinity[] = { @@ -389,14 +385,6 @@ const char sqlite3StdTypeAffinity[] = { SQLITE_AFF_REAL, SQLITE_AFF_TEXT }; -const char sqlite3StdTypeMap[] = { - 0, - SQLITE_BLOB, - SQLITE_INTEGER, - SQLITE_INTEGER, - SQLITE_FLOAT, - SQLITE_TEXT -}; const char *sqlite3StdType[] = { "ANY", "BLOB", diff --git a/src/insert.c b/src/insert.c index 6c71391a1b..1f6cdffcc3 100644 --- a/src/insert.c +++ b/src/insert.c @@ -96,6 +96,7 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ aff = SQLITE_AFF_INTEGER; }else{ assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); assert( pIdx->aColExpr!=0 ); aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); } @@ -109,6 +110,28 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ return pIdx->zColAff; } +/* +** Compute an affinity string for a table. Space is obtained +** from sqlite3DbMalloc(). The caller is responsible for freeing +** the space when done. +*/ +char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){ + char *zColAff; + zColAff = (char *)sqlite3DbMallocRaw(db, pTab->nCol+1); + if( zColAff ){ + int i, j; + for(i=j=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; + } + } + do{ + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); + } + return zColAff; +} + /* ** Make changes to the evolving bytecode to do affinity transformations ** of values that are about to be gathered into a row for table pTab. @@ -150,7 +173,7 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ ** Apply the type checking to that array of registers. */ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ - int i, j; + int i; char *zColAff; if( pTab->tabFlags & TF_Strict ){ if( iReg==0 ){ @@ -173,22 +196,11 @@ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } zColAff = pTab->zColAff; if( zColAff==0 ){ - sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + zColAff = sqlite3TableAffinityStr(0, pTab); if( !zColAff ){ - sqlite3OomFault(db); + sqlite3OomFault(sqlite3VdbeDb(v)); return; } - - for(i=j=0; inCol; i++){ - assert( pTab->aCol[i].affinity!=0 || sqlite3VdbeParser(v)->nErr>0 ); - if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ - zColAff[j++] = pTab->aCol[i].affinity; - } - } - do{ - zColAff[j--] = 0; - }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } assert( zColAff!=0 ); diff --git a/src/loadext.c b/src/loadext.c index dd15d9a4da..421ccca804 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -508,7 +508,9 @@ static const sqlite3_api_routines sqlite3Apis = { 0, 0, #endif - sqlite3_db_name + sqlite3_db_name, + /* Version 3.40.0 and later */ + sqlite3_value_type }; /* True if x is the directory separator character diff --git a/src/main.c b/src/main.c index 21f50472c4..a731a0ff3d 100644 --- a/src/main.c +++ b/src/main.c @@ -3352,6 +3352,19 @@ static int openDatabase( goto opendb_out; } +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) + /* Process magic filenames ":localStorage:" and ":sessionStorage:" */ + if( zFilename && zFilename[0]==':' ){ + if( strcmp(zFilename, ":localStorage:")==0 ){ + zFilename = "file:local?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + }else if( strcmp(zFilename, ":sessionStorage:")==0 ){ + zFilename = "file:session?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + } + } +#endif /* SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) */ + /* Parse the filename/URI argument ** ** Only allow sensible combinations of bits in the flags argument. @@ -3382,6 +3395,12 @@ static int openDatabase( sqlite3_free(zErrMsg); goto opendb_out; } + assert( db->pVfs!=0 ); +#if SQLITE_OS_KV || defined(SQLITE_OS_KV_OPTIONAL) + if( sqlite3_stricmp(db->pVfs->zName, "kvvfs")==0 ){ + db->temp_store = 2; + } +#endif /* Open the backend database driver */ rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, @@ -4491,7 +4510,7 @@ static char *appendText(char *p, const char *z){ ** Memory layout must be compatible with that generated by the pager ** and expected by sqlite3_uri_parameter() and databaseName(). */ -char *sqlite3_create_filename( +const char *sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, @@ -4527,10 +4546,10 @@ char *sqlite3_create_filename( ** error to call this routine with any parameter other than a pointer ** previously obtained from sqlite3_create_filename() or a NULL pointer. */ -void sqlite3_free_filename(char *p){ +void sqlite3_free_filename(const char *p){ if( p==0 ) return; - p = (char*)databaseName(p); - sqlite3_free(p - 4); + p = databaseName(p); + sqlite3_free((char*)p - 4); } @@ -4781,8 +4800,8 @@ int sqlite3_snapshot_open( */ int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ int rc = SQLITE_ERROR; - int iDb; #ifndef SQLITE_OMIT_WAL + int iDb; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ diff --git a/src/os.c b/src/os.c index 13c9abcab4..e2914e03c0 100644 --- a/src/os.c +++ b/src/os.c @@ -106,9 +106,11 @@ int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ } int sqlite3OsLock(sqlite3_file *id, int lockType){ DO_OS_MALLOC_TEST(id); + assert( lockType>=SQLITE_LOCK_SHARED && lockType<=SQLITE_LOCK_EXCLUSIVE ); return id->pMethods->xLock(id, lockType); } int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + assert( lockType==SQLITE_LOCK_NONE || lockType==SQLITE_LOCK_SHARED ); return id->pMethods->xUnlock(id, lockType); } int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ diff --git a/src/os_kv.c b/src/os_kv.c new file mode 100644 index 0000000000..322588be9a --- /dev/null +++ b/src/os_kv.c @@ -0,0 +1,972 @@ +/* +** 2022-09-06 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an experimental VFS layer that operates on a +** Key/Value storage engine where both keys and values must be pure +** text. +*/ +#include +#if SQLITE_OS_KV || (SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)) + +/***************************************************************************** +** Debugging logic +*/ + +/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ +#if 0 +#define SQLITE_KV_TRACE(X) printf X +#else +#define SQLITE_KV_TRACE(X) +#endif + +/* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */ +#if 0 +#define SQLITE_KV_LOG(X) printf X +#else +#define SQLITE_KV_LOG(X) +#endif + + +/* +** Forward declaration of objects used by this VFS implementation +*/ +typedef struct KVVfsFile KVVfsFile; + +/* A single open file. There are only two files represented by this +** VFS - the database and the rollback journal. +*/ +struct KVVfsFile { + sqlite3_file base; /* IO methods */ + const char *zClass; /* Storage class */ + int isJournal; /* True if this is a journal file */ + unsigned int nJrnl; /* Space allocated for aJrnl[] */ + char *aJrnl; /* Journal content */ + int szPage; /* Last known page size */ + sqlite3_int64 szDb; /* Database file size. -1 means unknown */ +}; + +/* +** Methods for KVVfsFile +*/ +static int kvvfsClose(sqlite3_file*); +static int kvvfsReadDb(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsReadJrnl(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsWriteDb(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsWriteJrnl(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsTruncateDb(sqlite3_file*, sqlite3_int64 size); +static int kvvfsTruncateJrnl(sqlite3_file*, sqlite3_int64 size); +static int kvvfsSyncDb(sqlite3_file*, int flags); +static int kvvfsSyncJrnl(sqlite3_file*, int flags); +static int kvvfsFileSizeDb(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsFileSizeJrnl(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsLock(sqlite3_file*, int); +static int kvvfsUnlock(sqlite3_file*, int); +static int kvvfsCheckReservedLock(sqlite3_file*, int *pResOut); +static int kvvfsFileControlDb(sqlite3_file*, int op, void *pArg); +static int kvvfsFileControlJrnl(sqlite3_file*, int op, void *pArg); +static int kvvfsSectorSize(sqlite3_file*); +static int kvvfsDeviceCharacteristics(sqlite3_file*); + +/* +** Methods for sqlite3_vfs +*/ +static int kvvfsOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int kvvfsDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int kvvfsAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int kvvfsFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *kvvfsDlOpen(sqlite3_vfs*, const char *zFilename); +static int kvvfsRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int kvvfsSleep(sqlite3_vfs*, int microseconds); +static int kvvfsCurrentTime(sqlite3_vfs*, double*); +static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs sqlite3OsKvvfsObject = { + 1, /* iVersion */ + sizeof(KVVfsFile), /* szOsFile */ + 1024, /* mxPathname */ + 0, /* pNext */ + "kvvfs", /* zName */ + 0, /* pAppData */ + kvvfsOpen, /* xOpen */ + kvvfsDelete, /* xDelete */ + kvvfsAccess, /* xAccess */ + kvvfsFullPathname, /* xFullPathname */ + kvvfsDlOpen, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ + kvvfsRandomness, /* xRandomness */ + kvvfsSleep, /* xSleep */ + kvvfsCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + kvvfsCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +/* Methods for sqlite3_file objects referencing a database file +*/ +static sqlite3_io_methods kvvfs_db_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadDb, /* xRead */ + kvvfsWriteDb, /* xWrite */ + kvvfsTruncateDb, /* xTruncate */ + kvvfsSyncDb, /* xSync */ + kvvfsFileSizeDb, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlDb, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/* Methods for sqlite3_file objects referencing a rollback journal +*/ +static sqlite3_io_methods kvvfs_jrnl_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadJrnl, /* xRead */ + kvvfsWriteJrnl, /* xWrite */ + kvvfsTruncateJrnl, /* xTruncate */ + kvvfsSyncJrnl, /* xSync */ + kvvfsFileSizeJrnl, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlJrnl, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/****** Storage subsystem **************************************************/ +#include +#include +#include + +/* Forward declarations for the low-level storage engine +*/ +static int kvstorageWrite(const char*, const char *zKey, const char *zData); +static int kvstorageDelete(const char*, const char *zKey); +static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); +#define KVSTORAGE_KEY_SZ 32 + +/* Expand the key name with an appropriate prefix and put the result +** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least +** KVSTORAGE_KEY_SZ bytes. +*/ +static void kvstorageMakeKey( + const char *zClass, + const char *zKeyIn, + char *zKeyOut +){ + sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); +} + +/* Write content into a key. zClass is the particular namespace of the +** underlying key/value store to use - either "local" or "session". +** +** Both zKey and zData are zero-terminated pure text strings. +** +** Return the number of errors. +*/ +static int kvstorageWrite( + const char *zClass, + const char *zKey, + const char *zData +){ + FILE *fd; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + fd = fopen(zXKey, "wb"); + if( fd ){ + SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, + (int)strlen(zData), zData, + strlen(zData)>50 ? "..." : "")); + fputs(zData, fd); + fclose(fd); + return 0; + }else{ + return 1; + } +} + +/* Delete a key (with its corresponding data) from the key/value +** namespace given by zClass. If the key does not previously exist, +** this routine is a no-op. +*/ +static int kvstorageDelete(const char *zClass, const char *zKey){ + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + unlink(zXKey); + SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); + return 0; +} + +/* Read the value associated with a zKey from the key/value namespace given +** by zClass and put the text data associated with that key in the first +** nBuf bytes of zBuf[]. The value might be truncated if zBuf is not large +** enough to hold it all. The value put into zBuf must always be zero +** terminated, even if it gets truncated because nBuf is not large enough. +** +** Return the total number of bytes in the data, without truncation, and +** not counting the final zero terminator. Return -1 if the key does +** not exist. +** +** If nBuf<=0 then this routine simply returns the size of the data without +** actually reading it. +*/ +static int kvstorageRead( + const char *zClass, + const char *zKey, + char *zBuf, + int nBuf +){ + FILE *fd; + struct stat buf; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + if( access(zXKey, R_OK)!=0 + || stat(zXKey, &buf)!=0 + || !S_ISREG(buf.st_mode) + ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + } + if( nBuf<=0 ){ + return (int)buf.st_size; + }else if( nBuf==1 ){ + zBuf[0] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%d)\n", zXKey, + (int)buf.st_size)); + return (int)buf.st_size; + } + if( nBuf > buf.st_size + 1 ){ + nBuf = buf.st_size + 1; + } + fd = fopen(zXKey, "rb"); + if( fd==0 ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + }else{ + sqlite3_int64 n = fread(zBuf, 1, nBuf-1, fd); + fclose(fd); + zBuf[n] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%lld) %.50s%s\n", zXKey, + n, zBuf, n>50 ? "..." : "")); + return (int)n; + } +} + +/* +** An internal level of indirection which enables us to replace the +** kvvfs i/o methods with JavaScript implementations in WASM builds. +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3_wasm_enum_json(). There are no binary compatibility +** concerns, so it does not need an iVersion member. This file is +** necessarily always compiled together with sqlite3_wasm_enum_json(), +** and JS code dynamically creates the mapping of members based on +** that JSON description. +*/ +typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; +struct sqlite3_kvvfs_methods { + int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); + int (*xWrite)(const char *zClass, const char *zKey, const char *zData); + int (*xDelete)(const char *zClass, const char *zKey); + const int nKeySize; +}; + +/* +** This object holds the kvvfs I/O methods which may be swapped out +** for JavaScript-side implementations in WASM builds. In such builds +** it cannot be const, but in native builds it should be so that +** the compiler can hopefully optimize this level of indirection out. +** That said, kvvfs is intended primarily for use in WASM builds. +** +** Note that this is not explicitly flagged as static because the +** amalgamation build will tag it with SQLITE_PRIVATE. +*/ +#ifndef SQLITE_WASM +const +#endif +sqlite3_kvvfs_methods sqlite3KvvfsMethods = { +kvstorageRead, +kvstorageWrite, +kvstorageDelete, +KVSTORAGE_KEY_SZ +}; + +/****** Utility subroutines ************************************************/ + +/* +** Encode binary into the text encoded used to persist on disk. +** The output text is stored in aOut[], which must be at least +** nData+1 bytes in length. +** +** Return the actual length of the encoded text, not counting the +** zero terminator at the end. +** +** Encoding format +** --------------- +** +** * Non-zero bytes are encoded as upper-case hexadecimal +** +** * A sequence of one or more zero-bytes that are not at the +** beginning of the buffer are encoded as a little-endian +** base-26 number using a..z. "a" means 0. "b" means 1, +** "z" means 25. "ab" means 26. "ac" means 52. And so forth. +** +** * Because there is no overlap between the encoding characters +** of hexadecimal and base-26 numbers, it is always clear where +** one stops and the next begins. +*/ +static int kvvfsEncode(const char *aData, int nData, char *aOut){ + int i, j; + const unsigned char *a = (const unsigned char*)aData; + for(i=j=0; i>4]; + aOut[j++] = "0123456789ABCDEF"[c&0xf]; + }else{ + /* A sequence of 1 or more zeros is stored as a little-endian + ** base-26 number using a..z as the digits. So one zero is "b". + ** Two zeros is "c". 25 zeros is "z", 26 zeros is "ab", 27 is "bb", + ** and so forth. + */ + int k; + for(k=1; i+k0 ){ + aOut[j++] = 'a'+(k%26); + k /= 26; + } + } + } + aOut[j] = 0; + return j; +} + +static const signed char kvvfsHexValue[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* +** Decode the text encoding back to binary. The binary content is +** written into pOut, which must be at least nOut bytes in length. +** +** The return value is the number of bytes actually written into aOut[]. +*/ +static int kvvfsDecode(const char *a, char *aOut, int nOut){ + int i, j; + int c; + const unsigned char *aIn = (const unsigned char*)a; + i = 0; + j = 0; + while( 1 ){ + c = kvvfsHexValue[aIn[i]]; + if( c<0 ){ + int n = 0; + int mult = 1; + c = aIn[i]; + if( c==0 ) break; + while( c>='a' && c<='z' ){ + n += (c - 'a')*mult; + mult *= 26; + c = aIn[++i]; + } + if( j+n>nOut ) return -1; + memset(&aOut[j], 0, n); + j += n; + c = aIn[i]; + if( c==0 ) break; + }else{ + aOut[j] = c<<4; + c = kvvfsHexValue[aIn[++i]]; + if( c<0 ) break; + aOut[j++] += c; + i++; + } + } + return j; +} + +/* +** Decode a complete journal file. Allocate space in pFile->aJrnl +** and store the decoding there. Or leave pFile->aJrnl set to NULL +** if an error is encountered. +** +** The first few characters of the text encoding will be a little-endian +** base-26 number (digits a..z) that is the total number of bytes +** in the decoded journal file image. This base-26 number is followed +** by a single space, then the encoding of the journal. The space +** separator is required to act as a terminator for the base-26 number. +*/ +static void kvvfsDecodeJournal( + KVVfsFile *pFile, /* Store decoding in pFile->aJrnl */ + const char *zTxt, /* Text encoding. Zero-terminated */ + int nTxt /* Bytes in zTxt, excluding zero terminator */ +){ + unsigned int n = 0; + int c, i, mult; + i = 0; + mult = 1; + while( (c = zTxt[i++])>='a' && c<='z' ){ + n += (zTxt[i] - 'a')*mult; + mult *= 26; + } + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = sqlite3_malloc64( n ); + if( pFile->aJrnl==0 ){ + pFile->nJrnl = 0; + return; + } + pFile->nJrnl = n; + n = kvvfsDecode(zTxt+i, pFile->aJrnl, pFile->nJrnl); + if( nnJrnl ){ + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + } +} + +/* +** Read or write the "sz" element, containing the database file size. +*/ +static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ + char zData[50]; + zData[0] = 0; + sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + return strtoll(zData, 0, 0); +} +static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ + char zData[50]; + sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); + return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); +} + +/****** sqlite3_io_methods methods ******************************************/ + +/* +** Close an kvvfs-file. +*/ +static int kvvfsClose(sqlite3_file *pProtoFile){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + pFile->isJournal ? "journal" : "db")); + sqlite3_free(pFile->aJrnl); + return SQLITE_OK; +} + +/* +** Read from the -journal file. +*/ +static int kvvfsReadJrnl( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + assert( pFile->isJournal ); + SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( pFile->aJrnl==0 ){ + int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + char *aTxt; + if( szTxt<=4 ){ + return SQLITE_IOERR; + } + aTxt = sqlite3_malloc64( szTxt+1 ); + if( aTxt==0 ) return SQLITE_NOMEM; + kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); + kvvfsDecodeJournal(pFile, aTxt, szTxt); + sqlite3_free(aTxt); + if( pFile->aJrnl==0 ) return SQLITE_IOERR; + } + if( iOfst+iAmt>pFile->nJrnl ){ + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, pFile->aJrnl+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Read from the database file. +*/ +static int kvvfsReadDb( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + int got, n; + char zKey[30]; + char aData[133073]; + assert( iOfst>=0 ); + assert( iAmt>=0 ); + SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( iOfst+iAmt>=512 ){ + if( (iOfst % iAmt)!=0 ){ + return SQLITE_IOERR_READ; + } + if( (iAmt & (iAmt-1))!=0 || iAmt<512 || iAmt>65536 ){ + return SQLITE_IOERR_READ; + } + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + }else{ + pgno = 1; + } + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, aData, sizeof(aData)-1); + if( got<0 ){ + n = 0; + }else{ + aData[got] = 0; + if( iOfst+iAmt<512 ){ + int k = iOfst+iAmt; + aData[k*2] = 0; + n = kvvfsDecode(aData, &aData[2000], sizeof(aData)-2000); + if( n>=iOfst+iAmt ){ + memcpy(zBuf, &aData[2000+iOfst], iAmt); + n = iAmt; + }else{ + n = 0; + } + }else{ + n = kvvfsDecode(aData, zBuf, iAmt); + } + } + if( nzClass, iAmt, iOfst)); + if( iEnd>=0x10000000 ) return SQLITE_FULL; + if( pFile->aJrnl==0 || pFile->nJrnlaJrnl, iEnd); + if( aNew==0 ){ + return SQLITE_IOERR_NOMEM; + } + pFile->aJrnl = aNew; + if( pFile->nJrnlaJrnl+pFile->nJrnl, 0, iOfst-pFile->nJrnl); + } + pFile->nJrnl = iEnd; + } + memcpy(pFile->aJrnl+iOfst, zBuf, iAmt); + return SQLITE_OK; +} + +/* +** Write into the database file. +*/ +static int kvvfsWriteDb( + sqlite3_file *pProtoFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + char zKey[30]; + char aData[131073]; + SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + assert( iAmt>=512 && iAmt<=65536 ); + assert( (iAmt & (iAmt-1))==0 ); + assert( pFile->szPage<0 || pFile->szPage==iAmt ); + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + kvvfsEncode(zBuf, iAmt, aData); + if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ + return SQLITE_IOERR; + } + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } + return SQLITE_OK; +} + +/* +** Truncate an kvvfs-file. +*/ +static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); + assert( size==0 ); + sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + return SQLITE_OK; +} +static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + if( pFile->szDb>size + && pFile->szPage>0 + && (size % pFile->szPage)==0 + ){ + char zKey[50]; + unsigned int pgno, pgnoMax; + SQLITE_KV_LOG(("xTruncate('%s-db',%lld)\n", pFile->zClass, size)); + pgno = 1 + size/pFile->szPage; + pgnoMax = 2 + pFile->szDb/pFile->szPage; + while( pgno<=pgnoMax ){ + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); + pgno++; + } + pFile->szDb = size; + return kvvfsWriteFileSize(pFile, size) ? SQLITE_IOERR : SQLITE_OK; + } + return SQLITE_IOERR; +} + +/* +** Sync an kvvfs-file. +*/ +static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ + int i, n; + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + char *zOut; + SQLITE_KV_LOG(("xSync('%s-journal')\n", pFile->zClass)); + if( pFile->nJrnl<=0 ){ + return kvvfsTruncateJrnl(pProtoFile, 0); + } + zOut = sqlite3_malloc64( pFile->nJrnl*2 + 50 ); + if( zOut==0 ){ + return SQLITE_IOERR_NOMEM; + } + n = pFile->nJrnl; + i = 0; + do{ + zOut[i++] = 'a' + (n%26); + n /= 26; + }while( n>0 ); + zOut[i++] = ' '; + kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); + i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); + sqlite3_free(zOut); + return i ? SQLITE_IOERR : SQLITE_OK; +} +static int kvvfsSyncDb(sqlite3_file *pProtoFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an kvvfs-file. +*/ +static int kvvfsFileSizeJrnl(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-journal')\n", pFile->zClass)); + *pSize = pFile->nJrnl; + return SQLITE_OK; +} +static int kvvfsFileSizeDb(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-db')\n", pFile->zClass)); + if( pFile->szDb>=0 ){ + *pSize = pFile->szDb; + }else{ + *pSize = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Lock an kvvfs-file. +*/ +static int kvvfsLock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xLock(%s,%d)\n", pFile->zClass, eLock)); + + if( eLock!=SQLITE_LOCK_NONE ){ + pFile->szDb = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Unlock an kvvfs-file. +*/ +static int kvvfsUnlock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xUnlock(%s,%d)\n", pFile->zClass, eLock)); + if( eLock==SQLITE_LOCK_NONE ){ + pFile->szDb = -1; + } + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an kvvfs-file. +*/ +static int kvvfsCheckReservedLock(sqlite3_file *pProtoFile, int *pResOut){ + SQLITE_KV_LOG(("xCheckReservedLock\n")); + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an kvvfs-file. +*/ +static int kvvfsFileControlJrnl(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on journal\n", op)); + return SQLITE_NOTFOUND; +} +static int kvvfsFileControlDb(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on database\n", op)); + if( op==SQLITE_FCNTL_SYNC ){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + int rc = SQLITE_OK; + SQLITE_KV_LOG(("xSync('%s-db')\n", pFile->zClass)); + if( pFile->szDb>0 && 0!=kvvfsWriteFileSize(pFile, pFile->szDb) ){ + rc = SQLITE_IOERR; + } + return rc; + } + return SQLITE_NOTFOUND; +} + +/* +** Return the sector-size in bytes for an kvvfs-file. +*/ +static int kvvfsSectorSize(sqlite3_file *pFile){ + return 512; +} + +/* +** Return the device characteristic flags supported by an kvvfs-file. +*/ +static int kvvfsDeviceCharacteristics(sqlite3_file *pProtoFile){ + return 0; +} + +/****** sqlite3_vfs methods *************************************************/ + +/* +** Open an kvvfs file handle. +*/ +static int kvvfsOpen( + sqlite3_vfs *pProtoVfs, + const char *zName, + sqlite3_file *pProtoFile, + int flags, + int *pOutFlags +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + if( zName==0 ) zName = ""; + SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); + if( strcmp(zName, "local")==0 + || strcmp(zName, "session")==0 + ){ + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; + }else + if( strcmp(zName, "local-journal")==0 + || strcmp(zName, "session-journal")==0 + ){ + pFile->isJournal = 1; + pFile->base.pMethods = &kvvfs_jrnl_io_methods; + }else{ + return SQLITE_CANTOPEN; + } + if( zName[0]=='s' ){ + pFile->zClass = "session"; + }else{ + pFile->zClass = "local"; + } + pFile->aJrnl = 0; + pFile->nJrnl = 0; + pFile->szPage = -1; + pFile->szDb = -1; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + if( strcmp(zPath, "local-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("local", "jrnl"); + }else + if( strcmp(zPath, "session-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("session", "jrnl"); + } + return SQLITE_OK; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int kvvfsAccess( + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, + int *pResOut +){ + SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); + if( strcmp(zPath, "local-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "session-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "local")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; + }else + if( strcmp(zPath, "session")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; + }else + { + *pResOut = 0; + } + SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int kvvfsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + size_t nPath; +#ifdef SQLITE_OS_KV_ALWAYS_LOCAL + zPath = "local"; +#endif + nPath = strlen(zPath); + SQLITE_KV_LOG(("xFullPathname(\"%s\")\n", zPath)); + if( nOut +static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + struct timeval sNow; + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *pTimeOut = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */ + +#if SQLITE_OS_KV +/* +** This routine is called initialize the KV-vfs as the default VFS. +*/ +int sqlite3_os_init(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 1); +} +int sqlite3_os_end(void){ + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV */ + +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +int sqlite3KvvfsInit(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 0); +} +#endif diff --git a/src/os_setup.h b/src/os_setup.h index 08aaa1195a..a82f86fd9f 100644 --- a/src/os_setup.h +++ b/src/os_setup.h @@ -20,38 +20,72 @@ ** Figure out if we are dealing with Unix, Windows, or some other operating ** system. ** -** After the following block of preprocess macros, all of SQLITE_OS_UNIX, -** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of -** the three will be 1. The other two will be 0. +** After the following block of preprocess macros, all of +** +** SQLITE_OS_KV +** SQLITE_OS_OTHER +** SQLITE_OS_UNIX +** SQLITE_OS_WIN +** +** will defined to either 1 or 0. One of them will be 1. The others will be 0. +** If none of the macros are initially defined, then select either +** SQLITE_OS_UNIX or SQLITE_OS_WIN depending on the target platform. +** +** If SQLITE_OS_OTHER=1 is specified at compile-time, then the application +** must provide its own VFS implementation together with sqlite3_os_init() +** and sqlite3_os_end() routines. */ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX +#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \ + !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN) +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 # define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 # else -# undef SQLITE_OS_OTHER +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 # endif #endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +#if SQLITE_OS_OTHER+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_KV+1>1 +# undef SQLITE_OS_OTHER # define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ - defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# define SQLITE_OMIT_LOAD_EXTENSION 1 +# define SQLITE_OMIT_WAL 1 +# define SQLITE_OMIT_DEPRECATED 1 +# undef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 3 /* Always use memory for temporary storage */ +# define SQLITE_DQS 0 +# define SQLITE_OMIT_SHARED_CACHE 1 +# define SQLITE_OMIT_AUTOINIT 1 +#endif +#if SQLITE_OS_UNIX+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_WIN+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 #endif + #endif /* SQLITE_OS_SETUP_H */ diff --git a/src/os_unix.c b/src/os_unix.c index 27d9b2a200..09eebd504c 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -97,13 +97,13 @@ /* ** standard include files. */ -#include -#include +#include /* amalgamator: keep */ +#include /* amalgamator: keep */ #include #include -#include +#include /* amalgamator: keep */ #include -#include +#include /* amalgamator: keep */ #include #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 # include @@ -8251,6 +8251,9 @@ int sqlite3_os_init(void){ sqlite3_vfs_register(&aVfs[i], i==0); #endif } +#ifdef SQLITE_OS_KV_OPTIONAL + sqlite3KvvfsInit(); +#endif unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); #ifndef SQLITE_OMIT_WAL diff --git a/src/pragma.c b/src/pragma.c index 7de9cfb4f9..0875ed03e3 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1753,8 +1753,9 @@ void sqlite3Pragma( int loopTop; int iDataCur, iIdxCur; int r1 = -1; - int bStrict; + int bStrict; /* True for a STRICT table */ int r2; /* Previous key for WITHOUT ROWID tables */ + int mxCol; /* Maximum non-virtual column number */ if( !IsOrdinaryTable(pTab) ) continue; if( pObjTab && pObjTab!=pTab ) continue; @@ -1779,11 +1780,22 @@ void sqlite3Pragma( assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + + /* Fetch the right-most column from the table. This will cause + ** the entire record header to be parsed and sanity checked. It + ** will also prepopulate the cursor column cache that is used + ** by the OP_IsType code, so it is a required step. + */ + mxCol = pTab->nCol-1; + while( mxCol>=0 + && ((pTab->aCol[mxCol].colFlags & COLFLAG_VIRTUAL)!=0 + || pTab->iPKey==mxCol) ) mxCol--; + if( mxCol>=0 ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, mxCol, 3); + sqlite3VdbeTypeofColumn(v, 3); + } + if( !isQuick ){ - /* Sanity check on record header decoding */ - sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); - VdbeComment((v, "(right-most column)")); if( pPk ){ /* Verify WITHOUT ROWID keys are in ascending order */ int a1; @@ -1803,44 +1815,122 @@ void sqlite3Pragma( } } } - /* Verify that all NOT NULL columns really are NOT NULL. At the - ** same time verify the type of the content of STRICT tables */ + /* Verify datatypes for all columns: + ** + ** (1) NOT NULL columns may not contain a NULL + ** (2) Datatype must be exact for non-ANY columns in STRICT tables + ** (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. + ** (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be losslessly converted to numeric. + */ bStrict = (pTab->tabFlags & TF_Strict)!=0; for(j=0; jnCol; j++){ char *zErr; - Column *pCol = pTab->aCol + j; - int doError, jmp2; + Column *pCol = pTab->aCol + j; /* The column to be checked */ + int labelError; /* Jump here to report an error */ + int labelOk; /* Jump here if all looks ok */ + int p1, p3, p4; /* Operands to the OP_IsType opcode */ + int doTypeCheck; /* Check datatypes (besides NOT NULL) */ + if( j==pTab->iPKey ) continue; - if( pCol->notNull==0 && !bStrict ) continue; - doError = bStrict ? sqlite3VdbeMakeLabel(pParse) : 0; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); - if( sqlite3VdbeGetLastOp(v)->opcode==OP_Column ){ - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + if( bStrict ){ + doTypeCheck = pCol->eCType>COLTYPE_ANY; + }else{ + doTypeCheck = pCol->affinity>SQLITE_AFF_BLOB; } + if( pCol->notNull==0 && !doTypeCheck ) continue; + + /* Compute the operands that will be needed for OP_IsType */ + p4 = SQLITE_NULL; + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + p1 = -1; + p3 = 3; + }else{ + if( pCol->iDflt ){ + sqlite3_value *pDfltValue = 0; + sqlite3ValueFromExpr(db, sqlite3ColumnExpr(pTab,pCol), ENC(db), + pCol->affinity, &pDfltValue); + if( pDfltValue ){ + p4 = sqlite3_value_type(pDfltValue); + sqlite3ValueFree(pDfltValue); + } + } + p1 = iDataCur; + if( !HasRowid(pTab) ){ + testcase( j!=sqlite3TableColumnToStorage(pTab, j) ); + p3 = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), j); + }else{ + p3 = sqlite3TableColumnToStorage(pTab,j); + testcase( p3!=j); + } + } + + labelError = sqlite3VdbeMakeLabel(pParse); + labelOk = sqlite3VdbeMakeLabel(pParse); if( pCol->notNull ){ - jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + /* (1) NOT NULL columns may not contain a NULL */ + int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x0f); + VdbeCoverage(v); zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pCol->zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - if( bStrict && pCol->eCType!=COLTYPE_ANY ){ - sqlite3VdbeGoto(v, doError); + if( doTypeCheck ){ + sqlite3VdbeGoto(v, labelError); + sqlite3VdbeJumpHere(v, jmp2); }else{ - integrityCheckResultRow(v); + /* VDBE byte code will fall thru */ } - sqlite3VdbeJumpHere(v, jmp2); } - if( bStrict && pCol->eCType!=COLTYPE_ANY ){ - jmp2 = sqlite3VdbeAddOp3(v, OP_IsNullOrType, 3, 0, - sqlite3StdTypeMap[pCol->eCType-1]); + if( bStrict && doTypeCheck ){ + /* (2) Datatype must be exact for non-ANY columns in STRICT tables*/ + static unsigned char aStdTypeMask[] = { + 0x1f, /* ANY */ + 0x18, /* BLOB */ + 0x11, /* INT */ + 0x11, /* INTEGER */ + 0x13, /* REAL */ + 0x14 /* TEXT */ + }; + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + assert( pCol->eCType>=1 && pCol->eCType<=sizeof(aStdTypeMask) ); + sqlite3VdbeChangeP5(v, aStdTypeMask[pCol->eCType-1]); VdbeCoverage(v); zErr = sqlite3MPrintf(db, "non-%s value in %s.%s", sqlite3StdType[pCol->eCType-1], pTab->zName, pTab->aCol[j].zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeResolveLabel(v, doError); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, jmp2); + }else if( !bStrict && pCol->affinity==SQLITE_AFF_TEXT ){ + /* (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "NUMERIC value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + }else if( !bStrict && pCol->affinity>=SQLITE_AFF_NUMERIC ){ + /* (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be converted to numeric. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1b); /* NULL, INT, FLOAT, or BLOB */ + VdbeCoverage(v); + if( p1>=0 ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + } + sqlite3VdbeAddOp4(v, OP_Affinity, 3, 1, 0, "C", P4_STATIC); + sqlite3VdbeAddOp4Int(v, OP_IsType, -1, labelOk, 3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "TEXT value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); } + sqlite3VdbeResolveLabel(v, labelError); + integrityCheckResultRow(v); + sqlite3VdbeResolveLabel(v, labelOk); } /* Verify CHECK constraints */ if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ diff --git a/src/prepare.c b/src/prepare.c index f66c366b99..1e7a1222ba 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -705,7 +705,7 @@ static int sqlite3Prepare( sParse.disableLookaside++; DisableLookaside; } - sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0; + sParse.prepFlags = prepFlags & 0xff; /* Check to verify that it is possible to get a read lock on all ** database schemas. The inability to get a read lock indicates that @@ -746,7 +746,9 @@ static int sqlite3Prepare( } } - sqlite3VtabUnlockList(db); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( db->pDisconnect ) sqlite3VtabUnlockList(db); +#endif if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; diff --git a/src/select.c b/src/select.c index 1cba85d47c..ea6a3067af 100644 --- a/src/select.c +++ b/src/select.c @@ -1287,6 +1287,9 @@ static void selectInnerLoop( testcase( eDest==SRT_Fifo ); testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); + if( pDest->zAffSdst ){ + sqlite3VdbeChangeP4(v, -1, pDest->zAffSdst, nResultCol); + } #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -3750,6 +3753,7 @@ typedef struct SubstContext { int iNewTable; /* New table number */ int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ ExprList *pEList; /* Replacement expressions */ + ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; /* Forward Declarations */ @@ -3791,9 +3795,10 @@ static Expr *substExpr( #endif { Expr *pNew; - Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; + int iColumn = pExpr->iColumn; + Expr *pCopy = pSubst->pEList->a[iColumn].pExpr; Expr ifNullRow; - assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); + assert( pSubst->pEList!=0 && iColumnpEList->nExpr ); assert( pExpr->pRight==0 ); if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); @@ -3831,11 +3836,16 @@ static Expr *substExpr( /* Ensure that the expression now has an implicit collation sequence, ** just as it did when it was a column of a view or sub-query. */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ - CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); - pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, - (pColl ? pColl->zName : "BINARY") + { + CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, + pSubst->pCList->a[iColumn].pExpr ); + if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){ + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } } ExprClearProperty(pExpr, EP_Collate); } @@ -4028,6 +4038,18 @@ static void renumberCursors( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** If pSel is not part of a compound SELECT, return a pointer to its +** expression list. Otherwise, return a pointer to the expression list +** of the leftmost SELECT in the compound. +*/ +static ExprList *findLeftmostExprlist(Select *pSel){ + while( pSel->pPrior ){ + pSel = pSel->pPrior; + } + return pSel->pEList; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -4130,6 +4152,8 @@ static void renumberCursors( ** (17g) either the subquery is the first element of the outer ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) +** (17h) The corresponding result set expressions in all arms of the +** compound must have the same affinity. ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -4306,6 +4330,7 @@ static int flattenSubquery( ** queries. */ if( pSub->pPrior ){ + int ii; if( pSub->pOrderBy ){ return 0; /* Restriction (20) */ } @@ -4338,7 +4363,6 @@ static int flattenSubquery( /* Restriction (18). */ if( p->pOrderBy ){ - int ii; for(ii=0; iipOrderBy->nExpr; ii++){ if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; } @@ -4347,6 +4371,21 @@ static int flattenSubquery( /* Restriction (23) */ if( (p->selFlags & SF_Recursive) ) return 0; + /* Restriction (17h) */ + for(ii=0; iipEList->nExpr; ii++){ + char aff; + assert( pSub->pEList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr); + for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 0; + } + } + } + if( pSrc->nSrc>1 ){ if( pParse->nSelect>500 ) return 0; if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0; @@ -4580,6 +4619,7 @@ static int flattenSubquery( x.iNewTable = iNewParent; x.isOuterJoin = isOuterJoin; x.pEList = pSub->pEList; + x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); } @@ -4599,7 +4639,7 @@ static int flattenSubquery( pSub->pLimit = 0; } - /* Recompute the SrcList_item.colUsed masks for the flattened + /* Recompute the SrcItem.colUsed masks for the flattened ** tables. */ for(i=0; ia[i+iFrom]); @@ -4989,6 +5029,13 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** be materialized. (This restriction is implemented in the calling ** routine.) ** +** (8) The subquery may not be a compound that uses UNION, INTERSECT, +** or EXCEPT. (We could, perhaps, relax this restriction to allow +** this case if none of the comparisons operators between left and +** right arms of the compound use a collation other than BINARY. +** But it is a lot of work to check that case for an obscure and +** minor optimization, so we omit it for now.) +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -5008,6 +5055,10 @@ static int pushDownWhereTerms( if( pSubq->pPrior ){ Select *pSel; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + u8 op = pSel->op; + assert( op==TK_ALL || op==TK_SELECT + || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); + if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ if( pSel->pWin ) return 0; /* restriction (6b) */ } }else{ @@ -5062,6 +5113,7 @@ static int pushDownWhereTerms( x.iNewTable = pSrc->iCursor; x.isOuterJoin = 0; x.pEList = pSubq->pEList; + x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ @@ -5586,9 +5638,9 @@ void sqlite3SelectPopWith(Walker *pWalker, Select *p){ #endif /* -** The SrcList_item structure passed as the second argument represents a +** The SrcItem structure passed as the second argument represents a ** sub-query in the FROM clause of a SELECT statement. This function -** allocates and populates the SrcList_item.pTab object. If successful, +** allocates and populates the SrcItem.pTab object. If successful, ** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, ** SQLITE_NOMEM. */ @@ -6421,7 +6473,7 @@ static void havingToWhere(Parse *pParse, Select *p){ /* ** Check to see if the pThis entry of pTabList is a self-join of a prior view. -** If it is, then return the SrcList_item for the prior view. If it is not, +** If it is, then return the SrcItem for the prior view. If it is not, ** then return 0. */ static SrcItem *isSelfJoinView( @@ -7039,7 +7091,10 @@ int sqlite3Select( } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem)); + dest.zAffSdst = sqlite3TableAffinityStr(db, pItem->pTab); sqlite3Select(pParse, pSub, &dest); + sqlite3DbFree(db, dest.zAffSdst); + dest.zAffSdst = 0; pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); @@ -7465,7 +7520,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); SELECTTRACE(1,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, - 0, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) + p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ){ @@ -7764,7 +7819,7 @@ int sqlite3Select( SELECTTRACE(1,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, - pDistinct, 0, minMaxFlag|distFlag, 0); + pDistinct, p, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ goto select_end; } diff --git a/src/shell.c.in b/src/shell.c.in index d6f3a0aeb6..635361aa92 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -85,6 +85,14 @@ # define _LARGEFILE_SOURCE 1 #endif +#if defined(SQLITE_SHELL_FIDDLE) && !defined(_POSIX_SOURCE) +/* +** emcc requires _POSIX_SOURCE (or one of several similar defines) +** to expose strdup(). +*/ +# define _POSIX_SOURCE +#endif + #include #include #include @@ -241,6 +249,18 @@ static void setTextMode(FILE *file, int isOutput){ /* True if the timer is enabled */ static int enableTimer = 0; +/* A version of strcmp() that works with NULL values */ +static int cli_strcmp(const char *a, const char *b){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strcmp(a,b); +} +static int cli_strncmp(const char *a, const char *b, size_t n){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strncmp(a,b,n); +} + /* Return the current wall-clock time */ static sqlite3_int64 timeOfDay(void){ static sqlite3_vfs *clockVfs = 0; @@ -529,6 +549,7 @@ static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ int i; int n; int aw = w<0 ? -w : w; + if( zUtf==0 ) zUtf = ""; for(i=n=0; zUtf[i]; i++){ if( (zUtf[i]&0xc0)!=0x80 ){ n++; @@ -672,7 +693,7 @@ static char *local_getline(char *zLine, FILE *in){ if( stdin_is_interactive && in==stdin ){ char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ - int nTrans = strlen30(zTrans)+1; + i64 nTrans = strlen(zTrans)+1; if( nTrans>nLine ){ zLine = realloc(zLine, nTrans); shell_check_oom(zLine); @@ -808,9 +829,9 @@ static void freeText(ShellText *p){ ** quote character for zAppend. */ static void appendText(ShellText *p, const char *zAppend, char quote){ - int len; - int i; - int nAppend = strlen30(zAppend); + i64 len; + i64 i; + i64 nAppend = strlen30(zAppend); len = nAppend+p->n+1; if( quote ){ @@ -969,10 +990,10 @@ static void shellAddSchemaName( const char *zName = (const char*)sqlite3_value_text(apVal[2]); sqlite3 *db = sqlite3_context_db_handle(pCtx); UNUSED_PARAMETER(nVal); - if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){ + if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){ for(i=0; iautoEQPtest ){ utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); } @@ -2026,14 +2057,14 @@ static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ */ static void eqp_render_level(ShellState *p, int iEqpId){ EQPGraphRow *pRow, *pNext; - int n = strlen30(p->sGraph.zPrefix); + i64 n = strlen(p->sGraph.zPrefix); char *z; for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ pNext = eqp_next_row(p, iEqpId, pRow); z = pRow->zText; utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z); - if( n<(int)sizeof(p->sGraph.zPrefix)-7 ){ + if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); eqp_render_level(p, pRow->iEqpId); p->sGraph.zPrefix[n] = 0; @@ -2633,6 +2664,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){ while( (zSql[len]&0xc0)==0x80 ) len--; } zCode = sqlite3_mprintf("%.*s", len, zSql); + shell_check_oom(zCode); for(i=0; zCode[i]; i++){ if( IsSpace(zSql[i]) ) zCode[i] = ' '; } if( iOffset<25 ){ zMsg = sqlite3_mprintf("\n %z\n %*s^--- error here", zCode, iOffset, ""); @@ -2749,7 +2781,7 @@ static void displayLinuxIoStats(FILE *out){ int i; for(i=0; icMode = p->mode; sqlite3_reset(pSql); return; @@ -3787,10 +3819,10 @@ static int expertDotCommand( int n; if( z[0]=='-' && z[1]=='-' ) z++; n = strlen30(z); - if( n>=2 && 0==strncmp(z, "-verbose", n) ){ + if( n>=2 && 0==cli_strncmp(z, "-verbose", n) ){ pState->expert.bVerbose = 1; } - else if( n>=2 && 0==strncmp(z, "-sample", n) ){ + else if( n>=2 && 0==cli_strncmp(z, "-sample", n) ){ if( i==(nArg-1) ){ raw_printf(stderr, "option requires an argument: %s\n", z); rc = SQLITE_ERROR; @@ -4138,18 +4170,20 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ zTable = azArg[0]; zType = azArg[1]; zSql = azArg[2]; + if( zTable==0 ) return 0; + if( zType==0 ) return 0; dataOnly = (p->shellFlgs & SHFLG_DumpDataOnly)!=0; noSys = (p->shellFlgs & SHFLG_DumpNoSys)!=0; - if( strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ + if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n"); - }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){ return 0; }else if( dataOnly ){ /* no-op */ - }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ char *zIns; if( !p->writableSchema ){ raw_printf(p->out, "PRAGMA writable_schema=ON;\n"); @@ -4167,7 +4201,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ printSchemaLine(p->out, zSql, ";\n"); } - if( strcmp(zType, "table")==0 ){ + if( cli_strcmp(zType, "table")==0 ){ ShellText sSelect; ShellText sTable; char **azCol; @@ -4330,7 +4364,7 @@ static const char *(azHelp[]) = { ".connection [close] [#] Open or close an auxiliary database connection", ".databases List names and files of attached databases", ".dbconfig ?op? ?val? List or change sqlite3_db_config() options", -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#if SQLITE_SHELL_HAVE_RECOVER ".dbinfo ?DB? Show status information about the database", #endif ".dump ?OBJECTS? Render database content as SQL", @@ -4411,7 +4445,7 @@ static const char *(azHelp[]) = { " line One value per line", " list Values delimited by \"|\"", " markdown Markdown table format", - " qbox Shorthand for \"box --width 60 --quote\"", + " qbox Shorthand for \"box --wrap 60 --quote\"", " quote Escape answers as for SQL", " table ASCII-art table", " tabs Tab-separated values", @@ -4478,10 +4512,9 @@ static const char *(azHelp[]) = { ".read FILE Read input from FILE or command output", " If FILE begins with \"|\", it is a command that generates the input.", #endif -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#if SQLITE_SHELL_HAVE_RECOVER ".recover Recover as much data as possible from corrupt db.", - " --freelist-corrupt Assume the freelist is corrupt", - " --recovery-db NAME Store recovery metadata in database file NAME", + " --ignore-freelist Ignore pages that appear to be on db freelist", " --lost-and-found TABLE Alternative name for the lost-and-found table", " --no-rowids Do not attempt to recover rowid values", " that are not also INTEGER PRIMARY KEYs", @@ -4586,9 +4619,9 @@ static int showHelp(FILE *out, const char *zPattern){ char *zPat; if( zPattern==0 || zPattern[0]=='0' - || strcmp(zPattern,"-a")==0 - || strcmp(zPattern,"-all")==0 - || strcmp(zPattern,"--all")==0 + || cli_strcmp(zPattern,"-a")==0 + || cli_strcmp(zPattern,"-all")==0 + || cli_strcmp(zPattern,"--all")==0 ){ /* Show all commands, but only one line per command */ if( zPattern==0 ) zPattern = ""; @@ -4825,7 +4858,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ iOffset = k; continue; } - if( strncmp(zLine, "| end ", 6)==0 ){ + if( cli_strncmp(zLine, "| end ", 6)==0 ){ break; } rc = sscanf(zLine,"| %d: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x", @@ -4853,7 +4886,7 @@ readHexDb_error: }else{ while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ nLine++; - if(strncmp(zLine, "| end ", 6)==0 ) break; + if(cli_strncmp(zLine, "| end ", 6)==0 ) break; } p->lineno = nLine; } @@ -4945,28 +4978,28 @@ static void shellEscapeCrnl( const char *zText = (const char*)sqlite3_value_text(argv[0]); UNUSED_PARAMETER(argc); if( zText && zText[0]=='\'' ){ - int nText = sqlite3_value_bytes(argv[0]); - int i; + i64 nText = sqlite3_value_bytes(argv[0]); + i64 i; char zBuf1[20]; char zBuf2[20]; const char *zNL = 0; const char *zCR = 0; - int nCR = 0; - int nNL = 0; + i64 nCR = 0; + i64 nNL = 0; for(i=0; zText[i]; i++){ if( zNL==0 && zText[i]=='\n' ){ zNL = unused_string(zText, "\\n", "\\012", zBuf1); - nNL = (int)strlen(zNL); + nNL = strlen(zNL); } if( zCR==0 && zText[i]=='\r' ){ zCR = unused_string(zText, "\\r", "\\015", zBuf2); - nCR = (int)strlen(zCR); + nCR = strlen(zCR); } } if( zNL || zCR ){ - int iOut = 0; + i64 iOut = 0; i64 nMax = (nNL > nCR) ? nNL : nCR; i64 nAlloc = nMax * nText + (nMax+64)*2; char *zOut = (char*)sqlite3_malloc64(nAlloc); @@ -5093,7 +5126,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); #endif -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#if SQLITE_SHELL_HAVE_RECOVER sqlite3_dbdata_init(p->db, 0, 0); #endif #ifdef SQLITE_HAVE_ZLIB @@ -5207,8 +5240,8 @@ static char **readline_completion(const char *zText, int iStart, int iEnd){ ** Linenoise completion callback */ static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ - int nLine = strlen30(zLine); - int i, iStart; + i64 nLine = strlen(zLine); + i64 i, iStart; sqlite3_stmt *pStmt = 0; char *zSql; char zBuf[1000]; @@ -5346,11 +5379,11 @@ static void output_file_close(FILE *f){ */ static FILE *output_file_open(const char *zFile, int bTextMode){ FILE *f; - if( strcmp(zFile,"stdout")==0 ){ + if( cli_strcmp(zFile,"stdout")==0 ){ f = stdout; - }else if( strcmp(zFile, "stderr")==0 ){ + }else if( cli_strcmp(zFile, "stderr")==0 ){ f = stderr; - }else if( strcmp(zFile, "off")==0 ){ + }else if( cli_strcmp(zFile, "off")==0 ){ f = 0; }else{ f = fopen(zFile, bTextMode ? "w" : "wb"); @@ -5374,7 +5407,7 @@ static int sql_trace_callback( ShellState *p = (ShellState*)pArg; sqlite3_stmt *pStmt; const char *zSql; - int nSql; + i64 nSql; if( p->traceOut==0 ) return 0; if( mType==SQLITE_TRACE_CLOSE ){ utf8_printf(p->traceOut, "-- closing database connection\n"); @@ -5402,17 +5435,18 @@ static int sql_trace_callback( } } if( zSql==0 ) return 0; - nSql = strlen30(zSql); + nSql = strlen(zSql); + if( nSql>1000000000 ) nSql = 1000000000; while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } switch( mType ){ case SQLITE_TRACE_ROW: case SQLITE_TRACE_STMT: { - utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + utf8_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql); break; } case SQLITE_TRACE_PROFILE: { sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; - utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } } @@ -5961,7 +5995,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ } if( zDb==0 ){ zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); - }else if( strcmp(zDb,"temp")==0 ){ + }else if( cli_strcmp(zDb,"temp")==0 ){ zSchemaTab = sqlite3_mprintf("%s", "sqlite_temp_schema"); }else{ zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_schema", zDb); @@ -5977,8 +6011,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion); return 0; } -#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) - && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ +#endif /* SQLITE_SHELL_HAVE_RECOVER */ /* ** Print the current sqlite3_errmsg() value to stderr and return 1. @@ -6095,7 +6128,7 @@ static int optionMatch(const char *zStr, const char *zOpt){ if( zStr[0]!='-' ) return 0; zStr++; if( zStr[0]=='-' ) zStr++; - return strcmp(zStr, zOpt)==0; + return cli_strcmp(zStr, zOpt)==0; } /* @@ -7251,364 +7284,16 @@ end_ar_command: *******************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) -/* -** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, the SQL statement or statements in zSql are executed using -** database connection db and the error code written to *pRc before -** this function returns. -*/ -static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ - int rc = *pRc; - if( rc==SQLITE_OK ){ - char *zErr = 0; - rc = sqlite3_exec(db, zSql, 0, 0, &zErr); - if( rc!=SQLITE_OK ){ - raw_printf(stderr, "SQL error: %s\n", zErr); - } - sqlite3_free(zErr); - *pRc = rc; - } -} +#if SQLITE_SHELL_HAVE_RECOVER /* -** Like shellExec(), except that zFmt is a printf() style format string. +** This function is used as a callback by the recover extension. Simply +** print the supplied SQL statement to stdout. */ -static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ - char *z = 0; - if( *pRc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - shellExec(db, pRc, z); - } - sqlite3_free(z); - } -} - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, an attempt is made to allocate, zero and return a pointer -** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set -** to SQLITE_NOMEM and NULL returned. -*/ -static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ - void *pRet = 0; - if( *pRc==SQLITE_OK ){ - pRet = sqlite3_malloc64(nByte); - if( pRet==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - memset(pRet, 0, nByte); - } - } - return pRet; -} - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, zFmt is treated as a printf() style string. The result of -** formatting it along with any trailing arguments is written into a -** buffer obtained from sqlite3_malloc(), and pointer to which is returned. -** It is the responsibility of the caller to eventually free this buffer -** using a call to sqlite3_free(). -** -** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL -** pointer returned. -*/ -static char *shellMPrintf(int *pRc, const char *zFmt, ...){ - char *z = 0; - if( *pRc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - *pRc = SQLITE_NOMEM; - } - } - return z; -} - - -/* -** When running the ".recover" command, each output table, and the special -** orphaned row table if it is required, is represented by an instance -** of the following struct. -*/ -typedef struct RecoverTable RecoverTable; -struct RecoverTable { - char *zQuoted; /* Quoted version of table name */ - int nCol; /* Number of columns in table */ - char **azlCol; /* Array of column lists */ - int iPk; /* Index of IPK column */ -}; - -/* -** Free a RecoverTable object allocated by recoverFindTable() or -** recoverOrphanTable(). -*/ -static void recoverFreeTable(RecoverTable *pTab){ - if( pTab ){ - sqlite3_free(pTab->zQuoted); - if( pTab->azlCol ){ - int i; - for(i=0; i<=pTab->nCol; i++){ - sqlite3_free(pTab->azlCol[i]); - } - sqlite3_free(pTab->azlCol); - } - sqlite3_free(pTab); - } -} - -/* -** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. -** Otherwise, it allocates and returns a RecoverTable object based on the -** final four arguments passed to this function. It is the responsibility -** of the caller to eventually free the returned object using -** recoverFreeTable(). -*/ -static RecoverTable *recoverNewTable( - int *pRc, /* IN/OUT: Error code */ - const char *zName, /* Name of table */ - const char *zSql, /* CREATE TABLE statement */ - int bIntkey, - int nCol -){ - sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ - int rc = *pRc; - RecoverTable *pTab = 0; - - pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); - if( rc==SQLITE_OK ){ - int nSqlCol = 0; - int bSqlIntkey = 0; - sqlite3_stmt *pStmt = 0; - - rc = sqlite3_open("", &dbtmp); - if( rc==SQLITE_OK ){ - sqlite3_create_function(dbtmp, "shell_idquote", 1, SQLITE_UTF8, 0, - shellIdQuote, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); - if( rc==SQLITE_ERROR ){ - rc = SQLITE_OK; - goto finished; - } - } - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT count(*) FROM pragma_table_info(%Q)", zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - nSqlCol = sqlite3_column_int(pStmt, 0); - } - shellFinalize(&rc, pStmt); - - if( rc!=SQLITE_OK || nSqlColiPk to the index - ** of the column, where columns are 0-numbered from left to right. - ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, - ** leave zPk as "_rowid_" and pTab->iPk at -2. */ - pTab->iPk = -2; - if( bIntkey ){ - shellPreparePrintf(dbtmp, &rc, &pPkFinder, - "SELECT cid, name FROM pragma_table_info(%Q) " - " WHERE pk=1 AND type='integer' COLLATE nocase" - " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" - , zName, zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ - pTab->iPk = sqlite3_column_int(pPkFinder, 0); - zPk = (const char*)sqlite3_column_text(pPkFinder, 1); - if( zPk==0 ){ zPk = "_"; /* Defensive. Should never happen */ } - } - } - - pTab->zQuoted = shellMPrintf(&rc, "\"%w\"", zName); - pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); - pTab->nCol = nSqlCol; - - if( bIntkey ){ - pTab->azlCol[0] = shellMPrintf(&rc, "\"%w\"", zPk); - }else{ - pTab->azlCol[0] = shellMPrintf(&rc, ""); - } - i = 1; - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT %Q || group_concat(shell_idquote(name), ', ') " - " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " - "FROM pragma_table_info(%Q)", - bIntkey ? ", " : "", pTab->iPk, - bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", - zName - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zText = (const char*)sqlite3_column_text(pStmt, 0); - pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); - i++; - } - shellFinalize(&rc, pStmt); - - shellFinalize(&rc, pPkFinder); - } - } - - finished: - sqlite3_close(dbtmp); - *pRc = rc; - if( rc!=SQLITE_OK || (pTab && pTab->zQuoted==0) ){ - recoverFreeTable(pTab); - pTab = 0; - } - return pTab; -} - -/* -** This function is called to search the schema recovered from the -** sqlite_schema table of the (possibly) corrupt database as part -** of a ".recover" command. Specifically, for a table with root page -** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the -** table must be a WITHOUT ROWID table, or if non-zero, not one of -** those. -** -** If a table is found, a (RecoverTable*) object is returned. Or, if -** no such table is found, but bIntkey is false and iRoot is the -** root page of an index in the recovered schema, then (*pbNoop) is -** set to true and NULL returned. Or, if there is no such table or -** index, NULL is returned and (*pbNoop) set to 0, indicating that -** the caller should write data to the orphans table. -*/ -static RecoverTable *recoverFindTable( - ShellState *pState, /* Shell state object */ - int *pRc, /* IN/OUT: Error code */ - int iRoot, /* Root page of table */ - int bIntkey, /* True for an intkey table */ - int nCol, /* Number of columns in table */ - int *pbNoop /* OUT: True if iRoot is root of index */ -){ - sqlite3_stmt *pStmt = 0; - RecoverTable *pRet = 0; - int bNoop = 0; - const char *zSql = 0; - const char *zName = 0; - - /* Search the recovered schema for an object with root page iRoot. */ - shellPreparePrintf(pState->db, pRc, &pStmt, - "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot - ); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zType = (const char*)sqlite3_column_text(pStmt, 0); - if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ - bNoop = 1; - break; - } - if( sqlite3_stricmp(zType, "table")==0 ){ - zName = (const char*)sqlite3_column_text(pStmt, 1); - zSql = (const char*)sqlite3_column_text(pStmt, 2); - if( zName!=0 && zSql!=0 ){ - pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); - break; - } - } - } - - shellFinalize(pRc, pStmt); - *pbNoop = bNoop; - return pRet; -} - -/* -** Return a RecoverTable object representing the orphans table. -*/ -static RecoverTable *recoverOrphanTable( - ShellState *pState, /* Shell state object */ - int *pRc, /* IN/OUT: Error code */ - const char *zLostAndFound, /* Base name for orphans table */ - int nCol /* Number of user data columns */ -){ - RecoverTable *pTab = 0; - if( nCol>=0 && *pRc==SQLITE_OK ){ - int i; - - /* This block determines the name of the orphan table. The prefered - ** name is zLostAndFound. But if that clashes with another name - ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 - ** and so on until a non-clashing name is found. */ - int iTab = 0; - char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); - sqlite3_stmt *pTest = 0; - shellPrepare(pState->db, pRc, - "SELECT 1 FROM recovery.schema WHERE name=?", &pTest - ); - if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ - shellReset(pRc, pTest); - sqlite3_free(zTab); - zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); - sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); - } - shellFinalize(pRc, pTest); - - pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pTab ){ - pTab->zQuoted = shellMPrintf(pRc, "\"%w\"", zTab); - pTab->nCol = nCol; - pTab->iPk = -2; - if( nCol>0 ){ - pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); - if( pTab->azlCol ){ - pTab->azlCol[nCol] = shellMPrintf(pRc, ""); - for(i=nCol-1; i>=0; i--){ - pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); - } - } - } - - if( *pRc!=SQLITE_OK ){ - recoverFreeTable(pTab); - pTab = 0; - }else{ - raw_printf(pState->out, - "CREATE TABLE %s(rootpgno INTEGER, " - "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted - ); - for(i=0; iout, ", c%d", i); - } - raw_printf(pState->out, ");\n"); - } - } - sqlite3_free(zTab); - } - return pTab; +static int recoverSqlCb(void *pCtx, const char *zSql){ + ShellState *pState = (ShellState*)pCtx; + utf8_printf(pState->out, "%s;\n", zSql); + return SQLITE_OK; } /* @@ -7618,32 +7303,33 @@ static RecoverTable *recoverOrphanTable( */ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int rc = SQLITE_OK; - sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ - sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ - sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ - const char *zRecoveryDb = ""; /* Name of "recovery" database */ - const char *zLostAndFound = "lost_and_found"; - int i; - int nOrphan = -1; - RecoverTable *pOrphan = 0; - - int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ + const char *zRecoveryDb = ""; /* Name of "recovery" database. Debug only */ + const char *zLAF = "lost_and_found"; + int bFreelist = 1; /* 0 if --ignore-freelist is specified */ int bRowids = 1; /* 0 if --no-rowids */ + sqlite3_recover *p = 0; + int i = 0; + for(i=1; idb, &rc, - /* Attach an in-memory database named 'recovery'. Create an indexed - ** cache of the sqlite_dbptr virtual table. */ - "PRAGMA writable_schema = on;" - "ATTACH %Q AS recovery;" - "DROP TABLE IF EXISTS recovery.dbptr;" - "DROP TABLE IF EXISTS recovery.freelist;" - "DROP TABLE IF EXISTS recovery.map;" - "DROP TABLE IF EXISTS recovery.schema;" - "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb + p = sqlite3_recover_init_sql( + pState->db, "main", recoverSqlCb, (void*)pState ); - if( bFreelist ){ - shellExec(pState->db, &rc, - "WITH trunk(pgno) AS (" - " SELECT shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " - " WHERE x>0" - " UNION" - " SELECT shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " - " FROM trunk WHERE x>0" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, shell_int32(data, 1)-1), t.pgno " - " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" - " UNION ALL" - " SELECT data, n-1, shell_int32(data, 2+n) " - " FROM freelist WHERE n>=0" - ")" - "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" - ); + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); + sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); + sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); + + sqlite3_recover_run(p); + if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(p); + int errCode = sqlite3_recover_errcode(p); + raw_printf(stderr, "sql error: %s (%d)\n", zErr, errCode); } - - /* If this is an auto-vacuum database, add all pointer-map pages to - ** the freelist table. Do this regardless of whether or not - ** --freelist-corrupt was specified. */ - shellExec(pState->db, &rc, - "WITH ptrmap(pgno) AS (" - " SELECT 2 WHERE shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 13" - " )" - " UNION ALL " - " SELECT pgno+1+(SELECT page_size FROM pragma_page_size)/5 AS pp " - " FROM ptrmap WHERE pp<=(SELECT page_count FROM pragma_page_count)" - ")" - "REPLACE INTO recovery.freelist SELECT pgno FROM ptrmap" - ); - - shellExec(pState->db, &rc, - "CREATE TABLE recovery.dbptr(" - " pgno, child, PRIMARY KEY(child, pgno)" - ") WITHOUT ROWID;" - "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " - " SELECT * FROM sqlite_dbptr" - " WHERE pgno NOT IN freelist AND child NOT IN freelist;" - - /* Delete any pointer to page 1. This ensures that page 1 is considered - ** a root page, regardless of how corrupt the db is. */ - "DELETE FROM recovery.dbptr WHERE child = 1;" - - /* Delete all pointers to any pages that have more than one pointer - ** to them. Such pages will be treated as root pages when recovering - ** data. */ - "DELETE FROM recovery.dbptr WHERE child IN (" - " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" - ");" - - /* Create the "map" table that will (eventually) contain instructions - ** for dealing with each page in the db that contains one or more - ** records. */ - "CREATE TABLE recovery.map(" - "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" - ");" - - /* Populate table [map]. If there are circular loops of pages in the - ** database, the following adds all pages in such a loop to the map - ** as individual root pages. This could be handled better. */ - "WITH pages(i, maxlen) AS (" - " SELECT page_count, (" - " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" - " ) FROM pragma_page_count WHERE page_count>0" - " UNION ALL" - " SELECT i-1, (" - " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" - " ) FROM pages WHERE i>=2" - ")" - "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " - " SELECT i, maxlen, NULL, (" - " WITH p(orig, pgno, parent) AS (" - " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" - " UNION " - " SELECT i, p.parent, " - " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" - " )" - " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" - ") " - "FROM pages WHERE maxlen IS NOT NULL AND i NOT IN freelist;" - "UPDATE recovery.map AS o SET intkey = (" - " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" - ");" - - /* Extract data from page 1 and any linked pages into table - ** recovery.schema. With the same schema as an sqlite_schema table. */ - "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - "INSERT INTO recovery.schema SELECT " - " max(CASE WHEN field=0 THEN value ELSE NULL END)," - " max(CASE WHEN field=1 THEN value ELSE NULL END)," - " max(CASE WHEN field=2 THEN value ELSE NULL END)," - " max(CASE WHEN field=3 THEN value ELSE NULL END)," - " max(CASE WHEN field=4 THEN value ELSE NULL END)" - "FROM sqlite_dbdata WHERE pgno IN (" - " SELECT pgno FROM recovery.map WHERE root=1" - ")" - "GROUP BY pgno, cell;" - "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);" - ); - - /* Open a transaction, then print out all non-virtual, non-"sqlite_%" - ** CREATE TABLE statements that extracted from the existing schema. */ - if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - /* ".recover" might output content in an order which causes immediate - ** foreign key constraints to be violated. So disable foreign-key - ** constraint enforcement to prevent problems when running the output - ** script. */ - raw_printf(pState->out, "PRAGMA foreign_keys=OFF;\n"); - raw_printf(pState->out, "BEGIN;\n"); - raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); - shellPrepare(pState->db, &rc, - "SELECT sql FROM recovery.schema " - "WHERE type='table' AND sql LIKE 'create table%'", &pStmt - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); - raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", - &zCreateTable[12] - ); - } - shellFinalize(&rc, pStmt); - } - - /* Figure out if an orphan table will be required. And if so, how many - ** user columns it should contain */ - shellPrepare(pState->db, &rc, - "SELECT coalesce(max(maxlen), -2) FROM recovery.map WHERE root>1" - , &pLoop - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - nOrphan = sqlite3_column_int(pLoop, 0); - } - shellFinalize(&rc, pLoop); - pLoop = 0; - - shellPrepare(pState->db, &rc, - "SELECT pgno FROM recovery.map WHERE root=?", &pPages - ); - - shellPrepare(pState->db, &rc, - "SELECT max(field), group_concat(shell_escape_crnl(quote" - "(case when (? AND field<0) then NULL else value end)" - "), ', ')" - ", min(field) " - "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" - "GROUP BY cell", &pCells - ); - - /* Loop through each root page. */ - shellPrepare(pState->db, &rc, - "SELECT root, intkey, max(maxlen) FROM recovery.map" - " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" - " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" - ")", &pLoop - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - int iRoot = sqlite3_column_int(pLoop, 0); - int bIntkey = sqlite3_column_int(pLoop, 1); - int nCol = sqlite3_column_int(pLoop, 2); - int bNoop = 0; - RecoverTable *pTab; - - assert( bIntkey==0 || bIntkey==1 ); - pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); - if( bNoop || rc ) continue; - if( pTab==0 ){ - if( pOrphan==0 ){ - pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); - } - pTab = pOrphan; - if( pTab==0 ) break; - } - - if( 0==sqlite3_stricmp(pTab->zQuoted, "\"sqlite_sequence\"") ){ - raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); - } - sqlite3_bind_int(pPages, 1, iRoot); - if( bRowids==0 && pTab->iPk<0 ){ - sqlite3_bind_int(pCells, 1, 1); - }else{ - sqlite3_bind_int(pCells, 1, 0); - } - sqlite3_bind_int(pCells, 3, pTab->iPk); - - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ - int iPgno = sqlite3_column_int(pPages, 0); - sqlite3_bind_int(pCells, 2, iPgno); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ - int nField = sqlite3_column_int(pCells, 0); - int iMin = sqlite3_column_int(pCells, 2); - const char *zVal = (const char*)sqlite3_column_text(pCells, 1); - - RecoverTable *pTab2 = pTab; - if( pTab!=pOrphan && (iMin<0)!=bIntkey ){ - if( pOrphan==0 ){ - pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); - } - pTab2 = pOrphan; - if( pTab2==0 ) break; - } - - nField = nField+1; - if( pTab2==pOrphan ){ - raw_printf(pState->out, - "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", - pTab2->zQuoted, iRoot, iPgno, nField, - iMin<0 ? "" : "NULL, ", zVal, pTab2->azlCol[nField] - ); - }else{ - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab2->zQuoted, pTab2->azlCol[nField], zVal - ); - } - } - shellReset(&rc, pCells); - } - shellReset(&rc, pPages); - if( pTab!=pOrphan ) recoverFreeTable(pTab); - } - shellFinalize(&rc, pLoop); - shellFinalize(&rc, pPages); - shellFinalize(&rc, pCells); - recoverFreeTable(pOrphan); - - /* The rest of the schema */ - if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - shellPrepare(pState->db, &rc, - "SELECT sql, name FROM recovery.schema " - "WHERE sql NOT LIKE 'create table%'", &pStmt - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); - if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ - const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - char *zPrint = shellMPrintf(&rc, - "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", - zName, zName, zSql - ); - raw_printf(pState->out, "%s;\n", zPrint); - sqlite3_free(zPrint); - }else{ - raw_printf(pState->out, "%s;\n", zSql); - } - } - shellFinalize(&rc, pStmt); - } - - if( rc==SQLITE_OK ){ - raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); - raw_printf(pState->out, "COMMIT;\n"); - } - sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + rc = sqlite3_recover_finish(p); return rc; } -#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ +#endif /* SQLITE_SHELL_HAVE_RECOVER */ /* @@ -8201,7 +7631,7 @@ static int do_meta_command(char *zLine, ShellState *p){ clearTempFile(p); #ifndef SQLITE_OMIT_AUTHORIZATION - if( c=='a' && strncmp(azArg[0], "auth", n)==0 ){ + if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){ if( nArg!=2 ){ raw_printf(stderr, "Usage: .auth ON|OFF\n"); rc = 1; @@ -8220,7 +7650,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) \ && !defined(SQLITE_SHELL_FIDDLE) - if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){ + if( c=='a' && cli_strncmp(azArg[0], "archive", n)==0 ){ open_db(p, 0); failIfSafeMode(p, "cannot run .archive in safe mode"); rc = arDotCommand(p, 0, azArg, nArg); @@ -8228,8 +7658,8 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif #ifndef SQLITE_SHELL_FIDDLE - if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) - || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) + if( (c=='b' && n>=3 && cli_strncmp(azArg[0], "backup", n)==0) + || (c=='s' && n>=3 && cli_strncmp(azArg[0], "save", n)==0) ){ const char *zDestFile = 0; const char *zDb = 0; @@ -8243,10 +7673,10 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *z = azArg[j]; if( z[0]=='-' ){ if( z[1]=='-' ) z++; - if( strcmp(z, "-append")==0 ){ + if( cli_strcmp(z, "-append")==0 ){ zVfs = "apndvfs"; }else - if( strcmp(z, "-async")==0 ){ + if( cli_strcmp(z, "-async")==0 ){ bAsync = 1; }else { @@ -8298,7 +7728,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "bail", n)==0 ){ if( nArg==2 ){ bail_on_error = booleanValue(azArg[1]); }else{ @@ -8307,7 +7737,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){ if( nArg==2 ){ if( booleanValue(azArg[1]) ){ setBinaryMode(p->out, 1); @@ -8323,12 +7753,12 @@ static int do_meta_command(char *zLine, ShellState *p){ /* The undocumented ".breakpoint" command causes a call to the no-op ** routine named test_breakpoint(). */ - if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "breakpoint", n)==0 ){ test_breakpoint(); }else #ifndef SQLITE_SHELL_FIDDLE - if( c=='c' && strcmp(azArg[0],"cd")==0 ){ + if( c=='c' && cli_strcmp(azArg[0],"cd")==0 ){ failIfSafeMode(p, "cannot run .cd in safe mode"); if( nArg==2 ){ #if defined(_WIN32) || defined(WIN32) @@ -8349,7 +7779,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ + if( c=='c' && n>=3 && cli_strncmp(azArg[0], "changes", n)==0 ){ if( nArg==2 ){ setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); }else{ @@ -8363,7 +7793,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. */ - if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){ + if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){ char *zRes = 0; output_reset(p); if( nArg!=2 ){ @@ -8386,7 +7816,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_SHELL_FIDDLE - if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ + if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){ failIfSafeMode(p, "cannot run .clone in safe mode"); if( nArg==2 ){ tryToClone(p, azArg[1]); @@ -8397,7 +7827,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='c' && strncmp(azArg[0], "connection", n)==0 ){ + if( c=='c' && cli_strncmp(azArg[0], "connection", n)==0 ){ if( nArg==1 ){ /* List available connections */ int i; @@ -8424,7 +7854,7 @@ static int do_meta_command(char *zLine, ShellState *p){ globalDb = p->db = p->pAuxDb->db; p->pAuxDb->db = 0; } - }else if( nArg==3 && strcmp(azArg[1], "close")==0 + }else if( nArg==3 && cli_strcmp(azArg[1], "close")==0 && IsDigit(azArg[2][0]) && azArg[2][1]==0 ){ int i = azArg[2][0] - '0'; if( i<0 || i>=ArraySize(p->aAuxDb) ){ @@ -8443,7 +7873,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ + if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){ char **azName = 0; int nName = 0; sqlite3_stmt *pStmt; @@ -8482,7 +7912,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azName); }else - if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){ + if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbconfig", n)==0 ){ static const struct DbConfigChoices { const char *zName; int op; @@ -8507,7 +7937,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int ii, v; open_db(p, 0); for(ii=0; ii1 && strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; + if( nArg>1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; if( nArg>=3 ){ sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); } @@ -8521,18 +7951,18 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) - if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){ +#if SQLITE_SHELL_HAVE_RECOVER + if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbinfo", n)==0 ){ rc = shell_dbinfo_command(p, nArg, azArg); }else - if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ + if( c=='r' && cli_strncmp(azArg[0], "recover", n)==0 ){ open_db(p, 0); rc = recoverDatabaseCmd(p, nArg, azArg); }else -#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ +#endif /* SQLITE_SHELL_HAVE_RECOVER */ - if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + if( c=='d' && cli_strncmp(azArg[0], "dump", n)==0 ){ char *zLike = 0; char *zSql; int i; @@ -8545,7 +7975,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( azArg[i][0]=='-' ){ const char *z = azArg[i]+1; if( z[0]=='-' ) z++; - if( strcmp(z,"preserve-rowids")==0 ){ + if( cli_strcmp(z,"preserve-rowids")==0 ){ #ifdef SQLITE_OMIT_VIRTUALTABLE raw_printf(stderr, "The --preserve-rowids option is not compatible" " with SQLITE_OMIT_VIRTUALTABLE\n"); @@ -8556,13 +7986,13 @@ static int do_meta_command(char *zLine, ShellState *p){ ShellSetFlag(p, SHFLG_PreserveRowid); #endif }else - if( strcmp(z,"newlines")==0 ){ + if( cli_strcmp(z,"newlines")==0 ){ ShellSetFlag(p, SHFLG_Newlines); }else - if( strcmp(z,"data-only")==0 ){ + if( cli_strcmp(z,"data-only")==0 ){ ShellSetFlag(p, SHFLG_DumpDataOnly); }else - if( strcmp(z,"nosys")==0 ){ + if( cli_strcmp(z,"nosys")==0 ){ ShellSetFlag(p, SHFLG_DumpNoSys); }else { @@ -8644,7 +8074,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->shellFlgs = savedShellFlags; }else - if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ setOrClearFlag(p, SHFLG_Echo, azArg[1]); }else{ @@ -8653,22 +8083,22 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ p->autoEQPtest = 0; if( p->autoEQPtrace ){ if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); p->autoEQPtrace = 0; } - if( strcmp(azArg[1],"full")==0 ){ + if( cli_strcmp(azArg[1],"full")==0 ){ p->autoEQP = AUTOEQP_full; - }else if( strcmp(azArg[1],"trigger")==0 ){ + }else if( cli_strcmp(azArg[1],"trigger")==0 ){ p->autoEQP = AUTOEQP_trigger; #ifdef SQLITE_DEBUG - }else if( strcmp(azArg[1],"test")==0 ){ + }else if( cli_strcmp(azArg[1],"test")==0 ){ p->autoEQP = AUTOEQP_on; p->autoEQPtest = 1; - }else if( strcmp(azArg[1],"trace")==0 ){ + }else if( cli_strcmp(azArg[1],"trace")==0 ){ p->autoEQP = AUTOEQP_full; p->autoEQPtrace = 1; open_db(p, 0); @@ -8685,7 +8115,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_SHELL_FIDDLE - if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){ if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); rc = 2; }else @@ -8693,10 +8123,10 @@ static int do_meta_command(char *zLine, ShellState *p){ /* The ".explain" command is automatic now. It is largely pointless. It ** retained purely for backwards compatibility */ - if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){ int val = 1; if( nArg>=2 ){ - if( strcmp(azArg[1],"auto")==0 ){ + if( cli_strcmp(azArg[1],"auto")==0 ){ val = 99; }else{ val = booleanValue(azArg[1]); @@ -8716,7 +8146,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_OMIT_VIRTUALTABLE - if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ if( p->bSafeMode ){ raw_printf(stderr, "Cannot run experimental commands such as \"%s\" in safe mode\n", @@ -8729,7 +8159,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif - if( c=='f' && strncmp(azArg[0], "filectrl", n)==0 ){ + if( c=='f' && cli_strncmp(azArg[0], "filectrl", n)==0 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ @@ -8759,7 +8189,7 @@ static int do_meta_command(char *zLine, ShellState *p){ zCmd = nArg>=2 ? azArg[1] : "help"; if( zCmd[0]=='-' - && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0) + && (cli_strcmp(zCmd,"--schema")==0 || cli_strcmp(zCmd,"-schema")==0) && nArg>=4 ){ zSchema = azArg[2]; @@ -8775,7 +8205,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } /* --help lists all file-controls */ - if( strcmp(zCmd,"help")==0 ){ + if( cli_strcmp(zCmd,"help")==0 ){ utf8_printf(p->out, "Available file-controls:\n"); for(i=0; iout, " .filectrl %s %s\n", @@ -8789,7 +8219,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** of the option name, or a numerical value. */ n2 = strlen30(zCmd); for(i=0; ishowHeader = booleanValue(azArg[1]); p->shellFlgs |= SHFLG_HeaderSet; @@ -8933,7 +8363,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='h' && strncmp(azArg[0], "help", n)==0 ){ + if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){ if( nArg>=2 ){ n = showHelp(p->out, azArg[1]); if( n==0 ){ @@ -8945,7 +8375,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_SHELL_FIDDLE - if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ + if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ char *zSchema = 0; /* within this schema (may default to "main") */ char *zFile = 0; /* Name of file to extra content from */ @@ -8985,18 +8415,18 @@ static int do_meta_command(char *zLine, ShellState *p){ showHelp(p->out, "import"); goto meta_command_exit; } - }else if( strcmp(z,"-v")==0 ){ + }else if( cli_strcmp(z,"-v")==0 ){ eVerbose++; - }else if( strcmp(z,"-schema")==0 && imode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){ + if( nSep==2 && p->mode==MODE_Csv + && cli_strcmp(p->rowSeparator,SEP_CrLf)==0 + ){ /* When importing CSV (only), if the row separator is set to the ** default output row separator, change it to the default input ** row separator. This avoids having to maintain different input @@ -9238,7 +8670,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE - if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){ + if( c=='i' && cli_strncmp(azArg[0], "imposter", n)==0 ){ char *zSql; char *zCollist = 0; sqlite3_stmt *pStmt; @@ -9339,13 +8771,13 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ #ifdef SQLITE_ENABLE_IOTRACE - if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){ + if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){ SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); if( iotrace && iotrace!=stdout ) fclose(iotrace); iotrace = 0; if( nArg<2 ){ sqlite3IoTrace = 0; - }else if( strcmp(azArg[1], "-")==0 ){ + }else if( cli_strcmp(azArg[1], "-")==0 ){ sqlite3IoTrace = iotracePrintf; iotrace = stdout; }else{ @@ -9361,7 +8793,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif - if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){ + if( c=='l' && n>=5 && cli_strncmp(azArg[0], "limits", n)==0 ){ static const struct { const char *zLimitName; /* Name of a limit */ int limitCode; /* Integer code for that limit */ @@ -9420,13 +8852,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='l' && n>2 && strncmp(azArg[0], "lint", n)==0 ){ + if( c=='l' && n>2 && cli_strncmp(azArg[0], "lint", n)==0 ){ open_db(p, 0); lintDotCommand(p, azArg, nArg); }else #if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) - if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ + if( c=='l' && cli_strncmp(azArg[0], "load", n)==0 ){ const char *zFile, *zProc; char *zErrMsg = 0; failIfSafeMode(p, "cannot run .load in safe mode"); @@ -9448,7 +8880,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif #ifndef SQLITE_SHELL_FIDDLE - if( c=='l' && strncmp(azArg[0], "log", n)==0 ){ + if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){ failIfSafeMode(p, "cannot run .log in safe mode"); if( nArg!=2 ){ raw_printf(stderr, "Usage: .log FILENAME\n"); @@ -9461,7 +8893,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif - if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){ + if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ const char *zMode = 0; const char *zTabname = 0; int i, n2; @@ -9480,10 +8912,10 @@ static int do_meta_command(char *zLine, ShellState *p){ cmOpts.bQuote = 0; }else if( zMode==0 ){ zMode = z; - /* Apply defaults for qbox pseudo-mods. If that + /* Apply defaults for qbox pseudo-mode. If that * overwrites already-set values, user was informed of this. */ - if( strcmp(z, "qbox")==0 ){ + if( cli_strcmp(z, "qbox")==0 ){ ColModeOpts cmo = ColModeOpts_default_qbox; zMode = "box"; cmOpts = cmo; @@ -9522,58 +8954,58 @@ static int do_meta_command(char *zLine, ShellState *p){ zMode = modeDescr[p->mode]; } n2 = strlen30(zMode); - if( strncmp(zMode,"lines",n2)==0 ){ + if( cli_strncmp(zMode,"lines",n2)==0 ){ p->mode = MODE_Line; sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( strncmp(zMode,"columns",n2)==0 ){ + }else if( cli_strncmp(zMode,"columns",n2)==0 ){ p->mode = MODE_Column; if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ p->showHeader = 1; } sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); p->cmOpts = cmOpts; - }else if( strncmp(zMode,"list",n2)==0 ){ + }else if( cli_strncmp(zMode,"list",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( strncmp(zMode,"html",n2)==0 ){ + }else if( cli_strncmp(zMode,"html",n2)==0 ){ p->mode = MODE_Html; - }else if( strncmp(zMode,"tcl",n2)==0 ){ + }else if( cli_strncmp(zMode,"tcl",n2)==0 ){ p->mode = MODE_Tcl; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( strncmp(zMode,"csv",n2)==0 ){ + }else if( cli_strncmp(zMode,"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); - }else if( strncmp(zMode,"tabs",n2)==0 ){ + }else if( cli_strncmp(zMode,"tabs",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); - }else if( strncmp(zMode,"insert",n2)==0 ){ + }else if( cli_strncmp(zMode,"insert",n2)==0 ){ p->mode = MODE_Insert; set_table_name(p, zTabname ? zTabname : "table"); - }else if( strncmp(zMode,"quote",n2)==0 ){ + }else if( cli_strncmp(zMode,"quote",n2)==0 ){ p->mode = MODE_Quote; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( strncmp(zMode,"ascii",n2)==0 ){ + }else if( cli_strncmp(zMode,"ascii",n2)==0 ){ p->mode = MODE_Ascii; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); - }else if( strncmp(zMode,"markdown",n2)==0 ){ + }else if( cli_strncmp(zMode,"markdown",n2)==0 ){ p->mode = MODE_Markdown; p->cmOpts = cmOpts; - }else if( strncmp(zMode,"table",n2)==0 ){ + }else if( cli_strncmp(zMode,"table",n2)==0 ){ p->mode = MODE_Table; p->cmOpts = cmOpts; - }else if( strncmp(zMode,"box",n2)==0 ){ + }else if( cli_strncmp(zMode,"box",n2)==0 ){ p->mode = MODE_Box; p->cmOpts = cmOpts; - }else if( strncmp(zMode,"count",n2)==0 ){ + }else if( cli_strncmp(zMode,"count",n2)==0 ){ p->mode = MODE_Count; - }else if( strncmp(zMode,"off",n2)==0 ){ + }else if( cli_strncmp(zMode,"off",n2)==0 ){ p->mode = MODE_Off; - }else if( strncmp(zMode,"json",n2)==0 ){ + }else if( cli_strncmp(zMode,"json",n2)==0 ){ p->mode = MODE_Json; }else{ raw_printf(stderr, "Error: mode should be one of: " @@ -9585,11 +9017,11 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_SHELL_FIDDLE - if( c=='n' && strcmp(azArg[0], "nonce")==0 ){ + if( c=='n' && cli_strcmp(azArg[0], "nonce")==0 ){ if( nArg!=2 ){ raw_printf(stderr, "Usage: .nonce NONCE\n"); rc = 1; - }else if( p->zNonce==0 || strcmp(azArg[1],p->zNonce)!=0 ){ + }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){ raw_printf(stderr, "line %d: incorrect nonce: \"%s\"\n", p->lineno, azArg[1]); exit(1); @@ -9601,7 +9033,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){ + if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); @@ -9611,7 +9043,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){ + if( c=='o' && cli_strncmp(azArg[0], "open", n)==0 && n>=2 ){ const char *zFN = 0; /* Pointer to constant filename */ char *zNewFilename = 0; /* Name of the database file to open */ int iName = 1; /* Index in azArg[] of the filename */ @@ -9675,7 +9107,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->bSafeMode && p->openMode!=SHELL_OPEN_HEXDB && zFN - && strcmp(zFN,":memory:")!=0 + && cli_strcmp(zFN,":memory:")!=0 ){ failIfSafeMode(p, "cannot open disk-based database files in safe mode"); } @@ -9706,8 +9138,9 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( (c=='o' - && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) - || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) + && (cli_strncmp(azArg[0], "output", n)==0 + || cli_strncmp(azArg[0], "once", n)==0)) + || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0) ){ char *zFile = 0; int bTxtMode = 0; @@ -9721,21 +9154,21 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' ){ eMode = 'x'; bOnce = 2; - }else if( strncmp(azArg[0],"once",n)==0 ){ + }else if( cli_strncmp(azArg[0],"once",n)==0 ){ bOnce = 1; } for(i=1; iout, "ERROR: unknown option: \"%s\". Usage:\n", @@ -9808,7 +9241,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ p->out = output_file_open(zFile, bTxtMode); if( p->out==0 ){ - if( strcmp(zFile,"off")!=0 ){ + if( cli_strcmp(zFile,"off")!=0 ){ utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); } p->out = stdout; @@ -9822,14 +9255,14 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "parameter", n)==0 ){ open_db(p,0); if( nArg<=1 ) goto parameter_syntax_error; /* .parameter clear ** Clear all bind parameters by dropping the TEMP table that holds them. */ - if( nArg==2 && strcmp(azArg[1],"clear")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"clear")==0 ){ sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;", 0, 0, 0); }else @@ -9837,7 +9270,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .parameter list ** List all bind parameters. */ - if( nArg==2 && strcmp(azArg[1],"list")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"list")==0 ){ sqlite3_stmt *pStmt = 0; int rx; int len = 0; @@ -9866,7 +9299,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Make sure the TEMP table used to hold bind parameters exists. ** Create it if necessary. */ - if( nArg==2 && strcmp(azArg[1],"init")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"init")==0 ){ bind_table_init(p); }else @@ -9876,7 +9309,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** VALUE can be in either SQL literal notation, or if not it will be ** understood to be a text string. */ - if( nArg==4 && strcmp(azArg[1],"set")==0 ){ + if( nArg==4 && cli_strcmp(azArg[1],"set")==0 ){ int rx; char *zSql; sqlite3_stmt *pStmt; @@ -9914,7 +9347,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Remove the NAME binding from the parameter binding table, if it ** exists. */ - if( nArg==3 && strcmp(azArg[1],"unset")==0 ){ + if( nArg==3 && cli_strcmp(azArg[1],"unset")==0 ){ char *zSql = sqlite3_mprintf( "DELETE FROM temp.sqlite_parameters WHERE key=%Q", azArg[2]); shell_check_oom(zSql); @@ -9926,7 +9359,7 @@ static int do_meta_command(char *zLine, ShellState *p){ showHelp(p->out, "parameter"); }else - if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i1 ) raw_printf(p->out, " "); @@ -9936,7 +9369,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( c=='p' && n>=3 && strncmp(azArg[0], "progress", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "progress", n)==0 ){ int i; int nn = 0; p->flgProgress = 0; @@ -9947,19 +9380,19 @@ static int do_meta_command(char *zLine, ShellState *p){ if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; - if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){ + if( cli_strcmp(z,"quiet")==0 || cli_strcmp(z,"q")==0 ){ p->flgProgress |= SHELL_PROGRESS_QUIET; continue; } - if( strcmp(z,"reset")==0 ){ + if( cli_strcmp(z,"reset")==0 ){ p->flgProgress |= SHELL_PROGRESS_RESET; continue; } - if( strcmp(z,"once")==0 ){ + if( cli_strcmp(z,"once")==0 ){ p->flgProgress |= SHELL_PROGRESS_ONCE; continue; } - if( strcmp(z,"limit")==0 ){ + if( cli_strcmp(z,"limit")==0 ){ if( i+1>=nArg ){ utf8_printf(stderr, "Error: missing argument on --limit\n"); rc = 1; @@ -9981,7 +9414,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ - if( c=='p' && strncmp(azArg[0], "prompt", n)==0 ){ + if( c=='p' && cli_strncmp(azArg[0], "prompt", n)==0 ){ if( nArg >= 2) { strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); } @@ -9991,13 +9424,13 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_SHELL_FIDDLE - if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ + if( c=='q' && cli_strncmp(azArg[0], "quit", n)==0 ){ rc = 2; }else #endif #ifndef SQLITE_SHELL_FIDDLE - if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){ + if( c=='r' && n>=3 && cli_strncmp(azArg[0], "read", n)==0 ){ FILE *inSaved = p->in; int savedLineno = p->lineno; failIfSafeMode(p, "cannot run .read in safe mode"); @@ -10034,7 +9467,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_SHELL_FIDDLE - if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){ + if( c=='r' && n>=3 && cli_strncmp(azArg[0], "restore", n)==0 ){ const char *zSrcFile; const char *zDb; sqlite3 *pSrc; @@ -10087,7 +9520,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ p->scanstatsOn = (u8)booleanValue(azArg[1]); #ifndef SQLITE_ENABLE_STMT_SCANSTATUS @@ -10099,7 +9532,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){ ShellText sSelect; ShellState data; char *zErrMsg = 0; @@ -10242,15 +9675,15 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( (c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0) - || (c=='t' && n==9 && strncmp(azArg[0], "treetrace", n)==0) + if( (c=='s' && n==11 && cli_strncmp(azArg[0], "selecttrace", n)==0) + || (c=='t' && n==9 && cli_strncmp(azArg[0], "treetrace", n)==0) ){ unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &x); }else #if defined(SQLITE_ENABLE_SESSION) - if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){ + if( c=='s' && cli_strncmp(azArg[0],"session",n)==0 && n>=3 ){ struct AuxDb *pAuxDb = p->pAuxDb; OpenSession *pSession = &pAuxDb->aSession[0]; char **azCmd = &azArg[1]; @@ -10261,7 +9694,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg>=3 ){ for(iSes=0; iSesnSession; iSes++){ - if( strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break; + if( cli_strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break; } if( iSesnSession ){ pSession = &pAuxDb->aSession[iSes]; @@ -10277,7 +9710,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Invoke the sqlite3session_attach() interface to attach a particular ** table so that it is never filtered. */ - if( strcmp(azCmd[0],"attach")==0 ){ + if( cli_strcmp(azCmd[0],"attach")==0 ){ if( nCmd!=2 ) goto session_syntax_error; if( pSession->p==0 ){ session_not_open: @@ -10295,7 +9728,9 @@ static int do_meta_command(char *zLine, ShellState *p){ ** .session patchset FILE ** Write a changeset or patchset into a file. The file is overwritten. */ - if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){ + if( cli_strcmp(azCmd[0],"changeset")==0 + || cli_strcmp(azCmd[0],"patchset")==0 + ){ FILE *out = 0; failIfSafeMode(p, "cannot run \".session %s\" in safe mode", azCmd[0]); if( nCmd!=2 ) goto session_syntax_error; @@ -10329,7 +9764,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session close ** Close the identified session */ - if( strcmp(azCmd[0], "close")==0 ){ + if( cli_strcmp(azCmd[0], "close")==0 ){ if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ session_close(pSession); @@ -10340,7 +9775,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session enable ?BOOLEAN? ** Query or set the enable flag */ - if( strcmp(azCmd[0], "enable")==0 ){ + if( cli_strcmp(azCmd[0], "enable")==0 ){ int ii; if( nCmd>2 ) goto session_syntax_error; ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); @@ -10354,7 +9789,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session filter GLOB .... ** Set a list of GLOB patterns of table names to be excluded. */ - if( strcmp(azCmd[0], "filter")==0 ){ + if( cli_strcmp(azCmd[0], "filter")==0 ){ int ii, nByte; if( nCmd<2 ) goto session_syntax_error; if( pAuxDb->nSession ){ @@ -10379,7 +9814,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session indirect ?BOOLEAN? ** Query or set the indirect flag */ - if( strcmp(azCmd[0], "indirect")==0 ){ + if( cli_strcmp(azCmd[0], "indirect")==0 ){ int ii; if( nCmd>2 ) goto session_syntax_error; ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); @@ -10393,7 +9828,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session isempty ** Determine if the session is empty */ - if( strcmp(azCmd[0], "isempty")==0 ){ + if( cli_strcmp(azCmd[0], "isempty")==0 ){ int ii; if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ @@ -10406,7 +9841,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session list ** List all currently open sessions */ - if( strcmp(azCmd[0],"list")==0 ){ + if( cli_strcmp(azCmd[0],"list")==0 ){ for(i=0; inSession; i++){ utf8_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); } @@ -10416,13 +9851,13 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Open a new session called NAME on the attached database DB. ** DB is normally "main". */ - if( strcmp(azCmd[0],"open")==0 ){ + if( cli_strcmp(azCmd[0],"open")==0 ){ char *zName; if( nCmd!=3 ) goto session_syntax_error; zName = azCmd[2]; if( zName[0]==0 ) goto session_syntax_error; for(i=0; inSession; i++){ - if( strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ + if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ utf8_printf(stderr, "Session \"%s\" already exists\n", zName); goto meta_command_exit; } @@ -10453,15 +9888,15 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifdef SQLITE_DEBUG /* Undocumented commands for internal testing. Subject to change ** without notice. */ - if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){ - if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){ + if( c=='s' && n>=10 && cli_strncmp(azArg[0], "selftest-", 9)==0 ){ + if( cli_strncmp(azArg[0]+9, "boolean", n-9)==0 ){ int i, v; for(i=1; iout, "%s: %d 0x%x\n", azArg[i], v, v); } } - if( strncmp(azArg[0]+9, "integer", n-9)==0 ){ + if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){ int i; sqlite3_int64 v; for(i=1; i=4 && strncmp(azArg[0],"selftest",n)==0 ){ + if( c=='s' && n>=4 && cli_strncmp(azArg[0],"selftest",n)==0 ){ int bIsInit = 0; /* True to initialize the SELFTEST table */ int bVerbose = 0; /* Verbose output */ int bSelftestExists; /* True if SELFTEST already exists */ @@ -10487,10 +9922,10 @@ static int do_meta_command(char *zLine, ShellState *p){ for(i=1; i0 ){ printf("%d: %s %s\n", tno, zOp, zSql); } - if( strcmp(zOp,"memo")==0 ){ + if( cli_strcmp(zOp,"memo")==0 ){ utf8_printf(p->out, "%s\n", zSql); }else - if( strcmp(zOp,"run")==0 ){ + if( cli_strcmp(zOp,"run")==0 ){ char *zErrMsg = 0; str.n = 0; str.z[0] = 0; @@ -10560,7 +9995,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); sqlite3_free(zErrMsg); - }else if( strcmp(zAns,str.z)!=0 ){ + }else if( cli_strcmp(zAns,str.z)!=0 ){ nErr++; rc = 1; utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); @@ -10580,7 +10015,7 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); }else - if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){ if( nArg<2 || nArg>3 ){ raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); rc = 1; @@ -10595,7 +10030,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){ + if( c=='s' && n>=4 && cli_strncmp(azArg[0],"sha3sum",n)==0 ){ const char *zLike = 0; /* Which table to checksum. 0 means everything */ int i; /* Loop counter */ int bSchema = 0; /* Also hash the schema */ @@ -10613,15 +10048,15 @@ static int do_meta_command(char *zLine, ShellState *p){ if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; - if( strcmp(z,"schema")==0 ){ + if( cli_strcmp(z,"schema")==0 ){ bSchema = 1; }else - if( strcmp(z,"sha3-224")==0 || strcmp(z,"sha3-256")==0 - || strcmp(z,"sha3-384")==0 || strcmp(z,"sha3-512")==0 + if( cli_strcmp(z,"sha3-224")==0 || cli_strcmp(z,"sha3-256")==0 + || cli_strcmp(z,"sha3-384")==0 || cli_strcmp(z,"sha3-512")==0 ){ iSize = atoi(&z[5]); }else - if( strcmp(z,"debug")==0 ){ + if( cli_strcmp(z,"debug")==0 ){ bDebug = 1; }else { @@ -10661,20 +10096,20 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zTab = (const char*)sqlite3_column_text(pStmt,0); if( zTab==0 ) continue; if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; - if( strncmp(zTab, "sqlite_",7)!=0 ){ + if( cli_strncmp(zTab, "sqlite_",7)!=0 ){ appendText(&sQuery,"SELECT * FROM ", 0); appendText(&sQuery,zTab,'"'); appendText(&sQuery," NOT INDEXED;", 0); - }else if( strcmp(zTab, "sqlite_schema")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_schema")==0 ){ appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_schema" " ORDER BY name;", 0); - }else if( strcmp(zTab, "sqlite_sequence")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_sequence")==0 ){ appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence" " ORDER BY name;", 0); - }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_stat1")==0 ){ appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" " ORDER BY tbl,idx;", 0); - }else if( strcmp(zTab, "sqlite_stat4")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_stat4")==0 ){ appendText(&sQuery, "SELECT * FROM ", 0); appendText(&sQuery, zTab, 0); appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); @@ -10713,7 +10148,8 @@ static int do_meta_command(char *zLine, ShellState *p){ #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) if( c=='s' - && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) + && (cli_strncmp(azArg[0], "shell", n)==0 + || cli_strncmp(azArg[0],"system",n)==0) ){ char *zCmd; int i, x; @@ -10734,7 +10170,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "show", n)==0 ){ static const char *azBool[] = { "off", "on", "trigger", "full"}; const char *zOut; int i; @@ -10787,11 +10223,11 @@ static int do_meta_command(char *zLine, ShellState *p){ p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); }else - if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "stats", n)==0 ){ if( nArg==2 ){ - if( strcmp(azArg[1],"stmt")==0 ){ + if( cli_strcmp(azArg[1],"stmt")==0 ){ p->statsOn = 2; - }else if( strcmp(azArg[1],"vmstep")==0 ){ + }else if( cli_strcmp(azArg[1],"vmstep")==0 ){ p->statsOn = 3; }else{ p->statsOn = (u8)booleanValue(azArg[1]); @@ -10804,9 +10240,9 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0) - || (c=='i' && (strncmp(azArg[0], "indices", n)==0 - || strncmp(azArg[0], "indexes", n)==0) ) + if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) + || (c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 + || cli_strncmp(azArg[0], "indexes", n)==0) ) ){ sqlite3_stmt *pStmt; char **azResult; @@ -10914,7 +10350,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE /* Begin redirecting output to the file "testcase-out.txt" */ - if( c=='t' && strcmp(azArg[0],"testcase")==0 ){ + if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){ output_reset(p); p->out = output_file_open("testcase-out.txt", 0); if( p->out==0 ){ @@ -10929,7 +10365,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE - if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){ + if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ @@ -10976,7 +10412,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } /* --help lists all test-controls */ - if( strcmp(zCmd,"help")==0 ){ + if( cli_strcmp(zCmd,"help")==0 ){ utf8_printf(p->out, "Available test-controls:\n"); for(i=0; iout, " .testctrl %s %s\n", @@ -10990,7 +10426,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** of the option name, or a numerical value. */ n2 = strlen30(zCmd); for(i=0; i4 && strncmp(azArg[0], "timeout", n)==0 ){ + if( c=='t' && n>4 && cli_strncmp(azArg[0], "timeout", n)==0 ){ open_db(p, 0); sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0); }else - if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){ + if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ enableTimer = booleanValue(azArg[1]); if( enableTimer && !HAS_TIMER ){ @@ -11181,7 +10617,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_OMIT_TRACE - if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + if( c=='t' && cli_strncmp(azArg[0], "trace", n)==0 ){ int mType = 0; int jj; open_db(p, 0); @@ -11231,7 +10667,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_OMIT_TRACE) */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_VIRTUALTABLE) - if( c=='u' && strncmp(azArg[0], "unmodule", n)==0 ){ + if( c=='u' && cli_strncmp(azArg[0], "unmodule", n)==0 ){ int ii; int lenOpt; char *zOpt; @@ -11244,7 +10680,7 @@ static int do_meta_command(char *zLine, ShellState *p){ zOpt = azArg[1]; if( zOpt[0]=='-' && zOpt[1]=='-' && zOpt[2]!=0 ) zOpt++; lenOpt = (int)strlen(zOpt); - if( lenOpt>=3 && strncmp(zOpt, "-allexcept",lenOpt)==0 ){ + if( lenOpt>=3 && cli_strncmp(zOpt, "-allexcept",lenOpt)==0 ){ assert( azArg[nArg]==0 ); sqlite3_drop_modules(p->db, nArg>2 ? (const char**)(azArg+2) : 0); }else{ @@ -11256,14 +10692,14 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif #if SQLITE_USER_AUTHENTICATION - if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ + if( c=='u' && cli_strncmp(azArg[0], "user", n)==0 ){ if( nArg<2 ){ raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n"); rc = 1; goto meta_command_exit; } open_db(p, 0); - if( strcmp(azArg[1],"login")==0 ){ + if( cli_strcmp(azArg[1],"login")==0 ){ if( nArg!=4 ){ raw_printf(stderr, "Usage: .user login USER PASSWORD\n"); rc = 1; @@ -11275,7 +10711,7 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]); rc = 1; } - }else if( strcmp(azArg[1],"add")==0 ){ + }else if( cli_strcmp(azArg[1],"add")==0 ){ if( nArg!=5 ){ raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n"); rc = 1; @@ -11287,7 +10723,7 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(stderr, "User-Add failed: %d\n", rc); rc = 1; } - }else if( strcmp(azArg[1],"edit")==0 ){ + }else if( cli_strcmp(azArg[1],"edit")==0 ){ if( nArg!=5 ){ raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n"); rc = 1; @@ -11299,7 +10735,7 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(stderr, "User-Edit failed: %d\n", rc); rc = 1; } - }else if( strcmp(azArg[1],"delete")==0 ){ + }else if( cli_strcmp(azArg[1],"delete")==0 ){ if( nArg!=3 ){ raw_printf(stderr, "Usage: .user delete USER\n"); rc = 1; @@ -11318,7 +10754,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* SQLITE_USER_AUTHENTICATION */ - if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB @@ -11337,7 +10773,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif }else - if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfsinfo", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; sqlite3_vfs *pVfs = 0; if( p->db ){ @@ -11351,7 +10787,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfslist", n)==0 ){ sqlite3_vfs *pVfs; sqlite3_vfs *pCurrent = 0; if( p->db ){ @@ -11369,7 +10805,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfsname", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; char *zVfsName = 0; if( p->db ){ @@ -11381,12 +10817,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ + if( c=='w' && cli_strncmp(azArg[0], "wheretrace", n)==0 ){ unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &x); }else - if( c=='w' && strncmp(azArg[0], "width", n)==0 ){ + if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){ int j; assert( nArg<=ArraySize(azArg) ); p->nWidth = nArg-1; @@ -11564,10 +11000,10 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ if( zErrMsg==0 ){ zErrorType = "Error"; zErrorTail = sqlite3_errmsg(p->db); - }else if( strncmp(zErrMsg, "in prepare, ",12)==0 ){ + }else if( cli_strncmp(zErrMsg, "in prepare, ",12)==0 ){ zErrorType = "Parse error"; zErrorTail = &zErrMsg[12]; - }else if( strncmp(zErrMsg, "stepping, ", 10)==0 ){ + }else if( cli_strncmp(zErrMsg, "stepping, ", 10)==0 ){ zErrorType = "Runtime error"; zErrorTail = &zErrMsg[10]; }else{ @@ -11609,7 +11045,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ const char *zBegin = shellState.wasm.zPos; const char *z = zBegin; char *zLine = 0; - int nZ = 0; + i64 nZ = 0; UNUSED_PARAMETER(in); UNUSED_PARAMETER(isContinuation); @@ -11625,7 +11061,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ shellState.wasm.zPos = z; zLine = realloc(zPrior, nZ+1); shell_check_oom(zLine); - memcpy(zLine, zBegin, (size_t)nZ); + memcpy(zLine, zBegin, nZ); zLine[nZ] = 0; return zLine; } @@ -11643,12 +11079,12 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ static int process_input(ShellState *p){ char *zLine = 0; /* A single input line */ char *zSql = 0; /* Accumulated SQL text */ - int nLine; /* Length of current line */ - int nSql = 0; /* Bytes of zSql[] used */ - int nAlloc = 0; /* Allocated zSql[] space */ + i64 nLine; /* Length of current line */ + i64 nSql = 0; /* Bytes of zSql[] used */ + i64 nAlloc = 0; /* Allocated zSql[] space */ int rc; /* Error code */ int errCnt = 0; /* Number of errors seen */ - int startline = 0; /* Line number for start of current input */ + i64 startline = 0; /* Line number for start of current input */ QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */ if( p->inputNesting==MAX_INPUT_NESTING ){ @@ -11698,7 +11134,7 @@ static int process_input(ShellState *p){ continue; } /* No single-line dispositions remain; accumulate line(s). */ - nLine = strlen30(zLine); + nLine = strlen(zLine); if( nSql+nLine+2>=nAlloc ){ /* Grow buffer by half-again increments when big. */ nAlloc = nSql+(nSql>>1)+nLine+100; @@ -11706,7 +11142,7 @@ static int process_input(ShellState *p){ shell_check_oom(zSql); } if( nSql==0 ){ - int i; + i64 i; for(i=0; zLine[i] && IsSpace(zLine[i]); i++){} assert( nAlloc>0 && zSql!=0 ); memcpy(zSql, zLine+i, nLine+1-i); @@ -11806,7 +11242,7 @@ static char *find_home_dir(int clearFlag){ #endif /* !_WIN32_WCE */ if( home_dir ){ - int n = strlen30(home_dir) + 1; + i64 n = strlen(home_dir) + 1; char *z = malloc( n ); if( z ) memcpy(z, home_dir, n); home_dir = z; @@ -12049,6 +11485,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #ifdef SQLITE_SHELL_FIDDLE stdin_is_interactive = 0; stdout_is_console = 1; + data.wasm.zDefaultDbName = "/fiddle.sqlite3"; #else stdin_is_interactive = isatty(0); stdout_is_console = isatty(1); @@ -12076,7 +11513,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif #if USE_SYSTEM_SQLITE+0!=1 - if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ + if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); @@ -12098,9 +11535,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ argv = argvToFree + argc; for(i=0; i70000 ) sz = 70000; @@ -12195,7 +11632,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_config(SQLITE_CONFIG_PAGECACHE, (n>0 && sz>0) ? malloc(n*sz) : 0, sz, n); data.shellFlgs |= SHFLG_Pagecache; - }else if( strcmp(z,"-lookaside")==0 ){ + }else if( cli_strcmp(z,"-lookaside")==0 ){ int n, sz; sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( sz<0 ) sz = 0; @@ -12203,7 +11640,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( n<0 ) n = 0; sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; - }else if( strcmp(z,"-threadsafe")==0 ){ + }else if( cli_strcmp(z,"-threadsafe")==0 ){ int n; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); switch( n ){ @@ -12212,7 +11649,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ default: sqlite3_config(SQLITE_CONFIG_SERIALIZED); break; } #ifdef SQLITE_ENABLE_VFSTRACE - }else if( strcmp(z,"-vfstrace")==0 ){ + }else if( cli_strcmp(z,"-vfstrace")==0 ){ extern int vfstrace_register( const char *zTraceName, const char *zOldVfsName, @@ -12223,50 +11660,50 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); #endif #ifdef SQLITE_ENABLE_MULTIPLEX - }else if( strcmp(z,"-multiplex")==0 ){ + }else if( cli_strcmp(z,"-multiplex")==0 ){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif - }else if( strcmp(z,"-mmap")==0 ){ + }else if( cli_strcmp(z,"-mmap")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); #ifdef SQLITE_ENABLE_SORTER_REFERENCES - }else if( strcmp(z,"-sorterref")==0 ){ + }else if( cli_strcmp(z,"-sorterref")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); #endif - }else if( strcmp(z,"-vfs")==0 ){ + }else if( cli_strcmp(z,"-vfs")==0 ){ zVfs = cmdline_option_value(argc, argv, ++i); #ifdef SQLITE_HAVE_ZLIB - }else if( strcmp(z,"-zip")==0 ){ + }else if( cli_strcmp(z,"-zip")==0 ){ data.openMode = SHELL_OPEN_ZIPFILE; #endif - }else if( strcmp(z,"-append")==0 ){ + }else if( cli_strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; #ifndef SQLITE_OMIT_DESERIALIZE - }else if( strcmp(z,"-deserialize")==0 ){ + }else if( cli_strcmp(z,"-deserialize")==0 ){ data.openMode = SHELL_OPEN_DESERIALIZE; - }else if( strcmp(z,"-maxsize")==0 && i+10 ){ utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands" " with \"%s\"\n", z); @@ -12494,7 +11931,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ readStdin = 0; break; #endif - }else if( strcmp(z,"-safe")==0 ){ + }else if( cli_strcmp(z,"-safe")==0 ){ data.bSafeMode = data.bSafeModePersist = 1; }else{ utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); @@ -12619,30 +12056,30 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #ifdef SQLITE_SHELL_FIDDLE /* Only for emcc experimentation purposes. */ int fiddle_experiment(int a,int b){ - return a + b; + return a + b; } -/* Only for emcc experimentation purposes. - - Define this function in JS using: - - emcc ... --js-library somefile.js - - containing: - -mergeInto(LibraryManager.library, { - my_foo: function(){ - console.debug("my_foo()",arguments); - } -}); +/* +** Returns a pointer to the current DB handle. */ -/*extern void my_foo(sqlite3 *);*/ -/* Only for emcc experimentation purposes. */ -sqlite3 * fiddle_the_db(){ - printf("fiddle_the_db(%p)\n", (const void*)globalDb); - /*my_foo(globalDb);*/ - return globalDb; +sqlite3 * fiddle_db_handle(){ + return globalDb; } + +/* +** Returns a pointer to the given DB name's VFS. If zDbName is 0 then +** "main" is assumed. Returns 0 if no db with the given name is +** open. +*/ +sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ + sqlite3_vfs * pVfs = 0; + if(globalDb){ + sqlite3_file_control(globalDb, zDbName ? zDbName : "main", + SQLITE_FCNTL_VFS_POINTER, &pVfs); + } + return pVfs; +} + /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ printf("fiddle_db_arg(%p)\n", (const void*)arg); @@ -12656,7 +12093,7 @@ sqlite3 * fiddle_db_arg(sqlite3 *arg){ ** portable enough to make real use of. */ void fiddle_interrupt(void){ - if(globalDb) sqlite3_interrupt(globalDb); + if( globalDb ) sqlite3_interrupt(globalDb); } /* @@ -12670,71 +12107,72 @@ const char * fiddle_db_filename(const char * zDbName){ } /* -** Closes, unlinks, and reopens the db using its current filename (or -** the default if the db is currently closed). It is assumed, for -** purposes of the fiddle build, that the file is in a transient -** virtual filesystem within the browser. +** Completely wipes out the contents of the currently-opened database +** but leaves its storage intact for reuse. */ void fiddle_reset_db(void){ - char *zFilename = 0; - if(0==globalDb){ - shellState.pAuxDb->zDbFilename = "/fiddle.sqlite3"; - }else{ - zFilename = - sqlite3_mprintf("%s", sqlite3_db_filename(globalDb, "main")); - shell_check_oom(zFilename); - close_db(globalDb); - shellDeleteFile(zFilename); - shellState.db = 0; - shellState.pAuxDb->zDbFilename = zFilename; + if( globalDb ){ + int rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) rc = sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); + sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); } - open_db(&shellState, 0); - sqlite3_free(zFilename); } /* -** Trivial exportable function for emscripten. Needs to be exported using: -** -** emcc ..flags... -sEXPORTED_FUNCTIONS=_fiddle_exec -sEXPORTED_RUNTIME_METHODS=ccall,cwrap -** -** (Note the underscore before the function name.) It processes zSql -** as if it were input to the sqlite3 shell and redirects all output -** to the wasm binding. +** Uses the current database's VFS xRead to stream the db file's +** contents out to the given callback. The callback gets a single +** chunk of size n (its 2nd argument) on each call and must return 0 +** on success, non-0 on error. This function returns 0 on success, +** SQLITE_NOTFOUND if no db is open, or propagates any other non-0 +** code from the callback. Note that this is not thread-friendly: it +** expects that it will be the only thread reading the db file and +** takes no measures to ensure that is the case. +*/ +int fiddle_export_db( int (*xCallback)(unsigned const char *zOut, int n) ){ + sqlite3_int64 nSize = 0; + sqlite3_int64 nPos = 0; + sqlite3_file * pFile = 0; + unsigned char buf[1024 * 8]; + int nBuf = (int)sizeof(buf); + int rc = shellState.db + ? sqlite3_file_control(shellState.db, "main", + SQLITE_FCNTL_FILE_POINTER, &pFile) + : SQLITE_NOTFOUND; + if( rc ) return rc; + rc = pFile->pMethods->xFileSize(pFile, &nSize); + if( rc ) return rc; + if(nSize % nBuf){ + /* DB size is not an even multiple of the buffer size. Reduce + ** buffer size so that we do not unduly inflate the db size when + ** exporting. */ + if(0 == nSize % 4096) nBuf = 4096; + else if(0 == nSize % 2048) nBuf = 2048; + else if(0 == nSize % 1024) nBuf = 1024; + else nBuf = 512; + } + for( ; 0==rc && nPospMethods->xRead(pFile, buf, nBuf, nPos); + if(SQLITE_IOERR_SHORT_READ == rc){ + rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/; + } + if( 0==rc ) rc = xCallback(buf, nBuf); + } + return rc; +} + +/* +** Trivial exportable function for emscripten. It processes zSql as if +** it were input to the sqlite3 shell and redirects all output to the +** wasm binding. fiddle_main() must have been called before this +** is called, or results are undefined. */ void fiddle_exec(const char * zSql){ - static int once = 0; - int rc = 0; - if(!once){ - /* Simulate an argv array for main() */ - static char * argv[] = {"fiddle", - "-bail", - "-safe"}; - rc = fiddle_main((int)(sizeof(argv)/sizeof(argv[0])), argv); - once = rc ? -1 : 1; - memset(&shellState.wasm, 0, sizeof(shellState.wasm)); - printf( - "SQLite version %s %.19s\n" /*extra-version-info*/, - sqlite3_libversion(), sqlite3_sourceid() - ); - puts("WASM shell"); - puts("Enter \".help\" for usage hints."); - if(once>0){ - fiddle_reset_db(); - } - if(shellState.db){ - printf("Connected to %s.\n", fiddle_db_filename(NULL)); - }else{ - fprintf(stderr,"ERROR initializing db!\n"); - return; - } - } - if(once<0){ - puts("DB init failed. Not executing SQL."); - }else if(zSql && *zSql){ + if(zSql && *zSql){ + if('.'==*zSql) puts(zSql); shellState.wasm.zInput = zSql; shellState.wasm.zPos = zSql; process_input(&shellState); - memset(&shellState.wasm, 0, sizeof(shellState.wasm)); + shellState.wasm.zInput = shellState.wasm.zPos = 0; } } #endif /* SQLITE_SHELL_FIDDLE */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1cb74d14d1..721a227edf 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -670,13 +670,17 @@ int sqlite3_exec( ** ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods -** of an [sqlite3_io_methods] object. +** of an [sqlite3_io_methods] object. These values are ordered from +** lest restrictive to most restrictive. +** +** The argument to xLock() is always SHARED or higher. The argument to +** xUnlock is either SHARED or NONE. */ -#define SQLITE_LOCK_NONE 0 -#define SQLITE_LOCK_SHARED 1 -#define SQLITE_LOCK_RESERVED 2 -#define SQLITE_LOCK_PENDING 3 -#define SQLITE_LOCK_EXCLUSIVE 4 +#define SQLITE_LOCK_NONE 0 /* xUnlock() only */ +#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */ +#define SQLITE_LOCK_RESERVED 2 /* xLock() only */ +#define SQLITE_LOCK_PENDING 3 /* xLock() only */ +#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */ /* ** CAPI3REF: Synchronization Type Flags @@ -754,7 +758,14 @@ struct sqlite3_file { **
  • [SQLITE_LOCK_PENDING], or **
  • [SQLITE_LOCK_EXCLUSIVE]. ** -** xLock() increases the lock. xUnlock() decreases the lock. +** xLock() upgrades the database file lock. In other words, xLock() moves the +** database file lock in the direction NONE toward EXCLUSIVE. The argument to +** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** SQLITE_LOCK_NONE. If the database file lock is already at or above the +** requested lock, then the call to xLock() is a no-op. +** xUnlock() downgrades the database file lock to either SHARED or NONE. +* If the lock is already at or below the requested lock state, then the call +** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, ** PENDING, or EXCLUSIVE lock on the file. It returns true @@ -859,9 +870,8 @@ struct sqlite3_io_methods { ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) -** into an integer that the pArg argument points to. This capability -** is used during testing and is only available when the SQLITE_TEST -** compile-time option is used. +** into an integer that the pArg argument points to. +** This capability is only available if SQLite is compiled with [SQLITE_DEBUG]. ** **
  • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS @@ -1253,6 +1263,26 @@ typedef struct sqlite3_mutex sqlite3_mutex; */ typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** CAPI3REF: File Name +** +** Type [sqlite3_filename] is used by SQLite to pass filenames to the +** xOpen method of a [VFS]. It may be cast to (const char*) and treated +** as a normal, nul-terminated, UTF-8 buffer containing the filename, but +** may also be passed to special APIs such as: +** +**
      +**
    • sqlite3_filename_database() +**
    • sqlite3_filename_journal() +**
    • sqlite3_filename_wal() +**
    • sqlite3_uri_parameter() +**
    • sqlite3_uri_boolean() +**
    • sqlite3_uri_int64() +**
    • sqlite3_uri_key() +**
    +*/ +typedef const char *sqlite3_filename; + /* ** CAPI3REF: OS Interface Object ** @@ -1431,7 +1461,7 @@ struct sqlite3_vfs { sqlite3_vfs *pNext; /* Next registered VFS */ const char *zName; /* Name of this virtual file system */ void *pAppData; /* Pointer to application-specific data */ - int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*, int flags, int *pOutFlags); int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); @@ -3701,10 +3731,10 @@ int sqlite3_open_v2( ** ** See the [URI filename] documentation for additional information. */ -const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); -const char *sqlite3_uri_key(const char *zFilename, int N); +const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam); +int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault); +sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64); +const char *sqlite3_uri_key(sqlite3_filename z, int N); /* ** CAPI3REF: Translate filenames @@ -3733,9 +3763,9 @@ const char *sqlite3_uri_key(const char *zFilename, int N); ** return value from [sqlite3_db_filename()], then the result is ** undefined and is likely a memory access violation. */ -const char *sqlite3_filename_database(const char*); -const char *sqlite3_filename_journal(const char*); -const char *sqlite3_filename_wal(const char*); +const char *sqlite3_filename_database(sqlite3_filename); +const char *sqlite3_filename_journal(sqlite3_filename); +const char *sqlite3_filename_wal(sqlite3_filename); /* ** CAPI3REF: Database File Corresponding To A Journal @@ -3801,14 +3831,14 @@ sqlite3_file *sqlite3_database_file_object(const char*); ** then the corresponding [sqlite3_module.xClose() method should also be ** invoked prior to calling sqlite3_free_filename(Y). */ -char *sqlite3_create_filename( +sqlite3_filename sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, int nParam, const char **azParam ); -void sqlite3_free_filename(char*); +void sqlite3_free_filename(sqlite3_filename); /* ** CAPI3REF: Error Codes And Messages @@ -5511,6 +5541,16 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), ** then the conversion is performed. Otherwise no conversion occurs. ** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ ** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** sqlite3_value_text(X), sqlite3_value_text16(X), sqlite3_value_text16be(X), +** sqlite3_value_text16le(X), sqlite3_value_bytes(X), or +** sqlite3_value_bytes16(X) might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** ** ^Within the [xUpdate] method of a [virtual table], the ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation @@ -5575,6 +5615,7 @@ int sqlite3_value_type(sqlite3_value*); int sqlite3_value_numeric_type(sqlite3_value*); int sqlite3_value_nochange(sqlite3_value*); int sqlite3_value_frombind(sqlite3_value*); +int sqlite3_value_encoding(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values @@ -5628,7 +5669,7 @@ void sqlite3_value_free(sqlite3_value*); ** ** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer ** when first called if N is less than or equal to zero or if a memory -** allocate error occurs. +** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the @@ -6331,7 +6372,7 @@ const char *sqlite3_db_name(sqlite3 *db, int N); **
  • [sqlite3_filename_wal()] ** */ -const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 2cdd0e429b..79702d7b21 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -331,9 +331,9 @@ struct sqlite3_api_routines { const char *(*filename_journal)(const char*); const char *(*filename_wal)(const char*); /* Version 3.32.0 and later */ - char *(*create_filename)(const char*,const char*,const char*, + const char *(*create_filename)(const char*,const char*,const char*, int,const char**); - void (*free_filename)(char*); + void (*free_filename)(const char*); sqlite3_file *(*database_file_object)(const char*); /* Version 3.34.0 and later */ int (*txn_state)(sqlite3*,const char*); @@ -357,6 +357,8 @@ struct sqlite3_api_routines { unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*, unsigned int); const char *(*db_name)(sqlite3*,int); + /* Version 3.40.0 and later */ + int (*value_encoding)(sqlite3_value*); }; /* @@ -681,6 +683,8 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_serialize sqlite3_api->serialize #endif #define sqlite3_db_name sqlite3_api->db_name +/* Version 3.40.0 and later */ +#define sqlite3_value_encoding sqlite3_api->value_encoding #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e0c9b9af57..09746f2f2e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -208,7 +208,7 @@ ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -#include "config.h" +#include "sqlite_cfg.h" #define SQLITECONFIG_H 1 #endif @@ -1190,6 +1190,7 @@ typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexedExpr IndexedExpr; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; @@ -1255,6 +1256,7 @@ typedef struct With With; #define MASKBIT32(n) (((unsigned int)1)<<(n)) #define SMASKBIT32(n) ((n)<=31?((unsigned int)1)<<(n):0) #define ALLBITS ((Bitmask)-1) +#define TOPBIT (((Bitmask)1)<<(BMS-1)) /* A VList object records a mapping between parameters/variables/wildcards ** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer @@ -1269,11 +1271,11 @@ typedef int VList; ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. */ +#include "os.h" #include "pager.h" #include "btree.h" #include "vdbe.h" #include "pcache.h" -#include "os.h" #include "mutex.h" /* The SQLITE_EXTRA_DURABLE compile-time option used to set the default @@ -1807,6 +1809,7 @@ struct sqlite3 { #define SQLITE_ReleaseReg 0x00400000 /* Use OP_ReleaseReg for testing */ #define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ +#define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -2379,7 +2382,7 @@ struct Table { #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) ((X)->eTabType==TABTYP_VTAB) # define ExprIsVtab(X) \ - ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->eTabType==TABTYP_VTAB) + ((X)->op==TK_COLUMN && (X)->y.pTab->eTabType==TABTYP_VTAB) #else # define IsVirtual(X) 0 # define ExprIsVtab(X) 0 @@ -2596,10 +2599,22 @@ struct UnpackedRecord { ** The Index.onError field determines whether or not the indexed columns ** must be unique and what to do if they are not. When Index.onError=OE_None, ** it means this is not a unique index. Otherwise it is a unique index -** and the value of Index.onError indicate the which conflict resolution -** algorithm to employ whenever an attempt is made to insert a non-unique +** and the value of Index.onError indicates which conflict resolution +** algorithm to employ when an attempt is made to insert a non-unique ** element. ** +** The colNotIdxed bitmask is used in combination with SrcItem.colUsed +** for a fast test to see if an index can serve as a covering index. +** colNotIdxed has a 1 bit for every column of the original table that +** is *not* available in the index. Thus the expression +** "colUsed & colNotIdxed" will be non-zero if the index is not a +** covering index. The most significant bit of of colNotIdxed will always +** be true (note-20221022-a). If a column beyond the 63rd column of the +** table is used, the "colUsed & colNotIdxed" test will always be non-zero +** and we have to assume either that the index is not covering, or use +** an alternative (slower) algorithm to determine whether or not +** the index is covering. +** ** While parsing a CREATE TABLE or CREATE INDEX statement in order to ** generate VDBE code (as opposed to parsing one read from an sqlite_schema ** table as part of parsing an existing database schema), transient instances @@ -2635,6 +2650,8 @@ struct Index { unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ + unsigned bHasExpr:1; /* Index contains an expression, either a literal + ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ @@ -2643,7 +2660,7 @@ struct Index { tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif - Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */ + Bitmask colNotIdxed; /* Unindexed columns in pTab */ }; /* @@ -3096,6 +3113,14 @@ struct IdList { ** The SrcItem object represents a single term in the FROM clause of a query. ** The SrcList object is mostly an array of SrcItems. ** +** The jointype starts out showing the join type between the current table +** and the next table on the list. The parser builds the list this way. +** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each +** jointype expresses the join between the table and the previous table. +** +** In the colUsed field, the high-order bit (bit 63) is set if the table +** contains more than 63 columns and the 64-th or later column is used. +** ** Union member validity: ** ** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc @@ -3135,14 +3160,14 @@ struct SrcItem { Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ } u3; - Bitmask colUsed; /* Bit N (1<62 */ union { char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ } u1; union { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ - CteUse *pCteUse; /* CTE Usage info info fg.isCte is true */ + CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */ } u2; }; @@ -3156,23 +3181,11 @@ struct OnOrUsing { }; /* -** The following structure describes the FROM clause of a SELECT statement. -** Each table or subquery in the FROM clause is a separate element of -** the SrcList.a[] array. +** This object represents one or more tables that are the source of +** content for an SQL statement. For example, a single SrcList object +** is used to hold the FROM clause of a SELECT statement. SrcList also +** represents the target tables for DELETE, INSERT, and UPDATE statements. ** -** With the addition of multiple database support, the following structure -** can also be used to describe a particular table such as the table that -** is modified by an INSERT, DELETE, or UPDATE statement. In standard SQL, -** such a table must be a simple name: ID. But in SQLite, the table can -** now be identified by a database name, a dot, then the table name: ID.ID. -** -** The jointype starts out showing the join type between the current table -** and the next table on the list. The parser builds the list this way. -** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each -** jointype expresses the join between the table and the previous table. -** -** In the colUsed field, the high-order bit (bit 63) is set if the table -** contains more than 63 columns and the 64-th or later column is used. */ struct SrcList { int nSrc; /* Number of tables or subqueries in the FROM clause */ @@ -3517,7 +3530,7 @@ struct SelectDest { int iSDParm2; /* A second parameter for the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ - char *zAffSdst; /* Affinity used when eDest==SRT_Set */ + char *zAffSdst; /* Affinity used for SRT_Set, SRT_Table, and similar */ ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; @@ -3582,6 +3595,28 @@ struct TriggerPrg { # define DbMaskNonZero(M) (M)!=0 #endif +/* +** For each index X that has as one of its arguments either an expression +** or the name of a virtual generated column, and if X is in scope such that +** the value of the expression can simply be read from the index, then +** there is an instance of this object on the Parse.pIdxExpr list. +** +** During code generation, while generating code to evaluate expressions, +** this list is consulted and if a matching expression is found, the value +** is read from the index rather than being recomputed. +*/ +struct IndexedExpr { + Expr *pExpr; /* The expression contained in the index */ + int iDataCur; /* The data cursor associated with the index */ + int iIdxCur; /* The index cursor */ + int iIdxCol; /* The index column that contains value of pExpr */ + u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + const char *zIdxName; /* Name of index, used only for bytecode comments */ +#endif +}; + /* ** An instance of the ParseCleanup object specifies an operation that ** should be performed after parsing to deallocation resources obtained @@ -3623,7 +3658,7 @@ struct Parse { u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ - u8 disableVtab; /* Disable all virtual tables for this parse */ + u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ @@ -3640,6 +3675,7 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ + IndexedExpr *pIdxExpr;/* List of expressions used by active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ @@ -4075,15 +4111,15 @@ struct Walker { struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ int *aiCol; /* array of column indexes */ struct IdxCover *pIdxCover; /* Check for index coverage */ - struct IdxExprTrans *pIdxTrans; /* Convert idxed expr to column */ ExprList *pGroupBy; /* GROUP BY clause */ Select *pSelect; /* HAVING to WHERE clause ctx */ struct WindowRewrite *pRewrite; /* Window rewrite context */ struct WhereConst *pConst; /* WHERE clause constants */ struct RenameCtx *pRename; /* RENAME COLUMN context */ struct Table *pTab; /* Table of generated column */ + struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */ SrcItem *pSrcItem; /* A single FROM clause item */ - DbFixer *pFix; + DbFixer *pFix; /* See sqlite3FixSelect() */ } u; }; @@ -4410,12 +4446,16 @@ int sqlite3HeapNearlyFull(void); */ #ifdef SQLITE_USE_ALLOCA # define sqlite3StackAllocRaw(D,N) alloca(N) +# define sqlite3StackAllocRawNN(D,N) alloca(N) # define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) # define sqlite3StackFree(D,P) +# define sqlite3StackFreeNN(D,P) #else # define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) +# define sqlite3StackAllocRawNN(D,N) sqlite3DbMallocRawNN(D,N) # define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) # define sqlite3StackFree(D,P) sqlite3DbFree(D,P) +# define sqlite3StackFreeNN(D,P) sqlite3DbFreeNN(D,P) #endif /* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they @@ -4962,6 +5002,7 @@ int sqlite3VarintLen(u64 v); const char *sqlite3IndexAffinityStr(sqlite3*, Index*); +char *sqlite3TableAffinityStr(sqlite3*,const Table*); void sqlite3TableAffinity(Vdbe*, Table*, int); char sqlite3CompareAffinity(const Expr *pExpr, char aff2); int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); @@ -5033,7 +5074,6 @@ extern const unsigned char sqlite3OpcodeProperty[]; extern const char sqlite3StrBINARY[]; extern const unsigned char sqlite3StdTypeLen[]; extern const char sqlite3StdTypeAffinity[]; -extern const char sqlite3StdTypeMap[]; extern const char *sqlite3StdType[]; extern const unsigned char sqlite3UpperToLower[]; extern const unsigned char *sqlite3aLTb; @@ -5477,4 +5517,8 @@ void sqlite3VectorErrorMsg(Parse*, Expr*); const char **sqlite3CompileOptions(int *pnOpt); #endif +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +int sqlite3KvvfsInit(void); +#endif + #endif /* SQLITEINT_H */ diff --git a/src/test1.c b/src/test1.c index fc252a9d11..ea99bccc8b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1852,7 +1852,8 @@ static int SQLITE_TCLAPI test_create_function_v2( {"utf16le", SQLITE_UTF16LE }, {"utf16be", SQLITE_UTF16BE }, {"any", SQLITE_ANY }, - {"0", 0 } + {"0", 0 }, + {0, 0 } }; if( objc<5 || (objc%2)==0 ){ @@ -7273,6 +7274,7 @@ static int SQLITE_TCLAPI test_test_control( { "SQLITE_TESTCTRL_SORTER_MMAP", SQLITE_TESTCTRL_SORTER_MMAP }, { "SQLITE_TESTCTRL_IMPOSTER", SQLITE_TESTCTRL_IMPOSTER }, { "SQLITE_TESTCTRL_INTERNAL_FUNCTIONS", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS}, + { 0, 0 } }; int iVerb; int iFlag; @@ -7627,7 +7629,12 @@ static int SQLITE_TCLAPI win32_rmdir( ** ** Enable or disable query optimizations using the sqlite3_test_control() ** interface. Disable if BOOLEAN is false and enable if BOOLEAN is true. -** OPT is the name of the optimization to be disabled. +** OPT is the name of the optimization to be disabled. OPT can also be a +** list or optimizations names, in which case all optimizations named are +** enabled or disabled. +** +** Each invocation of this control overrides all prior invocations. The +** changes are not cumulative. */ static int SQLITE_TCLAPI optimization_control( void * clientData, @@ -7640,6 +7647,7 @@ static int SQLITE_TCLAPI optimization_control( const char *zOpt; int onoff; int mask = 0; + int cnt = 0; static const struct { const char *zOptName; int mask; @@ -7658,6 +7666,7 @@ static int SQLITE_TCLAPI optimization_control( { "skip-scan", SQLITE_SkipScan }, { "push-down", SQLITE_PushDown }, { "balanced-merge", SQLITE_BalancedMerge }, + { "propagate-const", SQLITE_PropagateConst }, }; if( objc!=4 ){ @@ -7668,13 +7677,13 @@ static int SQLITE_TCLAPI optimization_control( if( Tcl_GetBooleanFromObj(interp, objv[3], &onoff) ) return TCL_ERROR; zOpt = Tcl_GetString(objv[2]); for(i=0; i=sizeof(aOpt)/sizeof(aOpt[0]) ){ + if( cnt==0 ){ Tcl_AppendResult(interp, "unknown optimization - should be one of:", (char*)0); for(i=0; i1 && zDir[i]!='/'; i++); - zDir[i] = '\0'; - - /* Open a file-descriptor on the directory. Sync. Close. */ - dfd = open(zDir, O_RDONLY, 0); - if( dfd<0 ){ - rc = -1; - }else{ - rc = fsync(dfd); - close(dfd); + zSlash = strrchr(zDir,'/'); + if( zSlash ){ + /* Open a file-descriptor on the directory. Sync. Close. */ + zSlash[0] = 0; + dfd = open(zDir, O_RDONLY, 0); + if( dfd<0 ){ + rc = -1; + }else{ + rc = fsync(dfd); + close(dfd); + } } } return (rc==0 ? SQLITE_OK : SQLITE_IOERR_DELETE); diff --git a/src/test_multiplex.c b/src/test_multiplex.c index ff12817158..226131f75c 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -272,7 +272,7 @@ static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ return SQLITE_NOMEM; } multiplexFilename(pGroup->zName, pGroup->nName, pGroup->flags, iChunk, z); - pGroup->aReal[iChunk].z = sqlite3_create_filename(z,"","",0,0); + pGroup->aReal[iChunk].z = (char*)sqlite3_create_filename(z,"","",0,0); sqlite3_free(z); if( pGroup->aReal[iChunk].z==0 ) return SQLITE_NOMEM; } diff --git a/src/test_tclsh.c b/src/test_tclsh.c index 3ea9abe8ac..e9ab131b88 100644 --- a/src/test_tclsh.c +++ b/src/test_tclsh.c @@ -109,6 +109,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ extern int TestExpert_Init(Tcl_Interp*); extern int Sqlitetest_window_Init(Tcl_Interp *); extern int Sqlitetestvdbecov_Init(Tcl_Interp *); + extern int TestRecover_Init(Tcl_Interp*); Tcl_CmdInfo cmdInfo; @@ -178,6 +179,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ TestExpert_Init(interp); Sqlitetest_window_Init(interp); Sqlitetestvdbecov_Init(interp); + TestRecover_Init(interp); Tcl_CreateObjCommand( interp, "load_testfixture_extensions", load_testfixture_extensions,0,0 diff --git a/src/trigger.c b/src/trigger.c index 1c62fc231b..ca64d6145b 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -1191,7 +1191,7 @@ static TriggerPrg *codeRowTrigger( sSubParse.zAuthContext = pTrigger->zName; sSubParse.eTriggerOp = pTrigger->op; sSubParse.nQueryLoop = pParse->nQueryLoop; - sSubParse.disableVtab = pParse->disableVtab; + sSubParse.prepFlags = pParse->prepFlags; v = sqlite3GetVdbe(&sSubParse); if( v ){ diff --git a/src/update.c b/src/update.c index ee6044dd29..0d35034544 100644 --- a/src/update.c +++ b/src/update.c @@ -59,11 +59,14 @@ static void updateVirtualTable( ** it has been converted into REAL. */ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ + Column *pCol; assert( pTab!=0 ); - if( !IsView(pTab) ){ + assert( pTab->nCol>i ); + pCol = &pTab->aCol[i]; + if( pCol->iDflt ){ sqlite3_value *pValue = 0; u8 enc = ENC(sqlite3VdbeDb(v)); - Column *pCol = &pTab->aCol[i]; + assert( !IsView(pTab) ); VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName)); assert( inCol ); sqlite3ValueFromExpr(sqlite3VdbeDb(v), @@ -74,7 +77,7 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ } } #ifndef SQLITE_OMIT_FLOATING_POINT - if( pTab->aCol[i].affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ + if( pCol->affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); } #endif diff --git a/src/upsert.c b/src/upsert.c index fb6c7c0c07..85994020cf 100644 --- a/src/upsert.c +++ b/src/upsert.c @@ -166,6 +166,7 @@ int sqlite3UpsertAnalyzeTarget( if( pIdx->aiColumn[ii]==XN_EXPR ){ assert( pIdx->aColExpr!=0 ); assert( pIdx->aColExpr->nExpr>ii ); + assert( pIdx->bHasExpr ); pExpr = pIdx->aColExpr->a[ii].pExpr; if( pExpr->op!=TK_COLLATE ){ sCol[0].pLeft = pExpr; diff --git a/src/vacuum.c b/src/vacuum.c index 0e1eedab7e..967b6323f8 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -161,6 +161,7 @@ SQLITE_NOINLINE int sqlite3RunVacuum( int nDb; /* Number of attached databases */ const char *zDbMain; /* Schema name of database to vacuum */ const char *zOut; /* Name of output file */ + u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -232,12 +233,17 @@ SQLITE_NOINLINE int sqlite3RunVacuum( goto end_of_vacuum; } db->mDbFlags |= DBFLAG_VacuumInto; + + /* For a VACUUM INTO, the pager-flags are set to the same values as + ** they are for the database being vacuumed, except that PAGER_CACHESPILL + ** is always set. */ + pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); } nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); - sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL); + sqlite3BtreeSetPagerFlags(pTemp, pgflags|PAGER_CACHESPILL); /* Begin a transaction and take an exclusive lock on the main database ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, diff --git a/src/vdbe.c b/src/vdbe.c index e58ba74cf9..c4d13be7b0 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2588,19 +2588,90 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ break; } -/* Opcode: IsNullOrType P1 P2 P3 * * -** Synopsis: if typeof(r[P1]) IN (P3,5) goto P2 +/* Opcode: IsType P1 P2 P3 P4 P5 +** Synopsis: if typeof(P1.P3) in P5 goto P2 +** +** Jump to P2 if the type of a column in a btree is one of the types specified +** by the P5 bitmask. +** +** P1 is normally a cursor on a btree for which the row decode cache is +** valid through at least column P3. In other words, there should have been +** a prior OP_Column for column P3 or greater. If the cursor is not valid, +** then this opcode might give spurious results. +** The the btree row has fewer than P3 columns, then use P4 as the +** datatype. +** +** If P1 is -1, then P3 is a register number and the datatype is taken +** from the value in that register. +** +** P5 is a bitmask of data types. SQLITE_INTEGER is the least significant +** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04. +** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10. +** +** Take the jump to address P2 if and only if the datatype of the +** value determined by P1 and P3 corresponds to one of the bits in the +** P5 bitmask. ** -** Jump to P2 if the value in register P1 is NULL or has a datatype P3. -** P3 is an integer which should be one of SQLITE_INTEGER, SQLITE_FLOAT, -** SQLITE_BLOB, SQLITE_NULL, or SQLITE_TEXT. */ -case OP_IsNullOrType: { /* jump, in1 */ - int doTheJump; - pIn1 = &aMem[pOp->p1]; - doTheJump = (pIn1->flags & MEM_Null)!=0 || sqlite3_value_type(pIn1)==pOp->p3; - VdbeBranchTaken( doTheJump, 2); - if( doTheJump ) goto jump_to_p2; +case OP_IsType: { /* jump */ + VdbeCursor *pC; + u16 typeMask; + u32 serialType; + + assert( pOp->p1>=(-1) && pOp->p1nCursor ); + assert( pOp->p1>=0 || (pOp->p3>=0 && pOp->p3<=(p->nMem+1 - p->nCursor)) ); + if( pOp->p1>=0 ){ + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p3>=0 ); + if( pOp->p3nHdrParsed ){ + serialType = pC->aType[pOp->p3]; + if( serialType>=12 ){ + if( serialType&1 ){ + typeMask = 0x04; /* SQLITE_TEXT */ + }else{ + typeMask = 0x08; /* SQLITE_BLOB */ + } + }else{ + static const unsigned char aMask[] = { + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x2, + 0x01, 0x01, 0x10, 0x10 + }; + testcase( serialType==0 ); + testcase( serialType==1 ); + testcase( serialType==2 ); + testcase( serialType==3 ); + testcase( serialType==4 ); + testcase( serialType==5 ); + testcase( serialType==6 ); + testcase( serialType==7 ); + testcase( serialType==8 ); + testcase( serialType==9 ); + testcase( serialType==10 ); + testcase( serialType==11 ); + typeMask = aMask[serialType]; + } + }else{ + typeMask = 1 << (pOp->p4.i - 1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + }else{ + assert( memIsValid(&aMem[pOp->p3]) ); + typeMask = 1 << (sqlite3_value_type((sqlite3_value*)&aMem[pOp->p3])-1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + VdbeBranchTaken( (typeMask & pOp->p5)!=0, 2); + if( typeMask & pOp->p5 ){ + goto jump_to_p2; + } break; } @@ -2701,7 +2772,7 @@ case OP_Offset: { /* out3 */ ** Interpret the data that cursor P1 points to as a structure built using ** the MakeRecord instruction. (See the MakeRecord opcode for additional ** information about the format of the data.) Extract the P2-th column -** from this record. If there are less that (P2+1) +** from this record. If there are less than (P2+1) ** values in the record, extract a NULL. ** ** The value extracted is stored in register P3. @@ -2710,10 +2781,12 @@ case OP_Offset: { /* out3 */ ** if the P4 argument is a P4_MEM use the value of the P4 argument as ** the result. ** -** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 then -** the result is guaranteed to only be used as the argument of a length() -** or typeof() function, respectively. The loading of large blobs can be -** skipped for length() and all content loading can be skipped for typeof(). +** If the OPFLAG_LENGTHARG bit is set in P5 then the result is guaranteed +** to only be used by the length() function or the equivalent. The content +** of large blobs is not loaded, thus saving CPU cycles. If the +** OPFLAG_TYPEOFARG bit is set then the result will only be used by the +** typeof() function or the IS NULL or IS NOT NULL operators or the +** equivalent. In this case, all content loading can be omitted. */ case OP_Column: { u32 p2; /* column number to retrieve */ @@ -4688,7 +4761,13 @@ case OP_SeekGT: { /* jump, in3, group */ r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG - { int i; for(i=0; i0 ) REGISTER_TRACE(pOp->p3+i, &r.aMem[i]); + } + } #endif r.eqSeen = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &res); @@ -4751,7 +4830,7 @@ seek_not_found: } -/* Opcode: SeekScan P1 P2 * * * +/* Opcode: SeekScan P1 P2 * * P5 ** Synopsis: Scan-ahead up to P1 rows ** ** This opcode is a prefix opcode to OP_SeekGE. In other words, this @@ -4761,8 +4840,8 @@ seek_not_found: ** This opcode uses the P1 through P4 operands of the subsequent ** OP_SeekGE. In the text that follows, the operands of the subsequent ** OP_SeekGE opcode are denoted as SeekOP.P1 through SeekOP.P4. Only -** the P1 and P2 operands of this opcode are also used, and are called -** This.P1 and This.P2. +** the P1, P2 and P5 operands of this opcode are also used, and are called +** This.P1, This.P2 and This.P5. ** ** This opcode helps to optimize IN operators on a multi-column index ** where the IN operator is on the later terms of the index by avoiding @@ -4772,29 +4851,51 @@ seek_not_found: ** ** The SeekGE.P3 and SeekGE.P4 operands identify an unpacked key which ** is the desired entry that we want the cursor SeekGE.P1 to be pointing -** to. Call this SeekGE.P4/P5 row the "target". +** to. Call this SeekGE.P3/P4 row the "target". ** ** If the SeekGE.P1 cursor is not currently pointing to a valid row, ** then this opcode is a no-op and control passes through into the OP_SeekGE. ** ** If the SeekGE.P1 cursor is pointing to a valid row, then that row ** might be the target row, or it might be near and slightly before the -** target row. This opcode attempts to position the cursor on the target -** row by, perhaps by invoking sqlite3BtreeStep() on the cursor -** between 0 and This.P1 times. +** target row, or it might be after the target row. If the cursor is +** currently before the target row, then this opcode attempts to position +** the cursor on or after the target row by invoking sqlite3BtreeStep() +** on the cursor between 1 and This.P1 times. ** -** There are three possible outcomes from this opcode:
      +** The This.P5 parameter is a flag that indicates what to do if the +** cursor ends up pointing at a valid row that is past the target +** row. If This.P5 is false (0) then a jump is made to SeekGE.P2. If +** This.P5 is true (non-zero) then a jump is made to This.P2. The P5==0 +** case occurs when there are no inequality constraints to the right of +** the IN constraing. The jump to SeekGE.P2 ends the loop. The P5!=0 case +** occurs when there are inequality constraints to the right of the IN +** operator. In that case, the This.P2 will point either directly to or +** to setup code prior to the OP_IdxGT or OP_IdxGE opcode that checks for +** loop terminate. ** -**
    1. If after This.P1 steps, the cursor is still pointing to a place that -** is earlier in the btree than the target row, then fall through -** into the subsquence OP_SeekGE opcode. +** Possible outcomes from this opcode:
        ** -**
      1. If the cursor is successfully moved to the target row by 0 or more -** sqlite3BtreeNext() calls, then jump to This.P2, which will land just -** past the OP_IdxGT or OP_IdxGE opcode that follows the OP_SeekGE. +**
      2. If the cursor is initally not pointed to any valid row, then +** fall through into the subsequent OP_SeekGE opcode. ** -**
      3. If the cursor ends up past the target row (indicating that the target -** row does not exist in the btree) then jump to SeekOP.P2. +**
      4. If the cursor is left pointing to a row that is before the target +** row, even after making as many as This.P1 calls to +** sqlite3BtreeNext(), then also fall through into OP_SeekGE. +** +**
      5. If the cursor is left pointing at the target row, either because it +** was at the target row to begin with or because one or more +** sqlite3BtreeNext() calls moved the cursor to the target row, +** then jump to This.P2.., +** +**
      6. If the cursor started out before the target row and a call to +** to sqlite3BtreeNext() moved the cursor off the end of the index +** (indicating that the target row definitely does not exist in the +** btree) then jump to SeekGE.P2, ending the loop. +** +**
      7. If the cursor ends up on a valid row that is past the target row +** (indicating that the target row does not exist in the btree) then +** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0. **
      */ case OP_SeekScan: { @@ -4805,14 +4906,25 @@ case OP_SeekScan: { assert( pOp[1].opcode==OP_SeekGE ); - /* pOp->p2 points to the first instruction past the OP_IdxGT that - ** follows the OP_SeekGE. */ + /* If pOp->p5 is clear, then pOp->p2 points to the first instruction past the + ** OP_IdxGT that follows the OP_SeekGE. Otherwise, it points to the first + ** opcode past the OP_SeekGE itself. */ assert( pOp->p2>=(int)(pOp-aOp)+2 ); - assert( aOp[pOp->p2-1].opcode==OP_IdxGT || aOp[pOp->p2-1].opcode==OP_IdxGE ); - testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); - assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); - assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); - assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); +#ifdef SQLITE_DEBUG + if( pOp->p5==0 ){ + /* There are no inequality constraints following the IN constraint. */ + assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); + assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); + assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); + assert( aOp[pOp->p2-1].opcode==OP_IdxGT + || aOp[pOp->p2-1].opcode==OP_IdxGE ); + testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); + }else{ + /* There are inequality constraints. */ + assert( pOp->p2==(int)(pOp-aOp)+2 ); + assert( aOp[pOp->p2-1].opcode==OP_SeekGE ); + } +#endif assert( pOp->p1>0 ); pC = p->apCsr[pOp[1].p1]; @@ -4846,8 +4958,9 @@ case OP_SeekScan: { while(1){ rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res); if( rc ) goto abort_due_to_error; - if( res>0 ){ + if( res>0 && pOp->p5==0 ){ seekscan_search_fail: + /* Jump to SeekGE.P2, ending the loop */ #ifdef SQLITE_DEBUG if( db->flags&SQLITE_VdbeTrace ){ printf("... %d steps and then skip\n", pOp->p1 - nStep); @@ -4857,7 +4970,8 @@ case OP_SeekScan: { pOp++; goto jump_to_p2; } - if( res==0 ){ + if( res>=0 ){ + /* Jump to This.P2, bypassing the OP_SeekGE opcode */ #ifdef SQLITE_DEBUG if( db->flags&SQLITE_VdbeTrace ){ printf("... %d steps and then success\n", pOp->p1 - nStep); diff --git a/src/vdbe.h b/src/vdbe.h index eb1445f1db..a244468b52 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -228,6 +228,7 @@ void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); void sqlite3VdbeChangeP5(Vdbe*, u16 P5); +void sqlite3VdbeTypeofColumn(Vdbe*, int); void sqlite3VdbeJumpHere(Vdbe*, int addr); void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr); int sqlite3VdbeChangeToNoop(Vdbe*, int addr); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 6c5f7562b3..04295342b9 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -318,6 +318,9 @@ int sqlite3_value_type(sqlite3_value* pVal){ #endif return aType[pVal->flags&MEM_AffMask]; } +int sqlite3_value_encoding(sqlite3_value *pVal){ + return pVal->enc; +} /* Return true if a parameter to xUpdate represents an unchanged column */ int sqlite3_value_nochange(sqlite3_value *pVal){ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 2ff02c82e9..08c08e1327 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1155,6 +1155,18 @@ void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; } +/* +** If the previous opcode is an OP_Column that delivers results +** into register iDest, then add the OPFLAG_TYPEOFARG flag to that +** opcode. +*/ +void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){ + VdbeOp *pOp = sqlite3VdbeGetLastOp(p); + if( pOp->p3==iDest && pOp->opcode==OP_Column ){ + pOp->p5 |= OPFLAG_TYPEOFARG; + } +} + /* ** Change the P2 operand of instruction addr so that it points to ** the address of the next instruction to be coded. @@ -4583,7 +4595,7 @@ int sqlite3VdbeRecordCompareWithSkip( assert( pPKey2->pKeyInfo->aSortFlags!=0 ); assert( pPKey2->pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); - do{ + while( 1 /*exit-by-break*/ ){ u32 serial_type; /* RHS is an integer */ @@ -4593,7 +4605,7 @@ int sqlite3VdbeRecordCompareWithSkip( serial_type = aKey1[idx1]; testcase( serial_type==12 ); if( serial_type>=10 ){ - rc = +1; + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ @@ -4618,7 +4630,7 @@ int sqlite3VdbeRecordCompareWithSkip( ** numbers). Types 10 and 11 are currently "reserved for future ** use", so it doesn't really matter what the results of comparing ** them to numberic values are. */ - rc = +1; + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else{ @@ -4699,7 +4711,7 @@ int sqlite3VdbeRecordCompareWithSkip( /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0); + rc = (serial_type!=0 && serial_type!=10); } if( rc!=0 ){ @@ -4721,8 +4733,13 @@ int sqlite3VdbeRecordCompareWithSkip( if( i==pPKey2->nField ) break; pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); + if( d1>(unsigned)nKey1 ) break; idx1 += sqlite3VarintLen(serial_type); - }while( idx1<(unsigned)szHdr1 && d1<=(unsigned)nKey1 ); + if( idx1>=(unsigned)szHdr1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corrupt index */ + } + } /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a diff --git a/src/vdbemem.c b/src/vdbemem.c index c6c938cb5c..f14599def6 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -832,6 +832,7 @@ int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); + if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; return sqlite3VdbeChangeEncoding(pMem, encoding); } } @@ -1967,6 +1968,9 @@ int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ return p->n; } + if( (p->flags & MEM_Str)!=0 && enc!=SQLITE_UTF8 && pVal->enc!=SQLITE_UTF8 ){ + return p->n; + } if( (p->flags & MEM_Blob)!=0 ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; diff --git a/src/vtab.c b/src/vtab.c index 9bf031faf0..3814b0355e 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -1141,7 +1141,7 @@ FuncDef *sqlite3VtabOverloadFunction( if( pExpr->op!=TK_COLUMN ) return pDef; assert( ExprUseYTab(pExpr) ); pTab = pExpr->y.pTab; - if( pTab==0 ) return pDef; + if( NEVER(pTab==0) ) return pDef; if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); diff --git a/src/where.c b/src/where.c index b0c0ea7d4e..7ba72d2766 100644 --- a/src/where.c +++ b/src/where.c @@ -2287,7 +2287,6 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ pWInfo->pLoops = p->pNextLoop; whereLoopDelete(db, p); } - assert( pWInfo->pExprMods==0 ); while( pWInfo->pMemToFree ){ WhereMemBlock *pNext = pWInfo->pMemToFree->pNext; sqlite3DbNNFreeNN(db, pWInfo->pMemToFree); @@ -2296,17 +2295,6 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ sqlite3DbNNFreeNN(db, pWInfo); } -/* Undo all Expr node modifications -*/ -static void whereUndoExprMods(WhereInfo *pWInfo){ - while( pWInfo->pExprMods ){ - WhereExprMod *p = pWInfo->pExprMods; - pWInfo->pExprMods = p->pNext; - memcpy(p->pExpr, &p->orig, sizeof(p->orig)); - sqlite3DbFree(pWInfo->pParse->db, p); - } -} - /* ** Return TRUE if all of the following are true: ** @@ -3259,6 +3247,94 @@ static int whereUsablePartialIndex( return 0; } +/* +** Structure passed to the whereIsCoveringIndex Walker callback. +*/ +struct CoveringIndexCheck { + Index *pIdx; /* The index */ + int iTabCur; /* Cursor number for the corresponding table */ +}; + +/* +** Information passed in is pWalk->u.pCovIdxCk. Call is pCk. +** +** If the Expr node references the table with cursor pCk->iTabCur, then +** make sure that column is covered by the index pCk->pIdx. We know that +** all columns less than 63 (really BMS-1) are covered, so we don't need +** to check them. But we do need to check any column at 63 or greater. +** +** If the index does not cover the column, then set pWalk->eCode to +** non-zero and return WRC_Abort to stop the search. +** +** If this node does not disprove that the index can be a covering index, +** then just return WRC_Continue, to continue the search. +*/ +static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->iColumn<(BMS-1) ) return WRC_Continue; + if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; iiColumn ) return WRC_Continue; + } + pWalk->eCode = 1; + return WRC_Abort; +} + + +/* +** pIdx is an index that covers all of the low-number columns used by +** pWInfo->pSelect (columns from 0 through 62). But there are columns +** in pWInfo->pSelect beyond 62. This routine tries to answer the question +** of whether pIdx covers *all* columns in the query. +** +** Return 0 if pIdx is a covering index. Return non-zero if pIdx is +** not a covering index or if we are unable to determine if pIdx is a +** covering index. +** +** This routine is an optimization. It is always safe to return non-zero. +** But returning zero when non-zero should have been returned can lead to +** incorrect bytecode and assertion faults. +*/ +static SQLITE_NOINLINE u32 whereIsCoveringIndex( + WhereInfo *pWInfo, /* The WHERE clause context */ + Index *pIdx, /* Index that is being tested */ + int iTabCur /* Cursor for the table being indexed */ +){ + int i; + struct CoveringIndexCheck ck; + Walker w; + if( pWInfo->pSelect==0 ){ + /* We don't have access to the full query, so we cannot check to see + ** if pIdx is covering. Assume it is not. */ + return 1; + } + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 1; + } + ck.pIdx = pIdx; + ck.iTabCur = iTabCur; + memset(&w, 0, sizeof(w)); + w.xExprCallback = whereIsCoveringIndexWalkCallback; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pCovIdxCk = &ck; + w.eCode = 0; + sqlite3WalkSelect(&w, pWInfo->pSelect); + return w.eCode; +} + /* ** Add all WhereLoop objects for a single table of the join where the table ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be @@ -3476,6 +3552,9 @@ static int whereLoopAddBtree( m = 0; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; + if( m==TOPBIT ){ + m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + } pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } @@ -4701,7 +4780,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int mxChoice; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ Parse *pParse; /* Parsing context */ - sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ @@ -4720,7 +4798,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int nSpace; /* Bytes of space allocated at pSpace */ pParse = pWInfo->pParse; - db = pParse->db; nLoop = pWInfo->nLevel; /* TUNING: For simple queries, only the best path is tracked. ** For 2-way joins, the 5 best paths are followed. @@ -4743,7 +4820,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRawNN(db, nSpace); + pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace); if( pSpace==0 ) return SQLITE_NOMEM_BKPT; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; @@ -5001,7 +5078,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( nFrom==0 ){ sqlite3ErrorMsg(pParse, "no query solution"); - sqlite3DbFreeNN(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_ERROR; } @@ -5083,8 +5160,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ - assert( db!=0 ); - sqlite3DbNNFreeNN(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_OK; } @@ -5383,6 +5459,77 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } } +/* +** This is an sqlite3ParserAddCleanup() callback that is invoked to +** free the Parse->pIdxExpr list when the Parse object is destroyed. +*/ +static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ + Parse *pParse = (Parse*)pObject; + while( pParse->pIdxExpr!=0 ){ + IndexedExpr *p = pParse->pIdxExpr; + pParse->pIdxExpr = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } +} + +/* +** The index pIdx is used by a query and contains one or more expressions. +** In other words pIdx is an index on an expression. iIdxCur is the cursor +** number for the index and iDataCur is the cursor number for the corresponding +** table. +** +** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for +** each of the expressions in the index so that the expression code generator +** will know to replace occurrences of the indexed expression with +** references to the corresponding column of the index. +*/ +static SQLITE_NOINLINE void whereAddIndexedExpr( + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */ + Index *pIdx, /* The index-on-expression that contains the expressions */ + int iIdxCur, /* Cursor number for pIdx */ + SrcItem *pTabItem /* The FROM clause entry for the table */ +){ + int i; + IndexedExpr *p; + Table *pTab; + assert( pIdx->bHasExpr ); + pTab = pIdx->pTable; + for(i=0; inColumn; i++){ + Expr *pExpr; + int j = pIdx->aiColumn[i]; + int bMaybeNullRow; + if( j==XN_EXPR ){ + pExpr = pIdx->aColExpr->a[i].pExpr; + testcase( pTabItem->fg.jointype & JT_LEFT ); + testcase( pTabItem->fg.jointype & JT_RIGHT ); + testcase( pTabItem->fg.jointype & JT_LTORJ ); + bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; + }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ + pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); + bMaybeNullRow = 0; + }else{ + continue; + } + if( sqlite3ExprIsConstant(pExpr) ) continue; + p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); + if( p==0 ) break; + p->pIENext = pParse->pIdxExpr; + p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p->iDataCur = pTabItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = i; + p->bMaybeNullRow = bMaybeNullRow; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + p->zIdxName = pIdx->zName; +#endif + pParse->pIdxExpr = p; + if( p->pIENext==0 ){ + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); + } + } +} + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -5477,7 +5624,7 @@ WhereInfo *sqlite3WhereBegin( Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ - Select *pLimit, /* Use this LIMIT/OFFSET clause, if any */ + Select *pSelect, /* The entire SELECT statement */ u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number ** If WHERE_USE_LIMIT, then the limit amount */ @@ -5546,7 +5693,9 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; +#if WHERETRACE_ENABLED pWInfo->pWhere = pWhere; +#endif pWInfo->pResultSet = pResultSet; pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; @@ -5554,9 +5703,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->wctrlFlags = wctrlFlags; pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; -#ifndef SQLITE_OMIT_VIRTUALTABLE - pWInfo->pLimit = pLimit; -#endif + pWInfo->pSelect = pSelect; memset(&pWInfo->nOBSat, 0, offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); @@ -5625,7 +5772,9 @@ WhereInfo *sqlite3WhereBegin( /* Analyze all of the subexpressions. */ sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); - sqlite3WhereAddLimit(&pWInfo->sWC, pLimit); + if( pSelect && pSelect->pLimit ){ + sqlite3WhereAddLimit(&pWInfo->sWC, pSelect); + } if( pParse->nErr ) goto whereBeginError; /* Special case: WHERE terms that do not refer to any tables in the join @@ -5928,6 +6077,9 @@ WhereInfo *sqlite3WhereBegin( op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; + if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){ + whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); + } } pLevel->iIdxCur = iIndexCur; assert( pIx!=0 ); @@ -6050,8 +6202,6 @@ WhereInfo *sqlite3WhereBegin( /* Jump here if malloc fails */ whereBeginError: if( pWInfo ){ - testcase( pWInfo->pExprMods!=0 ); - whereUndoExprMods(pWInfo); pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); } @@ -6270,7 +6420,6 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } assert( pWInfo->nLevel<=pTabList->nSrc ); - if( pWInfo->pExprMods ) whereUndoExprMods(pWInfo); for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ int k, last; VdbeOp *pOp, *pLastOp; @@ -6324,6 +6473,16 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ }else{ last = pWInfo->iEndWhere; } + if( pIdx->bHasExpr ){ + IndexedExpr *p = pParse->pIdxExpr; + while( p ){ + if( p->iIdxCur==pLevel->iIdxCur ){ + p->iDataCur = -1; + p->iIdxCur = -1; + } + p = p->pIENext; + } + } k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ diff --git a/src/whereInt.h b/src/whereInt.h index fda207890c..28ede5be66 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -378,7 +378,7 @@ struct WhereAndInfo { ** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. ** ** The VDBE cursor numbers are small integers contained in -** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** SrcItem.iCursor and Expr.iTable fields. For any given WHERE ** clause, the cursor numbers might not begin with 0 and they might ** contain gaps in the numbering sequence. But we want to make maximum ** use of the bits in our bitmasks. This structure provides a mapping @@ -449,20 +449,6 @@ struct WhereLoopBuilder { # define SQLITE_QUERY_PLANNER_LIMIT_INCR 1000 #endif -/* -** Each instance of this object records a change to a single node -** in an expression tree to cause that node to point to a column -** of an index rather than an expression or a virtual column. All -** such transformations need to be undone at the end of WHERE clause -** processing. -*/ -typedef struct WhereExprMod WhereExprMod; -struct WhereExprMod { - WhereExprMod *pNext; /* Next translation on a list of them all */ - Expr *pExpr; /* The Expr node that was transformed */ - Expr orig; /* Original value of the Expr node */ -}; - /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second @@ -478,10 +464,10 @@ struct WhereInfo { SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ ExprList *pResultSet; /* Result set of the query */ +#if WHERETRACE_ENABLED Expr *pWhere; /* The complete WHERE clause */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - Select *pLimit; /* Used to access LIMIT expr/registers for vtabs */ #endif + Select *pSelect; /* The entire SELECT statement containing WHERE */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ @@ -500,7 +486,6 @@ struct WhereInfo { int iTop; /* The very beginning of the WHERE loop */ int iEndWhere; /* End of the WHERE clause itself */ WhereLoop *pLoops; /* List of all WhereLoop objects */ - WhereExprMod *pExprMods; /* Expression modifications */ WhereMemBlock *pMemToFree;/* Memory to free when this object destroyed */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereClause sWC; /* Decomposition of the WHERE clause */ diff --git a/src/wherecode.c b/src/wherecode.c index 4c34ea0dc6..e36d1c9964 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1217,143 +1217,6 @@ static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ } } -/* An instance of the IdxExprTrans object carries information about a -** mapping from an expression on table columns into a column in an index -** down through the Walker. -*/ -typedef struct IdxExprTrans { - Expr *pIdxExpr; /* The index expression */ - int iTabCur; /* The cursor of the corresponding table */ - int iIdxCur; /* The cursor for the index */ - int iIdxCol; /* The column for the index */ - int iTabCol; /* The column for the table */ - WhereInfo *pWInfo; /* Complete WHERE clause information */ - sqlite3 *db; /* Database connection (for malloc()) */ -} IdxExprTrans; - -/* -** Preserve pExpr on the WhereETrans list of the WhereInfo. -*/ -static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){ - WhereExprMod *pNew; - pNew = sqlite3DbMallocRaw(pTrans->db, sizeof(*pNew)); - if( pNew==0 ) return; - pNew->pNext = pTrans->pWInfo->pExprMods; - pTrans->pWInfo->pExprMods = pNew; - pNew->pExpr = pExpr; - memcpy(&pNew->orig, pExpr, sizeof(*pExpr)); -} - -/* The walker node callback used to transform matching expressions into -** a reference to an index column for an index on an expression. -** -** If pExpr matches, then transform it into a reference to the index column -** that contains the value of pExpr. -*/ -static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ - IdxExprTrans *pX = p->u.pIdxTrans; - if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ - pExpr = sqlite3ExprSkipCollate(pExpr); - preserveExpr(pX, pExpr); - pExpr->affExpr = sqlite3ExprAffinity(pExpr); - pExpr->op = TK_COLUMN; - pExpr->iTable = pX->iIdxCur; - pExpr->iColumn = pX->iIdxCol; - testcase( ExprHasProperty(pExpr, EP_Unlikely) ); - ExprClearProperty(pExpr, EP_Skip|EP_Unlikely|EP_WinFunc|EP_Subrtn); - pExpr->y.pTab = 0; - return WRC_Prune; - }else{ - return WRC_Continue; - } -} - -#ifndef SQLITE_OMIT_GENERATED_COLUMNS -/* A walker node callback that translates a column reference to a table -** into a corresponding column reference of an index. -*/ -static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){ - if( pExpr->op==TK_COLUMN ){ - IdxExprTrans *pX = p->u.pIdxTrans; - if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){ - assert( ExprUseYTab(pExpr) && pExpr->y.pTab!=0 ); - preserveExpr(pX, pExpr); - pExpr->affExpr = sqlite3TableColumnAffinity(pExpr->y.pTab,pExpr->iColumn); - pExpr->iTable = pX->iIdxCur; - pExpr->iColumn = pX->iIdxCol; - pExpr->y.pTab = 0; - } - } - return WRC_Continue; -} -#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ - -/* -** For an indexes on expression X, locate every instance of expression X -** in pExpr and change that subexpression into a reference to the appropriate -** column of the index. -** -** 2019-10-24: Updated to also translate references to a VIRTUAL column in -** the table into references to the corresponding (stored) column of the -** index. -*/ -static void whereIndexExprTrans( - Index *pIdx, /* The Index */ - int iTabCur, /* Cursor of the table that is being indexed */ - int iIdxCur, /* Cursor of the index itself */ - WhereInfo *pWInfo /* Transform expressions in this WHERE clause */ -){ - int iIdxCol; /* Column number of the index */ - ExprList *aColExpr; /* Expressions that are indexed */ - Table *pTab; - Walker w; - IdxExprTrans x; - aColExpr = pIdx->aColExpr; - if( aColExpr==0 && !pIdx->bHasVCol ){ - /* The index does not reference any expressions or virtual columns - ** so no translations are needed. */ - return; - } - pTab = pIdx->pTable; - memset(&w, 0, sizeof(w)); - w.u.pIdxTrans = &x; - x.iTabCur = iTabCur; - x.iIdxCur = iIdxCur; - x.pWInfo = pWInfo; - x.db = pWInfo->pParse->db; - for(iIdxCol=0; iIdxColnColumn; iIdxCol++){ - i16 iRef = pIdx->aiColumn[iIdxCol]; - if( iRef==XN_EXPR ){ - assert( aColExpr!=0 && aColExpr->a[iIdxCol].pExpr!=0 ); - x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; - if( sqlite3ExprIsConstant(x.pIdxExpr) ) continue; - w.xExprCallback = whereIndexExprTransNode; -#ifndef SQLITE_OMIT_GENERATED_COLUMNS - }else if( iRef>=0 - && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0 - && ((pTab->aCol[iRef].colFlags & COLFLAG_HASCOLL)==0 - || sqlite3StrICmp(sqlite3ColumnColl(&pTab->aCol[iRef]), - sqlite3StrBINARY)==0) - ){ - /* Check to see if there are direct references to generated columns - ** that are contained in the index. Pulling the generated column - ** out of the index is an optimization only - the main table is always - ** available if the index cannot be used. To avoid unnecessary - ** complication, omit this optimization if the collating sequence for - ** the column is non-standard */ - x.iTabCol = iRef; - w.xExprCallback = whereIndexExprTransColumn; -#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ - }else{ - continue; - } - x.iIdxCol = iIdxCol; - sqlite3WalkExpr(&w, pWInfo->pWhere); - sqlite3WalkExprList(&w, pWInfo->pOrderBy); - sqlite3WalkExprList(&w, pWInfo->pResultSet); - } -} - /* ** The pTruth expression is always true because it is the WHERE clause ** a partial index that is driving a query loop. Look through all of the @@ -1422,6 +1285,8 @@ static SQLITE_NOINLINE void filterPullDown( testcase( pTerm->wtFlags & TERM_VIRTUAL ); regRowid = sqlite3GetTempReg(pParse); regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, regRowid); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MustBeInt, regRowid, addrNxt); + VdbeCoverage(pParse->pVdbe); sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter, addrNxt, regRowid, 1); VdbeCoverage(pParse->pVdbe); @@ -1573,9 +1438,9 @@ Bitmask sqlite3WhereCodeOneLoopStart( && pLoop->u.vtab.bOmitOffset ){ assert( pTerm->eOperator==WO_AUX ); - assert( pWInfo->pLimit!=0 ); - assert( pWInfo->pLimit->iOffset>0 ); - sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pLimit->iOffset); + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->iOffset>0 ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pSelect->iOffset); VdbeComment((v,"Zero OFFSET counter")); } } @@ -1683,6 +1548,8 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; if( pLevel->regFilter ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + VdbeCoverage(v); sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, iRowidReg, 1); VdbeCoverage(v); @@ -2034,6 +1901,11 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** guess. */ addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, (pIdx->aiRowLogEst[0]+9)/10); + if( pRangeStart ){ + sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1); + addrSeekScan = 0; + } VdbeCoverage(v); } sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); @@ -2172,27 +2044,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( } if( pLevel->iLeftJoin==0 ){ - /* If pIdx is an index on one or more expressions, then look through - ** all the expressions in pWInfo and try to transform matching expressions - ** into reference to index columns. Also attempt to translate references - ** to virtual columns in the table into references to (stored) columns - ** of the index. - ** - ** Do not do this for the RHS of a LEFT JOIN. This is because the - ** expression may be evaluated after OP_NullRow has been executed on - ** the cursor. In this case it is important to do the full evaluation, - ** as the result of the expression may not be NULL, even if all table - ** column values are. https://www.sqlite.org/src/info/7fa8049685b50b5a - ** - ** Also, do not do this when processing one index an a multi-index - ** OR clause, since the transformation will become invalid once we - ** move forward to the next index. - ** https://sqlite.org/src/info/4e8e4857d32d401f - */ - if( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0 ){ - whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); - } - /* If a partial index is driving the loop, try to eliminate WHERE clause ** terms from the query that must be true due to the WHERE clause of ** the partial index. @@ -2305,7 +2156,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, + pOrTab = sqlite3DbMallocRawNN(db, sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); if( pOrTab==0 ) return notReady; pOrTab->nAlloc = (u8)(nNotReady + 1); @@ -2558,7 +2409,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->op==OP_Return ); pLevel->p2 = sqlite3VdbeCurrentAddr(v); - if( pWInfo->nLevel>1 ){ sqlite3StackFree(db, pOrTab); } + if( pWInfo->nLevel>1 ){ sqlite3DbFreeNN(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ diff --git a/src/whereexpr.c b/src/whereexpr.c index 8c9233ebd3..24246c1aaf 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -266,7 +266,7 @@ static int isLikeOrGlob( if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT || (ALWAYS( ExprUseYTab(pLeft) ) - && pLeft->y.pTab + && ALWAYS(pLeft->y.pTab) && IsVirtual(pLeft->y.pTab)) /* Might be numeric */ ){ int isNum; @@ -383,8 +383,7 @@ static int isAuxiliaryVtabOperator( ** MATCH(expression,vtab_column) */ pCol = pList->a[1].pExpr; - assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); - testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); if( ExprIsVtab(pCol) ){ for(i=0; ia[0].pExpr; assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); - testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); if( ExprIsVtab(pCol) ){ sqlite3_vtab *pVtab; sqlite3_module *pMod; @@ -434,13 +433,12 @@ static int isAuxiliaryVtabOperator( int res = 0; Expr *pLeft = pExpr->pLeft; Expr *pRight = pExpr->pRight; - assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); - testcase( pLeft->op==TK_COLUMN && pLeft->y.pTab==0 ); + assert( pLeft->op!=TK_COLUMN || (ExprUseYTab(pLeft) && pLeft->y.pTab!=0) ); if( ExprIsVtab(pLeft) ){ res++; } - assert( pRight==0 || pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); - testcase( pRight && pRight->op==TK_COLUMN && pRight->y.pTab==0 ); + assert( pRight==0 || pRight->op!=TK_COLUMN + || (ExprUseYTab(pRight) && pRight->y.pTab!=0) ); if( pRight && ExprIsVtab(pRight) ){ res++; SWAP(Expr*, pLeft, pRight); @@ -989,6 +987,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR; @@ -1602,9 +1601,9 @@ static void whereAddLimitExpr( ** exist only so that they may be passed to the xBestIndex method of the ** single virtual table in the FROM clause of the SELECT. */ -void sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ - assert( p==0 || (p->pGroupBy==0 && (p->selFlags & SF_Aggregate)==0) ); - if( (p && p->pLimit) /* 1 */ +void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ + assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */ + if( p->pGroupBy==0 && (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */ && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab)) /* 3 */ ){ diff --git a/test/bloom1.test b/test/bloom1.test new file mode 100644 index 0000000000..12b53ddf14 --- /dev/null +++ b/test/bloom1.test @@ -0,0 +1,70 @@ +# 2022 October 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for queries that use bloom filters + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl + +set testprefix bloom1 + +# Tests 1.* verify that the bloom filter code correctly handles the +# case where the RHS of an ( = ?) expression must be coerced +# to an integer before the comparison made. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(c INTEGER PRIMARY KEY, d); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('hello', 'world'); + INSERT INTO t2 VALUES(14, 'fourteen'); +} + +do_execsql_test 1.2 { + ANALYZE sqlite_schema; + INSERT INTO sqlite_stat1 VALUES('t2','idx1','6 6'); + ANALYZE sqlite_schema; +} + +do_execsql_test 1.3 { + SELECT 'affinity!' FROM t1 CROSS JOIN t2 WHERE t2.c = '14'; +} {affinity!} + + +reset_db +do_execsql_test 1.4 { + CREATE TABLE t1(a, b TEXT); + CREATE TABLE t2(c INTEGER PRIMARY KEY, d); + CREATE TABLE t3(e INTEGER PRIMARY KEY, f); + + ANALYZE sqlite_schema; + INSERT INTO sqlite_stat1 VALUES('t1','idx1','600 6'); + INSERT INTO sqlite_stat1 VALUES('t2','idx1','6 6'); + INSERT INTO sqlite_stat1 VALUES('t3','idx2','6 6'); + ANALYZE sqlite_schema; + + INSERT INTO t1 VALUES(1, '123'); + INSERT INTO t2 VALUES(123, 'one'); + INSERT INTO t3 VALUES(123, 'two'); +} + +do_execsql_test 1.5 { + SELECT 'result' FROM t1, t2, t3 + WHERE t2.c=t1.b AND t2.d!='silly' + AND t3.e=t1.b AND t3.f!='silly' +} {result} + +finish_test + diff --git a/test/cast.test b/test/cast.test index 6ac9bc7efa..cbeec47c9d 100644 --- a/test/cast.test +++ b/test/cast.test @@ -481,7 +481,7 @@ do_execsql_test cast-9.0 { CREATE VIEW v1(c0, c1) AS SELECT CAST(0.0 AS NUMERIC), COUNT(*) OVER () FROM t0; SELECT v1.c0 FROM v1, t0 WHERE v1.c0=0; -} {0.0} +} {0} finish_test diff --git a/test/collate5.test b/test/collate5.test index 5f8697ea51..71d4efe255 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -19,6 +19,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix collate5 + # # Tests are organised as follows: @@ -26,6 +28,7 @@ source $testdir/tester.tcl # collate5-2.* - Compound SELECT # collate5-3.* - ORDER BY on compound SELECT # collate5-4.* - GROUP BY +# collate5-5.* - Collation sequence cases # Create the collation sequence 'TEXT', purely for asthetic reasons. The # test cases in this script could just as easily use BINARY. @@ -289,4 +292,37 @@ do_test collate5-4.3 { } } {} +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 5.0 { + CREATE TABLE t1(a, b COLLATE nocase); + CREATE TABLE t2(c, d); + INSERT INTO t2 VALUES(1, 'bbb'); +} +do_execsql_test 5.1 { + SELECT * FROM ( + SELECT a, b FROM t1 UNION ALL SELECT c, d FROM t2 + ) WHERE b='BbB'; +} {1 bbb} + +reset_db +do_execsql_test 5.2 { + CREATE TABLE t1(a,b,c COLLATE NOCASE); + INSERT INTO t1 VALUES(NULL,'C','c'); + CREATE VIEW v2 AS + SELECT a,b,c FROM t1 INTERSECT SELECT a,b,b FROM t1 + WHERE 'eT"3qRkL+oJMJjQ9z0'>=b + ORDER BY a,b,c; +} + +do_execsql_test 5.3 { + SELECT * FROM v2; +} { {} C c } + +do_execsql_test 5.4 { + SELECT * FROM v2 WHERE c='c'; +} { {} C c } + + finish_test diff --git a/test/corruptL.test b/test/corruptL.test index 7361a0b35e..3a841a8199 100644 --- a/test/corruptL.test +++ b/test/corruptL.test @@ -1479,13 +1479,26 @@ do_test 19.0 { do_execsql_test 19.1 { PRAGMA writable_schema=ON; } - -set err "UNIQUE constraint failed: index 'a'" -ifcapable oversize_cell_check { - set err "database disk image is malformed" -} do_catchsql_test 19.2 { UPDATE t1 SET a=1; -} [list 1 $err] +} {1 {database disk image is malformed}} + +reset_db +do_execsql_test 19.3 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c INTEGER, d TEXT); + CREATE INDEX i1 ON t1((NULL)); + INSERT INTO t1 VALUES(1, NULL, 1, 'text value'); + PRAGMA writable_schema = on; + UPDATE sqlite_schema SET + sql = 'CREATE INDEX i1 ON t1(b, c, d)', + tbl_name = 't1', + type='index' + WHERE name='i1'; +} +db close +sqlite3 db test.db +do_catchsql_test 19.4 { + PRAGMA integrity_check; +} {1 {database disk image is malformed}} finish_test diff --git a/test/e_wal.test b/test/e_wal.test index 77ac83a0ae..c9c5e9643f 100644 --- a/test/e_wal.test +++ b/test/e_wal.test @@ -15,6 +15,7 @@ source $testdir/tester.tcl set testprefix e_wal db close +forcedelete test.db-shm testvfs oldvfs -iversion 1 diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index fe56262211..a0bcc3bf07 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -85,6 +85,7 @@ #include #include #include "sqlite3.h" +#include "sqlite3recover.h" #define ISSPACE(X) isspace((unsigned char)(X)) #define ISDIGIT(X) isdigit((unsigned char)(X)) @@ -158,12 +159,10 @@ static struct GlobalVars { } g; /* -** Include the external vt02.c module, if requested by compile-time -** options. +** Include the external vt02.c module. */ -#ifdef VT02_SOURCES -# include "vt02.c" -#endif +extern int sqlite3_vt02_init(sqlite3*,char***,void*); + /* ** Print an error message and quit. @@ -629,6 +628,9 @@ static unsigned int oomCounter = 0; /* Simulate OOM when equals 1 */ static unsigned int oomRepeat = 0; /* Number of OOMs in a row */ static void*(*defaultMalloc)(int) = 0; /* The low-level malloc routine */ +/* Enable recovery */ +static int bNoRecover = 0; + /* This routine is called when a simulated OOM occurs. It is broken ** out as a separate routine to make it easy to set a breakpoint on ** the OOM @@ -969,7 +971,7 @@ static int block_troublesome_sql( } /* Implementation found in fuzzinvariant.c */ -int fuzz_invariant( +extern int fuzz_invariant( sqlite3 *db, /* The database connection */ sqlite3_stmt *pStmt, /* Test statement stopped on an SQLITE_ROW */ int iCnt, /* Invariant sequence number, starting at 0 */ @@ -979,6 +981,50 @@ int fuzz_invariant( int eVerbosity /* How much debugging output */ ); +/* Implementation of sqlite_dbdata and sqlite_dbptr */ +extern int sqlite3_dbdata_init(sqlite3*,const char**,void*); + + +/* +** This function is used as a callback by the recover extension. Simply +** print the supplied SQL statement to stdout. +*/ +static int recoverSqlCb(void *pCtx, const char *zSql){ + if( eVerbosity>=2 ){ + printf("%s\n", zSql); + } + return SQLITE_OK; +} + +/* +** This function is called to recover data from the database. +*/ +static int recoverDatabase(sqlite3 *db){ + int rc; /* Return code from this routine */ + const char *zLAF = "lost_and_found"; /* Name of "lost_and_found" table */ + int bFreelist = 1; /* True to scan the freelist */ + int bRowids = 1; /* True to restore ROWID values */ + sqlite3_recover *p; /* The recovery object */ + + p = sqlite3_recover_init_sql(db, "main", recoverSqlCb, 0); + sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); + sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); + sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); + sqlite3_recover_run(p); + if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(p); + int errCode = sqlite3_recover_errcode(p); + if( eVerbosity>0 ){ + printf("recovery error: %s (%d)\n", zErr, errCode); + } + } + rc = sqlite3_recover_finish(p); + if( eVerbosity>0 && rc ){ + printf("recovery returns error code %d\n", rc); + } + return rc; +} + /* ** Run the SQL text */ @@ -1189,9 +1235,12 @@ int runCombinedDbSqlInput( ** deserialize to do this because deserialize depends on ATTACH */ sqlite3_set_authorizer(cx.db, block_troublesome_sql, &btsFlags); -#ifdef VT02_SOURCES + /* Add the vt02 virtual table */ sqlite3_vt02_init(cx.db, 0, 0); -#endif + + /* Add support for sqlite_dbdata and sqlite_dbptr virtual tables used + ** by the recovery API */ + sqlite3_dbdata_init(cx.db, 0, 0); /* Consistent PRNG seed */ #ifdef SQLITE_TESTCTRL_PRNG_SEED @@ -1201,6 +1250,12 @@ int runCombinedDbSqlInput( sqlite3_randomness(0,0); #endif + /* Run recovery on the initial database, just to make sure recovery + ** works. */ + if( !bNoRecover ){ + recoverDatabase(cx.db); + } + zSql = sqlite3_malloc( nSql + 1 ); if( zSql==0 ){ fprintf(stderr, "Out of memory!\n"); @@ -1700,6 +1755,7 @@ static void showHelp(void){ " -m TEXT Add a description to the database\n" " --native-vfs Use the native VFS for initially empty database files\n" " --native-malloc Turn off MEMSYS3/5 and Lookaside\n" +" --no-recover Do not run recovery on dbsqlfuzz databases\n" " --oss-fuzz Enable OSS-FUZZ testing\n" " --prng-seed N Seed value for the PRGN inside of SQLite\n" " -q|--quiet Reduced output\n" @@ -1851,6 +1907,9 @@ int main(int argc, char **argv){ if( strcmp(z,"native-vfs")==0 ){ nativeFlag = 1; }else + if( strcmp(z,"no-recover")==0 ){ + bNoRecover = 1; + }else if( strcmp(z,"oss-fuzz")==0 ){ ossFuzz = 1; }else diff --git a/test/fuzzdata8.db b/test/fuzzdata8.db index de7799f945..aff7e27340 100644 Binary files a/test/fuzzdata8.db and b/test/fuzzdata8.db differ diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 90f1fad492..c0ed2dde58 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -29,7 +29,7 @@ /* Forward references */ static char *fuzz_invariant_sql(sqlite3_stmt*, int); -static int sameValue(sqlite3_stmt*,int,sqlite3_stmt*,int); +static int sameValue(sqlite3_stmt*,int,sqlite3_stmt*,int,sqlite3_stmt*); static void reportInvariantFailed(sqlite3_stmt*,sqlite3_stmt*,int); /* @@ -46,11 +46,14 @@ static void reportInvariantFailed(sqlite3_stmt*,sqlite3_stmt*,int); ** ** SQLITE_OK This check was successful. ** -** SQLITE_DONE iCnt is out of range. +** SQLITE_DONE iCnt is out of range. The caller typically sets +** up a loop on iCnt starting with zero, and increments +** iCnt until this code is returned. ** ** SQLITE_CORRUPT The invariant failed, but the underlying database ** file is indicating that it is corrupt, which might -** be the cause of the malfunction. +** be the cause of the malfunction. The *pCorrupt +** value will also be set. ** ** SQLITE_INTERNAL The invariant failed, and the database file is not ** corrupt. (This never happens because this function @@ -105,13 +108,16 @@ int fuzz_invariant( } while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){ for(i=0; i=nCol ) break; } if( rc==SQLITE_DONE ){ /* No matching output row found */ sqlite3_stmt *pCk = 0; + + /* This is not a fault if the database file is corrupt, because anything + ** can happen with a corrupt database file */ rc = sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &pCk, 0); if( rc ){ sqlite3_finalize(pCk); @@ -129,6 +135,7 @@ int fuzz_invariant( return SQLITE_CORRUPT; } sqlite3_finalize(pCk); + if( sqlite3_strlike("%group%by%order%by%desc%",sqlite3_sql(pStmt),0)==0 ){ /* dbsqlfuzz crash-647c162051c9b23ce091b7bbbe5125ce5f00e922 ** Original statement is: @@ -142,6 +149,46 @@ int fuzz_invariant( */ goto not_a_fault; } + + if( sqlite3_strlike("%limit%)%order%by%", sqlite3_sql(pTestStmt),0)==0 ){ + /* crash-89bd6a6f8c6166e9a4c5f47b3e70b225f69b76c6 + ** Original statement is: + ** + ** SELECT a,b,c* FROM t1 LIMIT 1%5<4 + ** + ** When running: + ** + ** SELECT * FROM (...) ORDER BY 1 + ** + ** A different subset of the rows come out + */ + goto not_a_fault; + } + + /* The original sameValue() comparison assumed a collating sequence + ** of "binary". It can sometimes get an incorrect result for different + ** collating sequences. So rerun the test with no assumptions about + ** collations. + */ + rc = sqlite3_prepare_v2(db, + "SELECT ?1=?2 OR ?1=?2 COLLATE nocase OR ?1=?2 COLLATE rtrim", + -1, &pCk, 0); + if( rc==SQLITE_OK ){ + sqlite3_reset(pTestStmt); + while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){ + for(i=0; i=nCol ){ + sqlite3_finalize(pCk); + goto not_a_fault; + } + } + } + sqlite3_finalize(pCk); + + /* Invariants do not necessarily work if there are virtual tables + ** involved in the query */ rc = sqlite3_prepare_v2(db, "SELECT 1 FROM bytecode(?1) WHERE opcode='VOpen'", -1, &pCk, 0); if( rc==SQLITE_OK ){ @@ -166,6 +213,24 @@ not_a_fault: ** Generate SQL used to test a statement invariant. ** ** Return 0 if the iCnt is out of range. +** +** iCnt meanings: +** +** 0 SELECT * FROM () +** 1 SELECT DISTINCT * FROM () +** 2 SELECT * FROM () WHERE ORDER BY 1 +** 3 SELECT DISTINCT * FROM () ORDER BY 1 +** 4 SELECT * FROM () WHERE = +** 5 SELECT DISTINCT * FROM () WHERE ) WHERE = ORDER BY 1 +** 7 SELECT DISTINCT * FROM () WHERE = +** ORDER BY 1 +** N+0 SELECT * FROM () WHERE = +** N+1 SELECT DISTINCT * FROM () WHERE = +** N+2 SELECT * FROM () WHERE = ORDER BY 1 +** N+3 SELECT DISTINCT * FROM () WHERE = +** ORDER BY N +** */ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ const char *zIn; @@ -182,7 +247,6 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ int bOrderBy = 0; int nParam = sqlite3_bind_parameter_count(pStmt); - iCnt++; switch( iCnt % 4 ){ case 1: bDistinct = 1; break; case 2: bOrderBy = 1; break; @@ -197,9 +261,10 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ while( nIn>0 && (isspace(zIn[nIn-1]) || zIn[nIn-1]==';') ) nIn--; if( strchr(zIn, '?') ) return 0; pTest = sqlite3_str_new(0); - sqlite3_str_appendf(pTest, "SELECT %s* FROM (%s", - bDistinct ? "DISTINCT " : "", zIn); - sqlite3_str_appendf(pTest, ")"); + sqlite3_str_appendf(pTest, "SELECT %s* FROM (", + bDistinct ? "DISTINCT " : ""); + sqlite3_str_append(pTest, zIn, (int)nIn); + sqlite3_str_append(pTest, ")", 1); rc = sqlite3_prepare_v2(db, sqlite3_str_value(pTest), -1, &pBase, 0); if( rc ){ sqlite3_finalize(pBase); @@ -216,7 +281,8 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ ** WHERE clause. */ continue; } - if( i+1!=iCnt ) continue; + if( iCnt==0 ) continue; + if( iCnt>1 && i+2!=iCnt ) continue; if( zColName==0 ) continue; if( sqlite3_column_type(pStmt, i)==SQLITE_NULL ){ sqlite3_str_appendf(pTest, " %s \"%w\" ISNULL", zAnd, zColName); @@ -228,7 +294,7 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ } if( pBase!=pStmt ) sqlite3_finalize(pBase); if( bOrderBy ){ - sqlite3_str_appendf(pTest, " ORDER BY 1"); + sqlite3_str_appendf(pTest, " ORDER BY %d", iCnt>2 ? iCnt-1 : 1); } return sqlite3_str_finish(pTest); } @@ -236,7 +302,11 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ /* ** Return true if and only if v1 and is the same as v2. */ -static int sameValue(sqlite3_stmt *pS1, int i1, sqlite3_stmt *pS2, int i2){ +static int sameValue( + sqlite3_stmt *pS1, int i1, /* Value to text on the left */ + sqlite3_stmt *pS2, int i2, /* Value to test on the right */ + sqlite3_stmt *pTestCompare /* COLLATE comparison statement or NULL */ +){ int x = 1; int t1 = sqlite3_column_type(pS1,i1); int t2 = sqlite3_column_type(pS2,i2); @@ -259,10 +329,38 @@ static int sameValue(sqlite3_stmt *pS1, int i1, sqlite3_stmt *pS2, int i2){ break; } case SQLITE_TEXT: { - const char *z1 = (const char*)sqlite3_column_text(pS1,i1); - const char *z2 = (const char*)sqlite3_column_text(pS2,i2); - x = ((z1==0 && z2==0) || (z1!=0 && z2!=0 && strcmp(z1,z1)==0)); - break; + int e1 = sqlite3_value_encoding(sqlite3_column_value(pS1,i1)); + int e2 = sqlite3_value_encoding(sqlite3_column_value(pS2,i2)); + if( e1!=e2 ){ + const char *z1 = (const char*)sqlite3_column_text(pS1,i1); + const char *z2 = (const char*)sqlite3_column_text(pS2,i2); + x = ((z1==0 && z2==0) || (z1!=0 && z2!=0 && strcmp(z1,z1)==0)); + printf("Encodings differ. %d on left and %d on right\n", e1, e2); + abort(); + } + if( pTestCompare ){ + sqlite3_bind_value(pTestCompare, 1, sqlite3_column_value(pS1,i1)); + sqlite3_bind_value(pTestCompare, 2, sqlite3_column_value(pS2,i2)); + x = sqlite3_step(pTestCompare)==SQLITE_ROW + && sqlite3_column_int(pTestCompare,0)!=0; + sqlite3_reset(pTestCompare); + break; + } + if( e1!=SQLITE_UTF8 ){ + int len1 = sqlite3_column_bytes16(pS1,i1); + const unsigned char *b1 = sqlite3_column_blob(pS1,i1); + int len2 = sqlite3_column_bytes16(pS2,i2); + const unsigned char *b2 = sqlite3_column_blob(pS2,i2); + if( len1!=len2 ){ + x = 0; + }else if( len1==0 ){ + x = 1; + }else{ + x = (b1!=0 && b2!=0 && memcmp(b1,b2,len1)==0); + } + break; + } + /* Fall through into the SQLITE_BLOB case */ } case SQLITE_BLOB: { int len1 = sqlite3_column_bytes(pS1,i1); @@ -282,11 +380,23 @@ static int sameValue(sqlite3_stmt *pS1, int i1, sqlite3_stmt *pS2, int i2){ return x; } +/* +** Print binary data as hex +*/ +static void printHex(const unsigned char *a, int n, int mx){ + int j; + for(j=0; j=6 AND a='abc' + ORDER BY a, b; +} { + abc 234 6 + abc 345 7 +} + +do_execsql_test 1.4 { + SELECT a,b,c FROM t1 + WHERE b IN (234, 345) AND c<=7 AND a='abc' + ORDER BY a, b; +} { + abc 234 6 + abc 345 7 +} + + +finish_test diff --git a/test/selectA.test b/test/selectA.test index ce7f8eba9c..7d72bb3fa5 100644 --- a/test/selectA.test +++ b/test/selectA.test @@ -1483,4 +1483,28 @@ do_execsql_test 8.1 { AND (0 OR (SELECT 'xyz' INTERSECT SELECT a ORDER BY 1)) } {} +#------------------------------------------------------------------------- +# dbsqlfuzz a34f455c91ad75a0cf8cd9476841903f42930a7a +# +reset_db +do_execsql_test 9.0 { + CREATE TABLE t1(a COLLATE nocase); + CREATE TABLE t2(b COLLATE nocase); + + INSERT INTO t1 VALUES('ABC'); + INSERT INTO t2 VALUES('abc'); +} + +do_execsql_test 9.1 { + SELECT a FROM t1 INTERSECT SELECT b FROM t2; +} {ABC} + +do_execsql_test 9.2 { + SELECT * FROM ( + SELECT a FROM t1 INTERSECT SELECT b FROM t2 + ) WHERE a||'' = 'ABC'; +} {ABC} + + + finish_test diff --git a/test/speedtest1.c b/test/speedtest1.c index b115a57e2b..12c8d69b5b 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -7,7 +7,8 @@ static const char zHelp[] = "Usage: %s [--options] DATABASE\n" "Options:\n" " --autovacuum Enable AUTOVACUUM mode\n" - " --cachesize N Set the cache size to N\n" + " --big-transactions Add BEGIN/END around all large tests\n" + " --cachesize N Set PRAGMA cache_size=N. Note: N is pages, not bytes\n" " --checkpoint Run PRAGMA wal_checkpoint after each test case\n" " --exclusive Enable locking_mode=EXCLUSIVE\n" " --explain Like --sqlonly but with added EXPLAIN keywords\n" @@ -20,6 +21,7 @@ static const char zHelp[] = " --mmap SZ MMAP the first SZ bytes of the database file\n" " --multithread Set multithreaded mode\n" " --nomemstat Disable memory statistics\n" + " --nomutex Open db with SQLITE_OPEN_NOMUTEX\n" " --nosync Set PRAGMA synchronous=OFF\n" " --notnull Add NOT NULL constraints to table columns\n" " --output FILE Store SQL output in FILE\n" @@ -43,7 +45,8 @@ static const char zHelp[] = " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" " --utf16le Set text encoding to UTF-16LE\n" - " --verify Run additional verification steps.\n" + " --verify Run additional verification steps\n" + " --vfs NAME Use the given (preinstalled) VFS\n" " --without-rowid Use WITHOUT ROWID where appropriate\n" ; @@ -97,6 +100,7 @@ static struct Global { int nRepeat; /* Repeat selects this many times */ int doCheckpoint; /* Run PRAGMA wal_checkpoint after each trans */ int nReserve; /* Reserve bytes */ + int doBigTransactions; /* Enable transactions on tests 410 and 510 */ const char *zWR; /* Might be WITHOUT ROWID */ const char *zNN; /* Might be NOT NULL */ const char *zPK; /* Might be UNIQUE or PRIMARY KEY */ @@ -372,10 +376,12 @@ int speedtest1_numbername(unsigned int n, char *zOut, int nOut){ #define NAMEWIDTH 60 static const char zDots[] = "......................................................................."; +static int iTestNumber = 0; /* Current test # for begin/end_test(). */ void speedtest1_begin_test(int iTestNum, const char *zTestName, ...){ int n = (int)strlen(zTestName); char *zName; va_list ap; + iTestNumber = iTestNum; va_start(ap, zTestName); zName = sqlite3_vmprintf(zTestName, ap); va_end(ap); @@ -384,6 +390,11 @@ void speedtest1_begin_test(int iTestNum, const char *zTestName, ...){ zName[NAMEWIDTH] = 0; n = NAMEWIDTH; } + if( g.pScript ){ + fprintf(g.pScript,"-- begin test %d %.*s\n", iTestNumber, n, zName) + /* maintenance reminder: ^^^ code in ext/wasm expects %d to be + ** field #4 (as in: cut -d' ' -f4). */; + } if( g.bSqlOnly ){ printf("/* %4d - %s%.*s */\n", iTestNum, zName, NAMEWIDTH-n, zDots); }else{ @@ -404,6 +415,10 @@ void speedtest1_exec(const char*,...); void speedtest1_end_test(void){ sqlite3_int64 iElapseTime = speedtest1_timestamp() - g.iStart; if( g.doCheckpoint ) speedtest1_exec("PRAGMA wal_checkpoint;"); + assert( iTestNumber > 0 ); + if( g.pScript ){ + fprintf(g.pScript,"-- end test %d\n", iTestNumber); + } if( !g.bSqlOnly ){ g.iTotal += iElapseTime; printf("%4d.%03ds\n", (int)(iElapseTime/1000), (int)(iElapseTime%1000)); @@ -412,6 +427,7 @@ void speedtest1_end_test(void){ sqlite3_finalize(g.pStmt); g.pStmt = 0; } + iTestNumber = 0; } /* Report end of testing */ @@ -1105,12 +1121,24 @@ void testset_main(void){ speedtest1_exec("COMMIT"); speedtest1_end_test(); speedtest1_begin_test(410, "%d SELECTS on an IPK", n); + if( g.doBigTransactions ){ + /* Historical note: tests 410 and 510 have historically not used + ** explicit transactions. The --big-transactions flag was added + ** 2022-09-08 to support the WASM/OPFS build, as the run-times + ** approach 1 minute for each of these tests if they're not in an + ** explicit transaction. The run-time effect of --big-transaciions + ** on native builds is negligible. */ + speedtest1_exec("BEGIN"); + } speedtest1_prepare("SELECT b FROM t5 WHERE a=?1; -- %d times",n); for(i=1; i<=n; i++){ x1 = swizzle(i,maxb); sqlite3_bind_int(g.pStmt, 1, (sqlite3_int64)x1); speedtest1_run(); } + if( g.doBigTransactions ){ + speedtest1_exec("COMMIT"); + } speedtest1_end_test(); sz = n = g.szTest*700; @@ -1132,6 +1160,10 @@ void testset_main(void){ speedtest1_exec("COMMIT"); speedtest1_end_test(); speedtest1_begin_test(510, "%d SELECTS on a TEXT PK", n); + if( g.doBigTransactions ){ + /* See notes for test 410. */ + speedtest1_exec("BEGIN"); + } speedtest1_prepare("SELECT b FROM t6 WHERE a=?1; -- %d times",n); for(i=1; i<=n; i++){ x1 = swizzle(i,maxb); @@ -1139,6 +1171,9 @@ void testset_main(void){ sqlite3_bind_text(g.pStmt, 1, zNum, -1, SQLITE_STATIC); speedtest1_run(); } + if( g.doBigTransactions ){ + speedtest1_exec("COMMIT"); + } speedtest1_end_test(); speedtest1_begin_test(520, "%d SELECT DISTINCT", n); speedtest1_exec("SELECT DISTINCT b FROM t5;"); @@ -2162,7 +2197,6 @@ static int xCompileOptions(void *pCtx, int nVal, char **azVal, char **azCol){ printf("-- Compile option: %s\n", azVal[0]); return SQLITE_OK; } - int main(int argc, char **argv){ int doAutovac = 0; /* True for --autovacuum */ int cacheSize = 0; /* Desired cache size. 0 means default */ @@ -2180,7 +2214,10 @@ int main(int argc, char **argv){ int nThread = 0; /* --threads value */ int mmapSize = 0; /* How big of a memory map to use */ int memDb = 0; /* --memdb. Use an in-memory database */ + int openFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + ; /* SQLITE_OPEN_xxx flags. */ char *zTSet = "main"; /* Which --testset torun */ + const char * zVfs = 0; /* --vfs NAME */ int doTrace = 0; /* True for --trace */ const char *zEncoding = 0; /* --utf16be or --utf16le */ const char *zDbName = 0; /* Name of the test database */ @@ -2192,10 +2229,19 @@ int main(int argc, char **argv){ int i; /* Loop counter */ int rc; /* API return code */ +#ifdef SQLITE_SPEEDTEST1_WASM + /* Resetting all state is important for the WASM build, which may + ** call main() multiple times. */ + memset(&g, 0, sizeof(g)); + iTestNumber = 0; +#endif #ifdef SQLITE_CKSUMVFS_STATIC sqlite3_register_cksumvfs(0); #endif - + /* + ** Confirms that argc has at least N arguments following argv[i]. */ +#define ARGC_VALUE_CHECK(N) \ + if( i>=argc-(N) ) fatal_error("missing argument on %s\n", argv[i]) /* Display the version of SQLite being tested */ printf("-- Speedtest1 for SQLite %s %.48s\n", sqlite3_libversion(), sqlite3_sourceid()); @@ -2212,10 +2258,11 @@ int main(int argc, char **argv){ do{ z++; }while( z[0]=='-' ); if( strcmp(z,"autovacuum")==0 ){ doAutovac = 1; + }else if( strcmp(z,"big-transactions")==0 ){ + g.doBigTransactions = 1; }else if( strcmp(z,"cachesize")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); - i++; - cacheSize = integerValue(argv[i]); + ARGC_VALUE_CHECK(1); + cacheSize = integerValue(argv[++i]); }else if( strcmp(z,"exclusive")==0 ){ doExclusive = 1; }else if( strcmp(z,"checkpoint")==0 ){ @@ -2224,20 +2271,20 @@ int main(int argc, char **argv){ g.bSqlOnly = 1; g.bExplain = 1; }else if( strcmp(z,"heap")==0 ){ - if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]); + ARGC_VALUE_CHECK(2); nHeap = integerValue(argv[i+1]); mnHeap = integerValue(argv[i+2]); i += 2; }else if( strcmp(z,"incrvacuum")==0 ){ doIncrvac = 1; }else if( strcmp(z,"journal")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); zJMode = argv[++i]; }else if( strcmp(z,"key")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); zKey = argv[++i]; }else if( strcmp(z,"lookaside")==0 ){ - if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]); + ARGC_VALUE_CHECK(2); nLook = integerValue(argv[i+1]); szLook = integerValue(argv[i+2]); i += 2; @@ -2251,9 +2298,11 @@ int main(int argc, char **argv){ #endif #if SQLITE_VERSION_NUMBER>=3007017 }else if( strcmp(z, "mmap")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); mmapSize = integerValue(argv[++i]); #endif + }else if( strcmp(z,"nomutex")==0 ){ + openFlags |= SQLITE_OPEN_NOMUTEX; }else if( strcmp(z,"nosync")==0 ){ noSync = 1; }else if( strcmp(z,"notnull")==0 ){ @@ -2263,7 +2312,7 @@ int main(int argc, char **argv){ fatal_error("The --output option is not supported with" " -DSPEEDTEST_OMIT_HASH\n"); #else - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); i++; if( strcmp(argv[i],"-")==0 ){ g.hashFile = stdout; @@ -2275,10 +2324,10 @@ int main(int argc, char **argv){ } #endif }else if( strcmp(z,"pagesize")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); pageSize = integerValue(argv[++i]); }else if( strcmp(z,"pcache")==0 ){ - if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]); + ARGC_VALUE_CHECK(2); nPCache = integerValue(argv[i+1]); szPCache = integerValue(argv[i+2]); doPCache = 1; @@ -2286,9 +2335,8 @@ int main(int argc, char **argv){ }else if( strcmp(z,"primarykey")==0 ){ g.zPK = "PRIMARY KEY"; }else if( strcmp(z,"repeat")==0 ){ - if( i>=argc-1 ) fatal_error("missing arguments on %s\n", argv[i]); - g.nRepeat = integerValue(argv[i+1]); - i += 1; + ARGC_VALUE_CHECK(1); + g.nRepeat = integerValue(argv[++i]); }else if( strcmp(z,"reprepare")==0 ){ g.bReprepare = 1; #if SQLITE_VERSION_NUMBER>=3006000 @@ -2298,7 +2346,7 @@ int main(int argc, char **argv){ sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); #endif }else if( strcmp(z,"script")==0 ){ - if( i>=argc-1 ) fatal_error("missing arguments on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); if( g.pScript ) fclose(g.pScript); g.pScript = fopen(argv[++i], "wb"); if( g.pScript==0 ){ @@ -2309,24 +2357,24 @@ int main(int argc, char **argv){ }else if( strcmp(z,"shrink-memory")==0 ){ g.bMemShrink = 1; }else if( strcmp(z,"size")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); g.szTest = integerValue(argv[++i]); }else if( strcmp(z,"stats")==0 ){ showStats = 1; }else if( strcmp(z,"temp")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); i++; if( argv[i][0]<'0' || argv[i][0]>'9' || argv[i][1]!=0 ){ fatal_error("argument to --temp should be integer between 0 and 9"); } g.eTemp = argv[i][0] - '0'; }else if( strcmp(z,"testset")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); zTSet = argv[++i]; }else if( strcmp(z,"trace")==0 ){ doTrace = 1; }else if( strcmp(z,"threads")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); nThread = integerValue(argv[++i]); }else if( strcmp(z,"utf16le")==0 ){ zEncoding = "utf16le"; @@ -2337,8 +2385,11 @@ int main(int argc, char **argv){ #ifndef SPEEDTEST_OMIT_HASH HashInit(); #endif + }else if( strcmp(z,"vfs")==0 ){ + ARGC_VALUE_CHECK(1); + zVfs = argv[++i]; }else if( strcmp(z,"reserve")==0 ){ - if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + ARGC_VALUE_CHECK(1); g.nReserve = atoi(argv[++i]); }else if( strcmp(z,"without-rowid")==0 ){ if( strstr(g.zWR,"WITHOUT")!=0 ){ @@ -2371,7 +2422,7 @@ int main(int argc, char **argv){ argv[i], argv[0]); } } - if( zDbName!=0 ) unlink(zDbName); +#undef ARGC_VALUE_CHECK #if SQLITE_VERSION_NUMBER>=3006001 if( nHeap>0 ){ pHeap = malloc( nHeap ); @@ -2394,8 +2445,23 @@ int main(int argc, char **argv){ #endif sqlite3_initialize(); + if( zDbName!=0 ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs); + /* For some VFSes, e.g. opfs, unlink() is not sufficient. Use the + ** selected (or default) VFS's xDelete method to delete the + ** database. This is specifically important for the "opfs" VFS + ** when running from a WASM build of speedtest1, so that the db + ** can be cleaned up properly. For historical compatibility, we'll + ** also simply unlink(). */ + if( pVfs!=0 ){ + pVfs->xDelete(pVfs, zDbName, 1); + } + unlink(zDbName); + } + /* Open the database and the input file */ - if( sqlite3_open(memDb ? ":memory:" : zDbName, &g.db) ){ + if( sqlite3_open_v2(memDb ? ":memory:" : zDbName, &g.db, + openFlags, zVfs) ){ fatal_error("Cannot open database file: %s\n", zDbName); } #if SQLITE_VERSION_NUMBER>=3006001 @@ -2580,3 +2646,13 @@ int main(int argc, char **argv){ free( pHeap ); return 0; } + +#ifdef SQLITE_SPEEDTEST1_WASM +/* +** A workaround for some inconsistent behaviour with how +** main() does (or does not) get exported to WASM. +*/ +int wasm_main(int argc, char **argv){ + return main(argc, argv); +} +#endif diff --git a/test/tester.tcl b/test/tester.tcl index 8fd005a9bf..52675f2dec 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1549,6 +1549,47 @@ proc explain_i {sql {db db}} { output2 "---- ------------ ------ ------ ------ ---------------- -- -" } +proc execsql_pp {sql {db db}} { + set nCol 0 + $db eval $sql A { + if {$nCol==0} { + set nCol [llength $A(*)] + foreach c $A(*) { + set aWidth($c) [string length $c] + lappend data $c + } + } + foreach c $A(*) { + set n [string length $A($c)] + if {$n > $aWidth($c)} { + set aWidth($c) $n + } + lappend data $A($c) + } + } + if {$nCol>0} { + set nTotal 0 + foreach e [array names aWidth] { incr nTotal $aWidth($e) } + incr nTotal [expr ($nCol-1) * 3] + incr nTotal 4 + + set fmt "" + foreach c $A(*) { + lappend fmt "% -$aWidth($c)s" + } + set fmt "| [join $fmt { | }] |" + + puts [string repeat - $nTotal] + for {set i 0} {$i < [llength $data]} {incr i $nCol} { + set vals [lrange $data $i [expr $i+$nCol-1]] + puts [format $fmt {*}$vals] + if {$i==0} { puts [string repeat - $nTotal] } + } + puts [string repeat - $nTotal] + } +} + + # Show the VDBE program for an SQL statement but omit the Trace # opcode at the beginning. This procedure can be used to prove # that different SQL statements generate exactly the same VDBE code. diff --git a/test/unionall.test b/test/unionall.test index 7554a3b308..7783c04a67 100644 --- a/test/unionall.test +++ b/test/unionall.test @@ -55,6 +55,27 @@ do_execsql_test 1.3 { } {1 one 2 two 5 five 6 six} +# 2022-10-31 part of ticket 57c47526c34f01e8 +# The queries below were causing an assertion fault in +# the comparison operators of the VDBE. +# +reset_db +database_never_corrupt +optimization_control db all 0 +do_execsql_test 1.10 { + CREATE TABLE t0(c0 INT); + INSERT INTO t0 VALUES(0); + CREATE TABLE t1_a(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1_a VALUES(1,'one'); + CREATE TABLE t1_b(c INTEGER PRIMARY KEY, d TEXT); + INSERT INTO t1_b VALUES(2,'two'); + CREATE VIEW t1 AS SELECT a, b FROM t1_a UNION ALL SELECT c, c FROM t1_b; + SELECT * FROM (SELECT t1.a, t1.b AS b, t0.c0 FROM t0, t1); +} {1 one 0 2 2 0} +do_execsql_test 1.11 { + SELECT * FROM (SELECT t1.a, t1.b AS b, t0.c0 FROM t0, t1) WHERE b=2; +} {2 2 0} + #------------------------------------------------------------------------- reset_db @@ -364,4 +385,60 @@ do_execsql_test 7.1 { ORDER BY x, y; } {0 1 | 0 101 | 0 102 | 1 1 | 1 101 | 1 102 | 100 1 | 100 101 | 100 102 | 101 1 | 101 101 | 101 102 |} +# 2022-10-31 ticket https://sqlite.org/src/info/57c47526c34f01e8 +# dbsqlfuzz 37230460b46b3b6049f0d768eb801f3428189382 +# UNION ALL subqueries or views which have arms with different +# affinities should not be flattened. +# +reset_db +do_execsql_test 8.1 { + CREATE TABLE t0(c0 INT); + INSERT INTO t0 VALUES(0); + CREATE TABLE t1_a(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1_a VALUES(1,'one'); + INSERT INTO t1_a VALUES(4,'four'); + CREATE TABLE t1_b(c INTEGER PRIMARY KEY, d TEXT); + INSERT INTO t1_b VALUES(2,'two'); + INSERT INTO t1_b VALUES(5,'five'); + CREATE TABLE t1_c(e INTEGER PRIMARY KEY, f TEXT); + INSERT INTO t1_c VALUES(3,'three'); + INSERT INTO t1_c VALUES(6,'six'); + CREATE VIEW v0(c0) AS SELECT CAST(t0.c0 AS INTEGER) FROM t0; + CREATE VIEW t1 AS + SELECT a, b FROM t1_a UNION ALL + SELECT c, c FROM t1_b UNION ALL + SELECT e, f FROM t1_c; +} +optimization_control db all 1 +do_execsql_test 8.2 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2; +} {2 2 0 {}} +do_execsql_test 8.3 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2.0; +} {} +do_execsql_test 8.4 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b='2'; +} {2 2 0 {}} +optimization_control db query-flattener,push-down 0 +do_execsql_test 8.5 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2; +} {2 2 0 {}} +do_execsql_test 8.6 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2.0; +} {} +do_execsql_test 8.7 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b='2'; +} {2 2 0 {}} +optimization_control db all 0 +do_execsql_test 8.8 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2; +} {2 2 0 {}} +do_execsql_test 8.9 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2.0; +} {} +do_execsql_test 8.10 { + SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b='2'; +} {2 2 0 {}} + + finish_test diff --git a/test/vacuum-into.test b/test/vacuum-into.test index a46d95cf17..98692a108a 100644 --- a/test/vacuum-into.test +++ b/test/vacuum-into.test @@ -133,4 +133,58 @@ if {[wal_is_capable]} { } {1024 ok} } +#------------------------------------------------------------------------- + +testvfs tvfs -default 1 +tvfs filter xSync +tvfs script xSyncCb +proc xSyncCb {method file fileid flags} { + incr ::sync($flags) +} + +reset_db + +do_execsql_test vacuum-into-700 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} + +foreach {tn pragma res} { + 710 { + PRAGMA synchronous = normal + } {normal 2} + 720 { + PRAGMA synchronous = full + } {normal 3} + 730 { + PRAGMA synchronous = off + } {} + 740 { + PRAGMA synchronous = extra; + } {normal 3} + 750 { + PRAGMA fullfsync = 1; + PRAGMA synchronous = full; + } {full|dataonly 1 full 2} +} { + + forcedelete test.db2 + array unset ::sync + do_execsql_test vacuum-into-$tn.1 " + $pragma ; + VACUUM INTO 'test.db2' + " + + do_test vacuum-into-$tn.2 { + array get ::sync + } $res +} + +db close +tvfs delete + + finish_test + + + diff --git a/test/vt02.c b/test/vt02.c new file mode 100644 index 0000000000..40bd6fcb3e --- /dev/null +++ b/test/vt02.c @@ -0,0 +1,1019 @@ +/* +** This file implements an eponymous, read-only table-valued function +** (a virtual table) designed to be used for testing. We are not aware +** of any practical real-world use case for the virtual table. +** +** This virtual table originated in the TH3 test suite. It is still used +** there, but has now been copied into the public SQLite source tree and +** reused for a variety of testing purpose. The name "vt02" comes from the +** fact that there are many different testing virtual tables in TH3, of which +** this one is the second. +** +** ## SUBJECT TO CHANGE +** +** Because this virtual table is intended for testing, its interface is not +** guaranteed to be stable across releases. Future releases may contain +** changes in the vt02 design and interface. +** +** ## OVERVIEW +** +** The vt02 table-valued function has 10000 rows with 5 data columns. +** Column X contains all integer values between 0 and 9999 inclusive. +** Columns A, B, C, and D contain the individual base-10 digits associated +** with each X value: +** +** X A B C D +** ---- - - - - +** 0 0 0 0 0 +** 1 0 0 0 1 +** 2 0 0 0 2 +** ... +** 4998 4 9 9 8 +** 4999 4 9 9 9 +** 5000 5 0 0 0 +** ... +** 9995 9 9 9 5 +** 9996 9 9 9 6 +** 9997 9 9 9 7 +** +** The xBestIndex method recognizes a variety of equality constraints +** and attempts to optimize its output accordingly. +** +** x=... +** a=... +** a=... AND b=... +** a=... AND b=... AND c=... +** a=... AND b=... AND c=... AND d=... +** +** Various ORDER BY constraints are also recognized and consumed. The +** OFFSET constraint is recognized and consumed. +** +** ## TABLE-VALUED FUNCTION +** +** The vt02 virtual table is eponymous and has two hidden columns, meaning +** that it can functions a table-valued function. The two hidden columns +** are "flags" and "logtab", in that order. The "flags" column can be set +** to an integer where various bits enable or disable behaviors of the +** virtual table. The "logtab" can set to the name of an ordinary SQLite +** table into which is written information about each call to xBestIndex. +** +** The bits of "flags" are as follows: +** +** 0x01 Ignore the aConstraint[].usable flag. This might +** result in the xBestIndex method incorrectly using +** unusable entries in the aConstraint[] array, which +** should result in the SQLite core detecting and +** reporting that the virtual table is not behaving +** to spec. +** +** 0x02 Do not set the orderByConsumed flag, even if it +** could be set. +** +** 0x04 Do not consume the OFFSET constraint, if there is +** one. Instead, let the generated byte-code visit +** and ignore the first few columns of output. +** +** 0x08 Use sqlite3_mprintf() to allocate an idxStr string. +** The string is never used, but allocating it does +** test the idxStr deallocation logic inside of the +** SQLite core. +** +** 0x10 Cause the xBestIndex method to generate an idxNum +** that xFilter does not understand, thus causing +** the OP_VFilter opcode to raise an error. +** +** 0x20 Set the omit flag for all equality constraints on +** columns X, A, B, C, and D that are used to limit +** the search. +** +** 0x40 Add all constraints against X,A,B,C,D to the +** vector of results sent to xFilter. Only the first +** few are used, as required by idxNum. +** +** Because these flags take effect during xBestIndex, the RHS of the +** flag= constraint must be accessible. In other words, the RHS of flag= +** needs to be an integer literal, not another column of a join or a +** bound parameter. +** +** ## LOGGING OUTPUT +** +** If the "logtab" columns is set, then each call to the xBestIndex method +** inserts multiple rows into the table identified by "logtab". These +** rows collectively show the content of the sqlite3_index_info object and +** other context associated with the xBestIndex call. +** +** If the table named by "logtab" does not previously exist, it is created +** automatically. The schema for the logtab table is like this: +** +** CREATE TEMP TABLE vt02_log( +** bi INT, -- BestIndex call counter +** vn TEXT, -- Variable Name +** ix INT, -- Index or value +** cn TEXT, -- Column Name +** op INT, -- Opcode or "DESC" value +** ux INT, -- "Usable" flag +** ra BOOLEAN, -- Right-hand side Available. +** rhs ANY, -- Right-Hand Side value +** cs TEXT -- Collating Sequence for this constraint +** ); +** +** Because logging happens during xBestIindex, the RHS value of "logtab" must +** be known to xBestIndex, which means it must be a string literal, not a +** column in a join, or a bound parameter. +** +** ## VIRTUAL TABLE SCHEMA +** +** CREATE TABLE vt02( +** x INT, -- integer between 0 and 9999 inclusive +** a INT, -- The 1000s digit +** b INT, -- The 100s digit +** c INT, -- The 10s digit +** d INT, -- The 1s digit +** flags INT HIDDEN, -- Option flags +** logtab TEXT HIDDEN, -- Name of table into which to log xBestIndex +** ); +** +** ## COMPILING AND RUNNING +** +** This file can also be compiled separately as a loadable extension +** for SQLite (as long as the -DTH3_VERSION is not defined). To compile as a +** loadable extension do his: +** +** gcc -Wall -g -shared -fPIC -I. -DSQLITE_DEBUG vt02.c -o vt02.so +** +** Or on Windows: +** +** cl vt02.c -link -dll -out:vt02.dll +** +** Then load into the CLI using: +** +** .load ./vt02 sqlite3_vt02_init +** +** ## IDXNUM SUMMARY +** +** The xBestIndex method communicates the query plan to xFilter using +** the idxNum value, as follows: +** +** 0 unconstrained +** 1 X=argv[0] +** 2 A=argv[0] +** 3 A=argv[0], B=argv[1] +** 4 A=argv[0], B=argv[1], C=argv[2] +** 5 A=argv[0], B=argv[1], C=argv[2], D=argv[3] +** 6 A=argv[0], D IN argv[2] +** 7 A=argv[0], B=argv[2], D IN argv[3] +** 8 A=argv[0], B=argv[2], C=argv[3], D IN argv[4] +** 1x increment by 10 +** 2x increment by 100 +** 3x increment by 1000 +** 1xx Use offset provided by argv[N] +*/ +#ifndef TH3_VERSION + /* These bits for separate compilation as a loadable extension, only */ + #include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 + #include + #include + #include +#endif + +/* Forward declarations */ +typedef struct vt02_vtab vt02_vtab; +typedef struct vt02_cur vt02_cur; + +/* +** The complete virtual table +*/ +struct vt02_vtab { + sqlite3_vtab parent; /* Base clase. Must be first. */ + sqlite3 *db; /* Database connection */ + int busy; /* Currently running xBestIndex */ +}; + +#define VT02_IGNORE_USABLE 0x0001 /* Ignore usable flags */ +#define VT02_NO_SORT_OPT 0x0002 /* Do not do any sorting optimizations */ +#define VT02_NO_OFFSET 0x0004 /* Omit the offset optimization */ +#define VT02_ALLOC_IDXSTR 0x0008 /* Alloate an idxStr */ +#define VT02_BAD_IDXNUM 0x0010 /* Generate an invalid idxNum */ + +/* +** A cursor +*/ +struct vt02_cur { + sqlite3_vtab_cursor parent; /* Base class. Must be first */ + sqlite3_int64 i; /* Current entry */ + sqlite3_int64 iEof; /* Indicate EOF when reaching this value */ + int iIncr; /* Amount by which to increment */ + unsigned int mD; /* Mask of allowed D-column values */ +}; + +/* The xConnect method */ +int vt02Connect( + sqlite3 *db, /* The database connection */ + void *pAux, /* Pointer to an alternative schema */ + int argc, /* Number of arguments */ + const char *const*argv, /* Text of the arguments */ + sqlite3_vtab **ppVTab, /* Write the new vtab here */ + char **pzErr /* Error message written here */ +){ + vt02_vtab *pVtab; + int rc; + const char *zSchema = (const char*)pAux; + static const char zDefaultSchema[] = + "CREATE TABLE x(x INT, a INT, b INT, c INT, d INT," + " flags INT HIDDEN, logtab TEXT HIDDEN);"; +#define VT02_COL_X 0 +#define VT02_COL_A 1 +#define VT02_COL_B 2 +#define VT02_COL_C 3 +#define VT02_COL_D 4 +#define VT02_COL_FLAGS 5 +#define VT02_COL_LOGTAB 6 +#define VT02_COL_NONE 7 + + pVtab = sqlite3_malloc( sizeof(*pVtab) ); + if( pVtab==0 ){ + *pzErr = sqlite3_mprintf("out of memory"); + return SQLITE_NOMEM; + } + memset(pVtab, 0, sizeof(*pVtab)); + pVtab->db = db; + rc = sqlite3_declare_vtab(db, zSchema ? zSchema : zDefaultSchema); + if( rc ){ + sqlite3_free(pVtab); + }else{ + *ppVTab = &pVtab->parent; + } + return rc; +} + +/* the xDisconnect method +*/ +int vt02Disconnect(sqlite3_vtab *pVTab){ + sqlite3_free(pVTab); + return SQLITE_OK; +} + +/* Put an error message into the zErrMsg string of the virtual table. +*/ +static void vt02ErrMsg(sqlite3_vtab *pVtab, const char *zFormat, ...){ + va_list ap; + sqlite3_free(pVtab->zErrMsg); + va_start(ap, zFormat); + pVtab->zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + + +/* Open a cursor for scanning +*/ +static int vt02Open(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + vt02_cur *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ){ + vt02ErrMsg(pVTab, "out of memory"); + return SQLITE_NOMEM; + } + *ppCursor = &pCur->parent; + pCur->i = -1; + return SQLITE_OK; +} + +/* Close a cursor +*/ +static int vt02Close(sqlite3_vtab_cursor *pCursor){ + vt02_cur *pCur = (vt02_cur*)pCursor; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* Return TRUE if we are at the end of the BVS and there are +** no more entries. +*/ +static int vt02Eof(sqlite3_vtab_cursor *pCursor){ + vt02_cur *pCur = (vt02_cur*)pCursor; + return pCur->i<0 || pCur->i>=pCur->iEof; +} + +/* Advance the cursor to the next row in the table +*/ +static int vt02Next(sqlite3_vtab_cursor *pCursor){ + vt02_cur *pCur = (vt02_cur*)pCursor; + do{ + pCur->i += pCur->iIncr; + if( pCur->i<0 ) pCur->i = pCur->iEof; + }while( (pCur->mD & (1<<(pCur->i%10)))==0 && pCur->iiEof ); + return SQLITE_OK; +} + +/* Rewind a cursor back to the beginning of its scan. +** +** Scanning is always increasing. +** +** idxNum +** 0 unconstrained +** 1 X=argv[0] +** 2 A=argv[0] +** 3 A=argv[0], B=argv[1] +** 4 A=argv[0], B=argv[1], C=argv[2] +** 5 A=argv[0], B=argv[1], C=argv[2], D=argv[3] +** 6 A=argv[0], D IN argv[2] +** 7 A=argv[0], B=argv[2], D IN argv[3] +** 8 A=argv[0], B=argv[2], C=argv[3], D IN argv[4] +** 1x increment by 10 +** 2x increment by 100 +** 3x increment by 1000 +** 1xx Use offset provided by argv[N] +*/ +static int vt02Filter( + sqlite3_vtab_cursor *pCursor, /* The cursor to rewind */ + int idxNum, /* Search strategy */ + const char *idxStr, /* Not used */ + int argc, /* Not used */ + sqlite3_value **argv /* Not used */ +){ + vt02_cur *pCur = (vt02_cur*)pCursor; /* The vt02 cursor */ + int bUseOffset = 0; /* True to use OFFSET value */ + int iArg = 0; /* argv[] values used so far */ + int iOrigIdxNum = idxNum; /* Original value for idxNum */ + + pCur->iIncr = 1; + pCur->mD = 0x3ff; + if( idxNum>=100 ){ + bUseOffset = 1; + idxNum -= 100; + } + if( idxNum<0 || idxNum>38 ) goto vt02_bad_idxnum; + while( idxNum>=10 ){ + pCur->iIncr *= 10; + idxNum -= 10; + } + if( idxNum==0 ){ + pCur->i = 0; + pCur->iEof = 10000; + }else if( idxNum==1 ){ + pCur->i = sqlite3_value_int64(argv[0]); + if( pCur->i<0 ) pCur->i = -1; + if( pCur->i>9999 ) pCur->i = 10000; + pCur->iEof = pCur->i+1; + if( pCur->i<0 || pCur->i>9999 ) pCur->i = pCur->iEof; + }else if( idxNum>=2 && idxNum<=5 ){ + int i, e, m; + e = idxNum - 2; + assert( e<=argc-1 ); + pCur->i = 0; + for(m=1000, i=0; i<=e; i++, m /= 10){ + sqlite3_int64 v = sqlite3_value_int64(argv[iArg++]); + if( v<0 ) v = 0; + if( v>9 ) v = 9; + pCur->i += m*v; + pCur->iEof = pCur->i+m; + } + }else if( idxNum>=6 && idxNum<=8 ){ + int i, e, m, rc; + sqlite3_value *pIn, *pVal; + e = idxNum - 6; + assert( e<=argc-2 ); + pCur->i = 0; + for(m=1000, i=0; i<=e; i++, m /= 10){ + sqlite3_int64 v; + pVal = 0; + if( sqlite3_vtab_in_first(0, &pVal)!=SQLITE_MISUSE + || sqlite3_vtab_in_first(argv[iArg], &pVal)!=SQLITE_MISUSE + ){ + vt02ErrMsg(pCursor->pVtab, + "unexpected success from sqlite3_vtab_in_first()"); + return SQLITE_ERROR; + } + v = sqlite3_value_int64(argv[iArg++]); + if( v<0 ) v = 0; + if( v>9 ) v = 9; + pCur->i += m*v; + pCur->iEof = pCur->i+m; + } + pCur->mD = 0; + pIn = argv[iArg++]; + assert( sqlite3_value_type(pIn)==SQLITE_NULL ); + for( rc = sqlite3_vtab_in_first(pIn, &pVal); + rc==SQLITE_OK && pVal!=0; + rc = sqlite3_vtab_in_next(pIn, &pVal) + ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_FLOAT ){ + double r = sqlite3_value_double(pVal); + if( r<0.0 || r>9.0 || r!=(int)r ) continue; + }else if( eType!=SQLITE_INTEGER ){ + continue; + } + i = sqlite3_value_int(pVal); + if( i<0 || i>9 ) continue; + pCur->mD |= 1<pVtab, "Error from sqlite3_vtab_in_first/next()"); + return rc; + } + }else{ + goto vt02_bad_idxnum; + } + if( bUseOffset ){ + int nSkip = sqlite3_value_int(argv[iArg]); + while( nSkip-- > 0 ) vt02Next(pCursor); + } + return SQLITE_OK; + +vt02_bad_idxnum: + vt02ErrMsg(pCursor->pVtab, "invalid idxNum for vt02: %d", iOrigIdxNum); + return SQLITE_ERROR; +} + +/* Return the Nth column of the current row. +*/ +static int vt02Column( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *context, + int N +){ + vt02_cur *pCur = (vt02_cur*)pCursor; + int v = pCur->i; + if( N==VT02_COL_X ){ + sqlite3_result_int(context, v); + }else if( N>=VT02_COL_A && N<=VT02_COL_D ){ + static const int iDivisor[] = { 1, 1000, 100, 10, 1 }; + v = (v/iDivisor[N])%10; + sqlite3_result_int(context, v); + } + return SQLITE_OK; +} + +/* Return the rowid of the current row +*/ +static int vt02Rowid(sqlite3_vtab_cursor *pCursor, sqlite3_int64 *pRowid){ + vt02_cur *pCur = (vt02_cur*)pCursor; + *pRowid = pCur->i+1; + return SQLITE_OK; +} + +/************************************************************************* +** Logging Subsystem +** +** The sqlite3BestIndexLog() routine implements a logging system for +** xBestIndex calls. This code is portable to any virtual table. +** +** sqlite3BestIndexLog() is the main routine, sqlite3RunSql() is a +** helper routine used for running various SQL statements as part of +** creating the log. +** +** These two routines should be portable to other virtual tables. Simply +** extract this code and call sqlite3BestIndexLog() near the end of the +** xBestIndex method in cases where logging is desired. +*/ +/* +** Run SQL on behalf of sqlite3BestIndexLog. +** +** Construct the SQL using the zFormat string and subsequent arguments. +** Or if zFormat is NULL, take the SQL as the first argument after the +** zFormat. In either case, the dynamically allocated SQL string is +** freed after it has been run. If something goes wrong with the SQL, +** then an error is left in pVTab->zErrMsg. +*/ +static void sqlite3RunSql( + sqlite3 *db, /* Run the SQL on this database connection */ + sqlite3_vtab *pVTab, /* Report errors to this virtual table */ + const char *zFormat, /* Format string for SQL, or NULL */ + ... /* Arguments, according to the format string */ +){ + char *zSql; + + va_list ap; + va_start(ap, zFormat); + if( zFormat==0 ){ + zSql = va_arg(ap, char*); + }else{ + zSql = sqlite3_vmprintf(zFormat, ap); + } + va_end(ap); + if( zSql ){ + char *zErrMsg = 0; + (void)sqlite3_exec(db, zSql, 0, 0, &zErrMsg); + if( zErrMsg ){ + if( pVTab->zErrMsg==0 ){ + pVTab->zErrMsg = sqlite3_mprintf("%s in [%s]", zErrMsg, zSql); + } + sqlite3_free(zErrMsg); + } + sqlite3_free(zSql); + } +} + +/* +** Record information about each xBestIndex method call in a separate +** table: +** +** CREATE TEMP TABLE [log-table-name] ( +** bi INT, -- BestIndex call number +** vn TEXT, -- Variable Name +** ix INT, -- Index or value +** cn TEXT, -- Column Name +** op INT, -- Opcode or argvIndex +** ux INT, -- "usable" or "omit" flag +** rx BOOLEAN, -- True if has a RHS value +** rhs ANY, -- The RHS value +** cs TEXT, -- Collating Sequence +** inop BOOLEAN -- True if this is a batchable IN operator +** ); +** +** If an error occurs, leave an error message in pVTab->zErrMsg. +*/ +static void sqlite3BestIndexLog( + sqlite3_index_info *pInfo, /* The sqlite3_index_info object */ + const char *zLogTab, /* Log into this table */ + sqlite3 *db, /* Database connection containing zLogTab */ + const char **azColname, /* Names of columns in the virtual table */ + sqlite3_vtab *pVTab /* Record errors into this object */ +){ + int i, rc; + sqlite3_str *pStr; + int iBI; + + if( sqlite3_table_column_metadata(db,0,zLogTab,0,0,0,0,0,0) ){ + /* The log table does not previously exist. Create it. */ + sqlite3RunSql(db,pVTab, + "CREATE TABLE IF NOT EXISTS temp.\"%w\"(\n" + " bi INT, -- BestIndex call number\n" + " vn TEXT, -- Variable Name\n" + " ix INT, -- Index or value\n" + " cn TEXT, -- Column Name\n" + " op INT, -- Opcode or argvIndex\n" + " ux INT, -- usable for omit flag\n" + " rx BOOLEAN, -- Right-hand side value is available\n" + " rhs ANY, -- RHS value\n" + " cs TEXT, -- Collating Sequence\n" + " inop BOOLEAN -- IN operator capable of batch reads\n" + ");", zLogTab + ); + iBI = 1; + }else{ + /* The log table does already exist. We assume that it has the + ** correct schema and proceed to find the largest prior "bi" value. + ** If the schema is wrong, errors might result. The code is able + ** to deal with this. */ + sqlite3_stmt *pStmt; + char *zSql; + zSql = sqlite3_mprintf("SELECT max(bi) FROM temp.\"%w\"",zLogTab); + if( zSql==0 ){ + sqlite3_free(pVTab->zErrMsg); + pVTab->zErrMsg = sqlite3_mprintf("out of memory"); + return; + } + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_free(pVTab->zErrMsg); + pVTab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + iBI = 0; + }else if( sqlite3_step(pStmt)==SQLITE_ROW ){ + iBI = sqlite3_column_int(pStmt, 0)+1; + }else{ + iBI = 1; + } + sqlite3_finalize(pStmt); + } + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix) VALUES(%d,'nConstraint',%d)", + zLogTab, iBI, pInfo->nConstraint + ); + for(i=0; inConstraint; i++){ + sqlite3_value *pVal; + char *zSql; + int iCol = pInfo->aConstraint[i].iColumn; + int op = pInfo->aConstraint[i].op; + const char *zCol; + if( op==SQLITE_INDEX_CONSTRAINT_LIMIT + || op==SQLITE_INDEX_CONSTRAINT_OFFSET + ){ + zCol = ""; + }else if( iCol<0 ){ + zCol = "rowid"; + }else{ + zCol = azColname[iCol]; + } + pStr = sqlite3_str_new(0); + sqlite3_str_appendf(pStr, + "INSERT INTO temp.\"%w\"(bi,vn,ix,cn,op,ux,rx,rhs,cs,inop)" + "VALUES(%d,'aConstraint',%d,%Q,%d,%d", + zLogTab, iBI, + i, + zCol, + op, + pInfo->aConstraint[i].usable); + pVal = 0; + rc = sqlite3_vtab_rhs_value(pInfo, i, &pVal); + assert( pVal!=0 || rc!=SQLITE_OK ); + if( rc==SQLITE_OK ){ + sqlite3_str_appendf(pStr,",1,?1"); + }else{ + sqlite3_str_appendf(pStr,",0,NULL"); + } + sqlite3_str_appendf(pStr,",%Q,%d)", + sqlite3_vtab_collation(pInfo,i), + sqlite3_vtab_in(pInfo,i,-1)); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ){ + if( pVTab->zErrMsg==0 ) pVTab->zErrMsg = sqlite3_mprintf("out of memory"); + }else{ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ){ + if( pVTab->zErrMsg==0 ){ + pVTab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + }else{ + if( pVal ) sqlite3_bind_value(pStmt, 1, pVal); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc && pVTab->zErrMsg==0 ){ + pVTab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + } + sqlite3_finalize(pStmt); + sqlite3_free(zSql); + } + } + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix) VALUES(%d,'nOrderBy',%d)", + zLogTab, iBI, pInfo->nOrderBy + ); + for(i=0; inOrderBy; i++){ + int iCol = pInfo->aOrderBy[i].iColumn; + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix,cn,op)VALUES(%d,'aOrderBy',%d,%Q,%d)", + zLogTab, iBI, + i, + iCol>=0 ? azColname[iCol] : "rowid", + pInfo->aOrderBy[i].desc + ); + } + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix) VALUES(%d,'sqlite3_vtab_distinct',%d)", + zLogTab, iBI, sqlite3_vtab_distinct(pInfo) + ); + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix) VALUES(%d,'colUsed',%lld)", + zLogTab, iBI, pInfo->colUsed + ); + for(i=0; inConstraint; i++){ + int iCol = pInfo->aConstraint[i].iColumn; + int op = pInfo->aConstraint[i].op; + const char *zCol; + if( op==SQLITE_INDEX_CONSTRAINT_LIMIT + || op==SQLITE_INDEX_CONSTRAINT_OFFSET + ){ + zCol = ""; + }else if( iCol<0 ){ + zCol = "rowid"; + }else{ + zCol = azColname[iCol]; + } + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix,cn,op,ux)" + "VALUES(%d,'aConstraintUsage',%d,%Q,%d,%d)", + zLogTab, iBI, + i, + zCol, + pInfo->aConstraintUsage[i].argvIndex, + pInfo->aConstraintUsage[i].omit + ); + } + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix)VALUES(%d,'idxNum',%d)", + zLogTab, iBI, pInfo->idxNum + ); + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix)VALUES(%d,'estimatedCost',%f)", + zLogTab, iBI, pInfo->estimatedCost + ); + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix)VALUES(%d,'estimatedRows',%lld)", + zLogTab, iBI, pInfo->estimatedRows + ); + if( pInfo->idxStr ){ + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix)VALUES(%d,'idxStr',%Q)", + zLogTab, iBI, pInfo->idxStr + ); + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix)VALUES(%d,'needToFreeIdxStr',%d)", + zLogTab, iBI, pInfo->needToFreeIdxStr + ); + } + if( pInfo->nOrderBy ){ + sqlite3RunSql(db,pVTab, + "INSERT INTO temp.\"%w\"(bi,vn,ix)VALUES(%d,'orderByConsumed',%d)", + zLogTab, iBI, pInfo->orderByConsumed + ); + } +} +/* +** End of Logging Subsystem +*****************************************************************************/ + + +/* Find an estimated cost of running a query against vt02. +*/ +static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + int i; /* Loop counter */ + int isEq[5]; /* Equality constraints on X, A, B, C, and D */ + int isUsed[5]; /* Other non-== cosntraints X, A, B, C, and D */ + int argvIndex = 0; /* Next available argv[] slot */ + int iOffset = -1; /* Constraint for OFFSET */ + void *pX = 0; /* idxStr value */ + int flags = 0; /* RHS value for flags= */ + const char *zLogTab = 0; /* RHS value for logtab= */ + int iFlagTerm = -1; /* Constraint term for flags= */ + int iLogTerm = -1; /* Constraint term for logtab= */ + int iIn = -1; /* Index of the IN constraint */ + vt02_vtab *pSelf; /* This virtual table */ + + pSelf = (vt02_vtab*)pVTab; + if( pSelf->busy ){ + vt02ErrMsg(pVTab, "recursive use of vt02 prohibited"); + return SQLITE_CONSTRAINT; + } + pSelf->busy++; + + + /* Do an initial scan for flags=N and logtab=TAB constraints with + ** usable RHS values */ + for(i=0; inConstraint; i++){ + sqlite3_value *pVal; + if( !pInfo->aConstraint[i].usable ) continue; + if( pInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pInfo->aConstraint[i].iColumn ){ + case VT02_COL_FLAGS: + if( sqlite3_vtab_rhs_value(pInfo, i, &pVal)==SQLITE_OK + && sqlite3_value_type(pVal)==SQLITE_INTEGER + ){ + flags = sqlite3_value_int(pVal); + } + iFlagTerm = i; + break; + case VT02_COL_LOGTAB: + if( sqlite3_vtab_rhs_value(pInfo, i, &pVal)==SQLITE_OK + && sqlite3_value_type(pVal)==SQLITE_TEXT + ){ + zLogTab = (const char*)sqlite3_value_text(pVal); + } + iLogTerm = i; + break; + } + } + + /* Do a second scan to actually analyze the index information */ + memset(isEq, 0xff, sizeof(isEq)); + memset(isUsed, 0xff, sizeof(isUsed)); + for(i=0; inConstraint; i++){ + int j = pInfo->aConstraint[i].iColumn; + if( j>=VT02_COL_FLAGS ) continue; + if( pInfo->aConstraint[i].usable==0 + && (flags & VT02_IGNORE_USABLE)==0 ) continue; + if( j<0 ) j = VT02_COL_X; + switch( pInfo->aConstraint[i].op ){ + case SQLITE_INDEX_CONSTRAINT_FUNCTION: + case SQLITE_INDEX_CONSTRAINT_EQ: + isEq[j] = i; + break; + case SQLITE_INDEX_CONSTRAINT_LT: + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_GT: + case SQLITE_INDEX_CONSTRAINT_GE: + isUsed[j] = i; + break; + case SQLITE_INDEX_CONSTRAINT_OFFSET: + iOffset = i; + break; + } + } + + /* Use the analysis to find an appropriate query plan */ + if( isEq[0]>=0 ){ + /* A constraint of X= takes priority */ + pInfo->estimatedCost = 1; + pInfo->aConstraintUsage[isEq[0]].argvIndex = ++argvIndex; + if( flags & 0x20 ) pInfo->aConstraintUsage[isEq[0]].omit = 1; + pInfo->idxNum = 1; + }else if( isEq[1]<0 ){ + /* If there is no X= nor A= then we have to do a full scan */ + pInfo->idxNum = 0; + pInfo->estimatedCost = 10000; + }else{ + int v = 1000; + pInfo->aConstraintUsage[isEq[1]].argvIndex = ++argvIndex; + if( flags & 0x20 ) pInfo->aConstraintUsage[isEq[1]].omit = 1; + for(i=2; i<=4 && isEq[i]>=0; i++){ + if( i==4 && sqlite3_vtab_in(pInfo, isEq[4], 0) ) break; + pInfo->aConstraintUsage[isEq[i]].argvIndex = ++argvIndex; + if( flags & 0x20 ) pInfo->aConstraintUsage[isEq[i]].omit = 1; + v /= 10; + } + pInfo->idxNum = i; + if( isEq[4]>=0 && sqlite3_vtab_in(pInfo,isEq[4],1) ){ + iIn = isEq[4]; + pInfo->aConstraintUsage[iIn].argvIndex = ++argvIndex; + if( flags & 0x20 ) pInfo->aConstraintUsage[iIn].omit = 1; + v /= 5; + i++; + pInfo->idxNum += 4; + } + pInfo->estimatedCost = v; + } + pInfo->estimatedRows = (sqlite3_int64)pInfo->estimatedCost; + + /* Attempt to consume the ORDER BY clause. Except, always leave + ** orderByConsumed set to 0 for vt02_no_sort_opt. In this way, + ** we can compare vt02 and vt02_no_sort_opt to ensure they get + ** the same answer. + */ + if( pInfo->nOrderBy>0 && (flags & VT02_NO_SORT_OPT)==0 ){ + if( pInfo->idxNum==1 ){ + /* There will only be one row of output. So it is always sorted. */ + pInfo->orderByConsumed = 1; + }else + if( pInfo->aOrderBy[0].iColumn<=0 + && pInfo->aOrderBy[0].desc==0 + ){ + /* First column of order by is X ascending */ + pInfo->orderByConsumed = 1; + }else + if( sqlite3_vtab_distinct(pInfo)>=1 ){ + unsigned int x = 0; + for(i=0; inOrderBy; i++){ + int iCol = pInfo->aOrderBy[i].iColumn; + if( iCol<0 ) iCol = 0; + x |= 1<idxNum += 30; + pInfo->orderByConsumed = 1; + }else if( x==0x06 ){ + /* DISTINCT A,B */ + pInfo->idxNum += 20; + pInfo->orderByConsumed = 1; + }else if( x==0x0e ){ + /* DISTINCT A,B,C */ + pInfo->idxNum += 10; + pInfo->orderByConsumed = 1; + }else if( x & 0x01 ){ + /* DISTINCT X */ + pInfo->orderByConsumed = 1; + }else if( x==0x1e ){ + /* DISTINCT A,B,C,D */ + pInfo->orderByConsumed = 1; + } + }else{ + if( x==0x02 ){ + /* GROUP BY A */ + pInfo->orderByConsumed = 1; + }else if( x==0x06 ){ + /* GROUP BY A,B */ + pInfo->orderByConsumed = 1; + }else if( x==0x0e ){ + /* GROUP BY A,B,C */ + pInfo->orderByConsumed = 1; + }else if( x & 0x01 ){ + /* GROUP BY X */ + pInfo->orderByConsumed = 1; + }else if( x==0x1e ){ + /* GROUP BY A,B,C,D */ + pInfo->orderByConsumed = 1; + } + } + } + } + + if( flags & VT02_ALLOC_IDXSTR ){ + pInfo->idxStr = sqlite3_mprintf("test"); + pInfo->needToFreeIdxStr = 1; + } + if( flags & VT02_BAD_IDXNUM ){ + pInfo->idxNum += 1000; + } + + if( iOffset>=0 ){ + pInfo->aConstraintUsage[iOffset].argvIndex = ++argvIndex; + if( (flags & VT02_NO_OFFSET)==0 + && (pInfo->nOrderBy==0 || pInfo->orderByConsumed) + ){ + pInfo->aConstraintUsage[iOffset].omit = 1; + pInfo->idxNum += 100; + } + } + + + /* Always omit flags= and logtab= constraints to prevent them from + ** interfering with the bytecode. Put them at the end of the argv[] + ** array to keep them out of the way. + */ + if( iFlagTerm>=0 ){ + pInfo->aConstraintUsage[iFlagTerm].omit = 1; + pInfo->aConstraintUsage[iFlagTerm].argvIndex = ++argvIndex; + } + if( iLogTerm>=0 ){ + pInfo->aConstraintUsage[iLogTerm].omit = 1; + pInfo->aConstraintUsage[iLogTerm].argvIndex = ++argvIndex; + } + + /* The 0x40 flag means add all usable constraints to the output set */ + if( flags & 0x40 ){ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraintUsage[i].argvIndex==0 + ){ + pInfo->aConstraintUsage[i].argvIndex = ++argvIndex; + if( flags & 0x20 ) pInfo->aConstraintUsage[i].omit = 1; + } + } + } + + + /* Generate the log if requested */ + if( zLogTab ){ + static const char *azColname[] = { + "x", "a", "b", "c", "d", "flags", "logtab" + }; + sqlite3 *db = ((vt02_vtab*)pVTab)->db; + sqlite3BestIndexLog(pInfo, zLogTab, db, azColname, pVTab); + } + pSelf->busy--; + + /* Try to do a memory allocation solely for the purpose of causing + ** an error under OOM testing loops */ + pX = sqlite3_malloc(800); + if( pX==0 ) return SQLITE_NOMEM; + sqlite3_free(pX); + + return pVTab->zErrMsg!=0 ? SQLITE_ERROR : SQLITE_OK; +} + +/* This is the sqlite3_module definition for the the virtual table defined +** by this include file. +*/ +const sqlite3_module vt02Module = { + /* iVersion */ 2, + /* xCreate */ 0, /* This is an eponymous table */ + /* xConnect */ vt02Connect, + /* xBestIndex */ vt02BestIndex, + /* xDisconnect */ vt02Disconnect, + /* xDestroy */ vt02Disconnect, + /* xOpen */ vt02Open, + /* xClose */ vt02Close, + /* xFilter */ vt02Filter, + /* xNext */ vt02Next, + /* xEof */ vt02Eof, + /* xColumn */ vt02Column, + /* xRowid */ vt02Rowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindFunction */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0 +}; + +static void vt02CoreInit(sqlite3 *db){ + static const char zPkXSchema[] = + "CREATE TABLE x(x INT NOT NULL PRIMARY KEY, a INT, b INT, c INT, d INT," + " flags INT HIDDEN, logtab TEXT HIDDEN);"; + static const char zPkABCDSchema[] = + "CREATE TABLE x(x INT, a INT NOT NULL, b INT NOT NULL, c INT NOT NULL, " + "d INT NOT NULL, flags INT HIDDEN, logtab TEXT HIDDEN, " + "PRIMARY KEY(a,b,c,d));"; + sqlite3_create_module(db, "vt02", &vt02Module, 0); + sqlite3_create_module(db, "vt02pkx", &vt02Module, (void*)zPkXSchema); + sqlite3_create_module(db, "vt02pkabcd", &vt02Module, (void*)zPkABCDSchema); +} + +#ifdef TH3_VERSION +static void vt02_init(th3state *p, int iDb, char *zArg){ + vt02CoreInit(th3dbPointer(p, iDb)); +} +#else +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_vt02_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + vt02CoreInit(db); + return SQLITE_OK; +} +#endif /* TH3_VERSION */ diff --git a/test/wapptest.tcl b/test/wapptest.tcl index fa28ec7600..d37b2e48c6 100755 --- a/test/wapptest.tcl +++ b/test/wapptest.tcl @@ -476,7 +476,7 @@ proc generate_main_page {{extra {}}} { generate_select_widget Test control_test $lOpt $G(test) # Build the "jobs" select widget. Options are 1 to 8. - generate_select_widget Jobs control_jobs {1 2 3 4 5 6 7 8} $G(jobs) + generate_select_widget Jobs control_jobs {1 2 3 4 5 6 7 8 12 16} $G(jobs) switch $G(state) { config { diff --git a/test/widetab1.test b/test/widetab1.test new file mode 100644 index 0000000000..39523ce882 --- /dev/null +++ b/test/widetab1.test @@ -0,0 +1,156 @@ +# 2022-10-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements test cases for wide table (tables with more than +# 64 columns) and indexes that reference columns beyond the 63rd or 64th +# column. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix widetab1 + + +# In order to pick the better index in the following query, SQLite needs to +# be able to detect when an index that references later columns in a wide +# table is a covering index. +# +do_execsql_test 100 { + CREATE TABLE a( + a00, a01, a02, a03, a04, a05, a06, a07, a08, a09, + a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, + a20, a21, a22, a23, a24, a25, a26, a27, a28, a29, + a30, a31, a32, a33, a34, a35, a36, a37, a38, a39, + a40, a41, a42, a43, a44, a45, a46, a47, a48, a49, + a50, a51, a52, a53, a54, a55, a56, a57, a58, a59, + pd, bn, vb, bc, cn, ie, qm); + CREATE INDEX a1 on a(pd, bn, vb, bc, cn); -- preferred index + CREATE INDEX a2 on a(pd, bc, ie, qm); -- suboptimal index + CREATE TABLE b(bg, bc, bn, iv, ln, mg); + CREATE INDEX b1 on b(bn, iv, bg); +} +do_eqp_test 110 { + SELECT dc, count(cn) + FROM (SELECT coalesce(b.bg, a.bc) as dc, cn + FROM a LEFT JOIN b + ON a.bn = b.bn + AND CASE WHEN a.vb IS NOT NULL THEN 1 ELSE 0 END = b.iv + WHERE pd BETWEEN 0 AND 10) + GROUP BY dc; +} { + QUERY PLAN + |--SEARCH a USING COVERING INDEX a1 (pd>? AND pd diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index aaeb67bdfb..8f9a1b7b09 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -129,6 +129,8 @@ REM SET __ECHO2=ECHO REM SET __ECHO3=ECHO IF NOT DEFINED _AECHO (SET _AECHO=REM) IF NOT DEFINED _CECHO (SET _CECHO=REM) +IF NOT DEFINED _CECHO2 (SET _CECHO2=REM) +IF NOT DEFINED _CECHO3 (SET _CECHO3=REM) IF NOT DEFINED _VECHO (SET _VECHO=REM) SET REDIRECT=^> @@ -177,6 +179,7 @@ REM REM NOTE: Change the current directory to the root of the source tree, saving REM the current directory on the directory stack. REM +%_CECHO2% PUSHD "%ROOT%" %__ECHO2% PUSHD "%ROOT%" IF ERRORLEVEL 1 ( @@ -524,6 +527,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM REM NOTE: Attempt to setup the MSVC environment for this platform. REM + %_CECHO3% CALL "%VCVARSALL%" %%P %__ECHO3% CALL "%VCVARSALL%" %%P IF ERRORLEVEL 1 ( @@ -749,6 +753,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM REM NOTE: Restore the saved current directory from the directory stack. REM +%_CECHO2% POPD %__ECHO2% POPD IF ERRORLEVEL 1 ( diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl index 1120bc1316..0438b11b35 100755 --- a/tool/mkctimec.tcl +++ b/tool/mkctimec.tcl @@ -43,7 +43,7 @@ set ::headCode " ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -#include \"config.h\" +#include \"sqlite_cfg.h\" #define SQLITECONFIG_H 1 #endif @@ -304,6 +304,7 @@ set value_options { SQLITE_DEFAULT_WAL_AUTOCHECKPOINT SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_WORKER_THREADS + SQLITE_DQS SQLITE_ENABLE_8_3_NAMES SQLITE_ENABLE_CEROD SQLITE_ENABLE_LOCKING_STYLE diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 595fc4c602..0cdf514e44 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -355,6 +355,7 @@ foreach file { hash.c opcodes.c + os_kv.c os_unix.c os_win.c memdb.c diff --git a/tool/stripccomments.c b/tool/stripccomments.c new file mode 100644 index 0000000000..53933c0138 --- /dev/null +++ b/tool/stripccomments.c @@ -0,0 +1,228 @@ +/** + Strips C- and C++-style comments from stdin, sending the results to + stdout. It assumes that its input is legal C-like code, and does + only little error handling. + + It treats string literals as anything starting and ending with + matching double OR single quotes OR backticks (for use with + scripting languages which use those). It assumes that a quote + character within a string which uses the same quote type is escaped + by a backslash. It should not be used on any code which might + contain C/C++ comments inside heredocs, and similar constructs, as + it will strip those out. + + Usage: $0 [--keep-first|-k] < input > output + + The --keep-first (-k) flag tells it to retain the first comment in the + input stream (which is often a license or attribution block). It + may be given repeatedly, each one incrementing the number of + retained comments by one. + + License: Public Domain + Author: Stephan Beal (stephan@wanderinghorse.net) +*/ +#include +#include +#include + +#if 1 +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:\t",__FILE__,__LINE__); \ + printf pfexp; \ + } while(0) +#else +#define MARKER(exp) if(0) printf +#endif + +struct { + FILE * input; + FILE * output; + int rc; + int keepFirst; +} App = { + 0/*input*/, + 0/*output*/, + 0/*rc*/, + 0/*keepFirst*/ +}; + +void do_it_all(void){ + enum states { + S_NONE = 0 /* not in comment */, + S_SLASH1 = 1 /* slash - possibly comment prefix */, + S_CPP = 2 /* in C++ comment */, + S_C = 3 /* in C comment */ + }; + int ch, prev = EOF; + FILE * out = App.output; + int const slash = '/'; + int const star = '*'; + int line = 1; + int col = 0; + enum states state = S_NONE /* current state */; + int elide = 0 /* true if currently eliding output */; + int state3Col = -99 + /* huge kludge for odd corner case: */ + /*/ <--- here. state3Col marks the source column in which a C-style + comment starts, so that it can tell if star-slash inside a + C-style comment is the end of the comment or is the weird corner + case marked at the start of _this_ comment block. */; + for( ; EOF != (ch = fgetc(App.input)); prev = ch, + ++col){ + switch(state){ + case S_NONE: + if('\''==ch || '"'==ch || '`'==ch){ + /* Read string literal... + needed to properly catch comments in strings. */ + int const quote = ch, + startLine = line, startCol = col; + int ch2, escaped = 0, endOfString = 0; + fputc(ch, out); + for( ++col; !endOfString && EOF != (ch2 = fgetc(App.input)); + ++col ){ + switch(ch2){ + case '\\': escaped = !escaped; + break; + case '`': + case '\'': + case '"': + if(!escaped && quote == ch2) endOfString = 1; + escaped = 0; + break; + default: + escaped = 0; + break; + } + if('\n'==ch2){ + ++line; + col = 0; + } + fputc(ch2, out); + } + if(EOF == ch2){ + fprintf(stderr, "Unexpected EOF while reading %s literal " + "on line %d column %d.\n", + ('\''==ch) ? "char" : "string", + startLine, startCol); + App.rc = 1; + return; + } + break; + } + else if(slash == ch){ + /* MARKER(("state 0 ==> 1 @ %d:%d\n", line, col)); */ + state = S_SLASH1; + break; + } + fputc(ch, out); + break; + case S_SLASH1: /* 1 slash */ + /* MARKER(("SLASH1 @ %d:%d App.keepFirst=%d\n", + line, col, App.keepFirst)); */ + switch(ch){ + case '*': + /* Enter C comment */ + if(App.keepFirst>0){ + elide = 0; + --App.keepFirst; + }else{ + elide = 1; + } + /*MARKER(("state 1 ==> 3 @ %d:%d\n", line, col));*/ + state = S_C; + state3Col = col-1; + if(!elide){ + fputc(prev, out); + fputc(ch, out); + } + break; + case '/': + /* Enter C++ comment */ + if(App.keepFirst>0){ + elide = 0; + --App.keepFirst; + }else{ + elide = 1; + } + /*MARKER(("state 1 ==> 2 @ %d:%d\n", line, col));*/ + state = S_CPP; + if(!elide){ + fputc(prev, out); + fputc(ch, out); + } + break; + default: + /* It wasn't a comment after all. */ + state = S_NONE; + if(!elide){ + fputc(prev, out); + fputc(ch, out); + } + } + break; + case S_CPP: /* C++ comment */ + if('\n' == ch){ + /* MARKER(("state 2 ==> 0 @ %d:%d\n", line, col)); */ + state = S_NONE; + elide = 0; + } + if(!elide){ + fputc(ch, out); + } + break; + case S_C: /* C comment */ + if(!elide){ + fputc(ch, out); + } + if(slash == ch){ + if(star == prev){ + /* MARKER(("state 3 ==> 0 @ %d:%d\n", line, col)); */ + /* Corner case which breaks this: */ + /*/ <-- slash there */ + /* That shows up twice in a piece of 3rd-party + code i use. */ + /* And thus state3Col was introduced :/ */ + if(col!=state3Col+2){ + state = S_NONE; + elide = 0; + state3Col = -99; + } + } + } + break; + default: + assert(!"impossible!"); + break; + } + if('\n' == ch){ + ++line; + col = 0; + state3Col = -99; + } + } +} + +static void usage(char const *zAppName){ + fprintf(stderr, "Strips C- and C++-style comments from stdin and sends " + "the results to stdout.\n"); + fprintf(stderr, "Usage: %s [--keep-first|-k] < input > output\n", zAppName); +} + +int main( int argc, char const * const * argv ){ + int i; + for(i = 1; i < argc; ++i){ + char const * zArg = argv[i]; + while( '-'==*zArg ) ++zArg; + if( 0==strcmp(zArg,"k") + || 0==strcmp(zArg,"keep-first") ){ + ++App.keepFirst; + }else{ + usage(argv[0]); + return 1; + } + } + App.input = stdin; + App.output = stdout; + do_it_all(); + return App.rc ? 1 : 0; +}