From b0ab21dee40144d8aec92df4a0c98eb069badeb3 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 3 Nov 2022 22:14:47 +0000 Subject: [PATCH 001/282] Experimentally use clang's C preprocessor to filter #ifdef's out of the generated sqlite3-api.js, the goal being to see if we can feasibly use cpp to include ES6 module-specific code in the main code base and conditionally filter it out. FossilOrigin-Name: 718a6d371e61359d73c8f80afdb248e3d9b4d8df4c4e5c122ac884344e31035b --- ext/wasm/GNUmakefile | 53 ++++++++++++++++++++++++++++++-- ext/wasm/api/pre-js.js | 7 ++++- ext/wasm/api/sqlite3-api-opfs.js | 11 +++++-- manifest | 19 +++++++----- manifest.uuid | 2 +- 5 files changed, 76 insertions(+), 16 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 993c668697..069e2a1da9 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -96,7 +96,7 @@ dir.dout := $(dir.wasm)/jswasm # dir.tmp = output dir for intermediary build files, as opposed to # end-user deliverables. dir.tmp := $(dir.wasm)/bld -#CLEAN_FILES += $(wildcard $(dir.dout)/*) $(wildcard $(dir.tmp)/*) +CLEAN_FILES += $(dir.tmp)/* $(dir.dout)/* ifeq (,$(wildcard $(dir.dout))) dir._tmp := $(shell mkdir -p $(dir.dout)) endif @@ -235,7 +235,8 @@ $(foreach X,$(SOAP.js) $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js),\ all: $(sqlite3-api.ext.jses) sqlite3-api.js := $(dir.tmp)/sqlite3-api.js -$(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) +sqlite3-api.pre-cpp.js := $(dir.tmp)/sqlite3-api.pre-cpp.js +$(sqlite3-api.pre-cpp.js): $(sqlite3-api.jses) $(MAKEFILE) @echo "Making $@..." @for i in $(sqlite3-api.jses); do \ echo "/* BEGIN FILE: $$i */"; \ @@ -256,7 +257,8 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) ######################################################################## # --post-js and --pre-js are emcc flags we use to append/prepend JS to # the generated emscripten module file. -pre-js.js := $(dir.api)/pre-js.js +pre-js.pre-cpp.js := $(dir.api)/pre-js.pre-cpp.js +pre-js.js := $(dir.tmp)/pre-js.js post-js.js := $(dir.tmp)/post-js.js post-jses := \ $(dir.api)/post-js-header.js \ @@ -288,6 +290,51 @@ $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) echo '*/'; \ } > $@ +######################################################################## +# Transform $(1) to $(2) via cpp -E -CC $(1) ... +# +# DO NOT use gcc as the preprocessor because it will emit its own +# license header to our output because that's a comment in its +# stdc-predef.h, which we cannot tell it to _not_ include. The only +# workaround to that is to allow gcc -E to strip all comments. The +# wasm build uses clang behind emcc, anyway, so we already have a +# clang dependency. However, the clang cpp refuses to read empty +# strings in the form '', so we have to be sure to use "" in JS code +# for those. +# +# It's tempting to build a custom mini-cpp-like binary for this +# purpose to avoid these dependencies and quirks. Maybe we could use +# lemon to do the heavy lifting for that, noting that we'd still need +# to tokenize it by hand (but only lines which start with "#" or +# backslash-continued lines, and we could treat all other lines as +# opaque content). +# +# In this build we may have #ifdef's (and the like) in arbitrary input +# JS files and we need to preprocess those before Emscripten gets +# ahold of them. We cannot simply preprocess the resulting +# Emscripten-generated sqlite3.js because (A) Emscripten may choke on +# C preprocessor directives in the input and (B) Emscripten's output +# may contain things which cpp cannot read (like single-quoted empty +# strings: ''). +bin.cpp ?= clang +ifneq (,$(filter ems,$(MAKECMDGOALS))) +js.cpp.defines ?= -DSQLITE_JS_ESM +ems: $(filter-out ems,$(MAKECMDGOALS)) +else +js.cpp.defines ?= +endif +define CPP_JS +# $1 = X.js. $2 = output file to generate by filtering $(1) through +# $(bin.cpp) -E -CC. +$(2): $(1) $$(MAKEFILE) + $$(bin.cpp) -E -CC -undef $(js.cpp.defines) -x c $(1) | sed -e '/^#/d' > $$@ +CLEAN_FILES += $(2) +endef +$(eval $(call CPP_JS,$(dir.tmp)/sqlite3-api.pre-cpp.js,$(dir.tmp)/sqlite3-api.js)) +$(eval $(call CPP_JS,$(dir.api)/pre-js.js,$(dir.tmp)/pre-js.js)) +# /end CPP-of-JS bits +######################################################################## + ######################################################################## # call-make-pre-js creates rules for pre-js-$(1).js. $1 = the base # name of the JS file on whose behalf this pre-js is for. diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.js index f31dea1794..f959c33072 100644 --- a/ext/wasm/api/pre-js.js +++ b/ext/wasm/api/pre-js.js @@ -29,6 +29,10 @@ sqlite3InitModuleState.debugModule('self.location =',self.location); 4) If none of the above apply, (prefix+path) is returned. */ Module['locateFile'] = function(path, prefix) { +#ifdef SQLITE_JS_ESM + return new URL(path, import.meta.url).href; +#else + 'use strict'; let theFile; const up = this.urlParams; if(up.has(path)){ @@ -47,6 +51,7 @@ Module['locateFile'] = function(path, prefix) { "result =", theFile ); return theFile; +#endif /* SQLITE_JS_EMS */ }.bind(sqlite3InitModuleState); /** @@ -65,7 +70,7 @@ Module[xNameOfInstantiateWasm] = function callee(imports,onSuccess){ const uri = Module.locateFile( callee.uri, ( ('undefined'===typeof scriptDirectory/*var defined by Emscripten glue*/) - ? '' : scriptDirectory) + ? "" : scriptDirectory) ); sqlite3InitModuleState.debugModule( "instantiateWasm() uri =", uri diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 86285df1d3..3cb5c8a3eb 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -166,7 +166,12 @@ const installOpfsVfs = function callee(options){ opfsVfs.dispose(); return promiseReject_(err); }; - const W = new Worker(options.proxyUri); + const W = +#ifdef SQLITE_JS_ESM + new Worker(new URL(options.proxyUri, import.meta.url)); +#else + new Worker(options.proxyUri); +#endif W._originalOnError = W.onerror /* will be restored later */; W.onerror = function(err){ // The error object doesn't contain any useful info when the @@ -566,7 +571,7 @@ const installOpfsVfs = function callee(options){ const ndx = Math.random() * (f._n * 64) % f._n | 0; a[i] = f._chars[ndx]; } - return a.join(''); + return a.join(""); }; /** @@ -1155,7 +1160,7 @@ const installOpfsVfs = function callee(options){ be set here. */ //"pragma cache_size=-8388608;" - ].join('') + ].join("") ); } diff --git a/manifest b/manifest index 3942c53535..d07080d368 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Globally\sreplace\s''\swith\s""\sfor\sempty\sJS\sstrings\sto\splease\sC\spreprocessor. -D 2022-11-03T21:21:10.223 +C Experimentally\suse\sclang's\sC\spreprocessor\sto\sfilter\s#ifdef's\sout\sof\sthe\sgenerated\ssqlite3-api.js,\sthe\sgoal\sbeing\sto\ssee\sif\swe\scan\sfeasibly\suse\scpp\sto\sinclude\sES6\smodule-specific\scode\sin\sthe\smain\scode\sbase\sand\sconditionally\sfilter\sit\sout. +D 2022-11-03T22:14:47.036 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -487,7 +487,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 78b64d110e66e18709c11437a74c29bc7855d85a713217db99127c7f304a332f +F ext/wasm/GNUmakefile 5f04a4703fa1cbf3b8a031b9875828326c120fd98f08b68e889938f006e26973 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 @@ -497,11 +497,11 @@ F ext/wasm/api/extern-post-js.js f3dc4219a2a1f183d98452dcbd55a0c5351b759eccca754 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 -F ext/wasm/api/pre-js.js 287e462f969342b032c03900e668099fa1471d852df7a472de5bc349161d9c04 +F ext/wasm/api/pre-js.js 9620327120abb15b3af96f72ef9efbcf69e78d90e501328521108b93547a8eb8 F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js cdcbb57acc66f4569ac9e18f9d13d5a3657d8aae195725c6324943da56c1005d +F ext/wasm/api/sqlite3-api-opfs.js 700c1b68c324fb783bb13588f972671e51fbc4b063402768be962df8be7e5142 F ext/wasm/api/sqlite3-api-prologue.js 952ba908cc5ee42728c5c09dd549af32ef0c3cc15ab7b919a8007c5684f69320 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -2054,8 +2054,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 70ee6ee014bc4e2c1daa9b4a8909cf76ccecf32de1eb39055f20d3d0b1daa1bd -R eaaf8a7ca25305cffe9e8f7e734e3ee8 +P e92e1f42bef94a1df29f66b4111ebfde93eba3759bc5d5a9c95f714508851346 +R c242b13966143759c21878706550a242 +T *branch * js-cpp +T *sym-js-cpp * +T -sym-trunk * Cancelled\sby\sbranch. U stephan -Z b98d96f31e9b735c045c03e6eefd13f8 +Z c0f79930e1701e29aa37bb40d0432abe # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8348da8467..125d8b2c56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e92e1f42bef94a1df29f66b4111ebfde93eba3759bc5d5a9c95f714508851346 \ No newline at end of file +718a6d371e61359d73c8f80afdb248e3d9b4d8df4c4e5c122ac884344e31035b \ No newline at end of file From 6e8a3341ea48c15066d11281d14363b5454538c1 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 6 Nov 2022 13:12:11 +0000 Subject: [PATCH 002/282] shell.c.in: on non-Windows platforms, check for $XDG_CONFIG_HOME/sqlite3/sqliterc before ~/.sqliterc, per request in [forum:7a16582b1e403c81|forum post 7a16582b1e403c81]. FossilOrigin-Name: 49c6e438a83b9ff40ebadd3dfd5f58e6eea053575e15335909f5ee59a6dba82c --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/shell.c.in | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index db23c1cbb3..bbdd1009b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\sfile\sdbpagefault.test\sto\saccount\sfor\sthe\srestriction\son\susing\sSQLITE_VTAB_DIRECTONLY\svirtual\stables\sfrom\swithin\striggers. -D 2022-11-05T19:26:45.587 +C shell.c.in:\son\snon-Windows\splatforms,\scheck\sfor\s$XDG_CONFIG_HOME/sqlite3/sqliterc\sbefore\s~/.sqliterc,\sper\srequest\sin\s[forum:7a16582b1e403c81|forum\spost\s7a16582b1e403c81]. +D 2022-11-06T13:12:11.629 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -637,7 +637,7 @@ F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 12cb5162e606290354f9512ff0c30fc6dc4d7a77a92b6c5b581395f9c4edd047 -F src/shell.c.in 84bb08d8762920285f08f1c0993f1b3992ac43af5d72445cb8a973fc50c71923 +F src/shell.c.in eae218c72d127dafdd46f768380a0c338e69dd50b9d7993ec3dbca7c11ab48ee F src/sqlite.h.in bf5846820130b6cf01b002e90427eae29f02db07d9cb9b5ccd0e0aad867eed14 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2054,8 +2054,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 454c61e818f6941c9a23f6600e37828a3f2b2ad3c6dbc8d6223570aed5d9cd8b -R 5bb95939440d8668f376bffbf10c30f6 -U dan -Z e0d6aeb7f7ce3a618693295f8bdfb5c8 +P 2b68fc8aa35cc69e4d2c26aaebbf3f2b1f1c08d15ca9efcbe5be21d45735d3f1 +R 761db6cbff4c93d54e2273bd93e61ecf +T *branch * sqliterc-xdg-config +T *sym-sqliterc-xdg-config * +T -sym-trunk * Cancelled\sby\sbranch. +U stephan +Z b1d4f5aca54a8028201ab20f9960723f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c457502351..ac44292681 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b68fc8aa35cc69e4d2c26aaebbf3f2b1f1c08d15ca9efcbe5be21d45735d3f1 \ No newline at end of file +49c6e438a83b9ff40ebadd3dfd5f58e6eea053575e15335909f5ee59a6dba82c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 635361aa92..9d4689bd9e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11251,9 +11251,43 @@ static char *find_home_dir(int clearFlag){ return home_dir; } +/* +** On non-Windows platforms, look for $XDG_CONFIG_HOME. +** If ${XDG_CONFIG_HOME}/sqlite3/sqliterc is found, return +** the path to it, else return 0. The result is cached for +** subsequent calls. +*/ +static const char *find_xdg_config(void){ +#if defined(_WIN32) || defined(WIN32) || defined(_WIN32_WCE) \ + || defined(__RTP__) || defined(_WRS_KERNEL) + return 0; +#else + static int alreadyTried = 0; + static char *zConfig = 0; + const char *zXdgHome; + + if( alreadyTried!=0 ){ + return zConfig; + } + alreadyTried = 1; + zXdgHome = getenv("XDG_CONFIG_HOME"); + if( zXdgHome==0 ){ + return 0; + } + zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome); + shell_check_oom(zConfig); + if( access(zConfig,0)!=0 ){ + sqlite3_free(zConfig); + zConfig = 0; + } + return zConfig; +#endif +} + /* ** Read input from the file given by sqliterc_override. Or if that -** parameter is NULL, take input from ~/.sqliterc +** parameter is NULL, take input from the first of find_xdg_config() +** or ~/.sqliterc which is found. ** ** Returns the number of errors. */ @@ -11267,7 +11301,10 @@ static void process_sqliterc( FILE *inSaved = p->in; int savedLineno = p->lineno; - if (sqliterc == NULL) { + if( sqliterc == NULL ){ + sqliterc = find_xdg_config(); + } + if( sqliterc == NULL ){ home_dir = find_home_dir(0); if( home_dir==0 ){ raw_printf(stderr, "-- warning: cannot find home directory;" From 873a84054c4ab9f8ef910d98970dd9b6882ff024 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 10 Nov 2022 19:19:52 +0000 Subject: [PATCH 003/282] Avoid returning SQLITE_SCHEMA if the first query run on a connection is "SELECT nosuchcolumn" or similar. Forum post . FossilOrigin-Name: d31c019fd6849e4c3f8452e75c6cfefd613ade5355e896be368bd16fef28c627 --- manifest | 18 ++++++++++-------- manifest.uuid | 2 +- src/prepare.c | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 4ca67cf05f..b7acdc649e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Automatically\srelinquish\simplicitly-acquired\sOPFS\sfile\slocks\sduring\sVFS\sidle\stime\sin\san\sattempt\sto\shelp\salleviate\scross-tab\slocking\scontention\slike\sthat\sdescribed\sin\s[forum:58a377083cd24a|forum\spost\s58a377083cd24a]. -D 2022-11-10T13:22:35.705 +C Avoid\sreturning\sSQLITE_SCHEMA\sif\sthe\sfirst\squery\srun\son\sa\sconnection\sis\s"SELECT\snosuchcolumn"\sor\ssimilar.\sForum\spost\s. +D 2022-11-10T19:19:52.657 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -632,7 +632,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 41430ca04735cc8e5d003bfd9315eadede3ec326e50805cc81bcf34e46601292 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 -F src/prepare.c 1b02be0441eda4579471fea097f678effcbb77ef0c39ab3f703c837822bcd674 +F src/prepare.c 9ebd3a1b12bbd1951f0d6db850f32cf5d4547a6ab8bb9e958d75dfbe4e60d0a3 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 @@ -2055,9 +2055,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ae43e97087a3207a5ca3ffae75fbe7a33c01f4a38ce0d1d7eed8591ae3083617 a7fe91afca473fe55c983bc81d214df4ef3699863c7423fa4b6b9cde23d6a3b4 -R 4034304670d7219ca07133486e9bbed5 -T +closed a7fe91afca473fe55c983bc81d214df4ef3699863c7423fa4b6b9cde23d6a3b4 Closed\sby\sintegrate-merge. -U stephan -Z 9ded40c19ad046a311beff933b79ea4d +P 8daf24ff73dd9928057412e0e4c2e2b2e47e1dca66acfb6b07c846e8d97582cc +R b4689da2653e425858ca303f2044a788 +T *branch * schema-error-fix +T *sym-schema-error-fix * +T -sym-trunk * +U dan +Z 8530f581085891f6457a19bb745014d9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7762727bb4..2ae594e645 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8daf24ff73dd9928057412e0e4c2e2b2e47e1dca66acfb6b07c846e8d97582cc \ No newline at end of file +d31c019fd6849e4c3f8452e75c6cfefd613ade5355e896be368bd16fef28c627 \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index 1e7a1222ba..7607387408 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -520,8 +520,8 @@ static void schemaIsValid(Parse *pParse){ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + if( DbHasProperty(db, iDb, DB_SchemaLoaded) ) pParse->rc = SQLITE_SCHEMA; sqlite3ResetOneSchema(db, iDb); - pParse->rc = SQLITE_SCHEMA; } /* Close the transaction, if one was opened. */ From 08efa4511314ca3e3ff75ab42463911c33c4468d Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 16 Nov 2022 19:57:21 +0000 Subject: [PATCH 004/282] Fix harmless typo in comment, reported by Debian in Fossil [https://fossil-scm.org/forum/forumpost/15f7327318]. FossilOrigin-Name: 5689f0d9ad1be532b274508938b25ff0d63027b8cc31f796dfaa2cca71d53642 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/btree.h | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 927dac267f..f6d2e042ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sreturning\sSQLITE_SCHEMA\sif\sthe\sfirst\squery\srun\son\sa\sconnection\nis\s"SELECT\snosuchcolumn"\sor\ssimilar.\n[forum:/forumpost/c1798f77ef|Forum\spost\sc1798f77ef] -D 2022-11-16T19:53:39.411 +C Fix\sharmless\stypo\sin\scomment,\sreported\sby\sDebian\sin\sFossil\n[https://fossil-scm.org/forum/forumpost/15f7327318]. +D 2022-11-16T19:57:21.702 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -577,7 +577,7 @@ F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca F src/btree.c 6321ff29261bf9726e6b231058ff21b1ccf9f441a0b718b76c37341b16fa14ce -F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 +F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a @@ -2055,9 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 89c459e766ea7e9165d0beeb124708b955a4950d0f4792f457465d71b158d318 d31c019fd6849e4c3f8452e75c6cfefd613ade5355e896be368bd16fef28c627 -R e94eb3c7c74638f5ff37a1f014ec2d64 -T +closed d31c019fd6849e4c3f8452e75c6cfefd613ade5355e896be368bd16fef28c627 +P 3dc4f75d77417df1ef19be8e3191d246fb44ca7f7d1de7b161c5cb0f8aafeded +R 71ea3d9237d5fb9ab3af1d4736adae67 U drh -Z 36ff02ae68537dd51ac89f63caa3f038 +Z 805fbf7484712ef0033dcf118d9ea96f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 010eb2606b..10751f5e9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3dc4f75d77417df1ef19be8e3191d246fb44ca7f7d1de7b161c5cb0f8aafeded \ No newline at end of file +5689f0d9ad1be532b274508938b25ff0d63027b8cc31f796dfaa2cca71d53642 \ No newline at end of file diff --git a/src/btree.h b/src/btree.h index f80ba4a97b..7f31c6020f 100644 --- a/src/btree.h +++ b/src/btree.h @@ -183,7 +183,7 @@ int sqlite3BtreeNewDb(Btree *p); ** reduce network bandwidth. ** ** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by -** standard SQLite. The other hints are provided for extentions that use +** standard SQLite. The other hints are provided for extensions that use ** the SQLite parser and code generator but substitute their own storage ** engine. */ From 1d4f1cf4266fb723673987fc1f3e27da78439d0c Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 16 Nov 2022 21:39:23 +0000 Subject: [PATCH 005/282] wasm build: include FTS5 instead of FTS4, per /chat and HN discussions. FossilOrigin-Name: 85c3f2ac5a887ba809f236c8c9d3837b2fdb578d4beae61bc78ada7ce03b58e2 --- ext/wasm/GNUmakefile | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 039dff410f..42a78c8000 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -110,7 +110,7 @@ emcc.WASM_BIGINT ?= 1 sqlite3.c := $(dir.top)/sqlite3.c sqlite3.h := $(dir.top)/sqlite3.h SQLITE_OPT = \ - -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ -DSQLITE_ENABLE_RTREE \ -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ diff --git a/manifest b/manifest index f6d2e042ea..1061913bf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\stypo\sin\scomment,\sreported\sby\sDebian\sin\sFossil\n[https://fossil-scm.org/forum/forumpost/15f7327318]. -D 2022-11-16T19:57:21.702 +C wasm\sbuild:\sinclude\sFTS5\sinstead\sof\sFTS4,\sper\s/chat\sand\sHN\sdiscussions. +D 2022-11-16T21:39:23.062 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,7 +488,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 3aa8c160705ab9d9d4d552a1f4e630925a65a27df216befe2e9a956904434c1d +F ext/wasm/GNUmakefile dd6352a52e2742b78ea472acc41c6eea52165e25756eba4f1fea3f5dad1632b7 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 @@ -2055,8 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3dc4f75d77417df1ef19be8e3191d246fb44ca7f7d1de7b161c5cb0f8aafeded -R 71ea3d9237d5fb9ab3af1d4736adae67 -U drh -Z 805fbf7484712ef0033dcf118d9ea96f +P 5689f0d9ad1be532b274508938b25ff0d63027b8cc31f796dfaa2cca71d53642 +R 1644a92a7b62cf30dbc95dcea2edd9c6 +U stephan +Z a1a626093eb098d45fed407572cb36cb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 10751f5e9a..5d3d4cc6d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5689f0d9ad1be532b274508938b25ff0d63027b8cc31f796dfaa2cca71d53642 \ No newline at end of file +85c3f2ac5a887ba809f236c8c9d3837b2fdb578d4beae61bc78ada7ce03b58e2 \ No newline at end of file From a872466c8ed936e2ad68a8a2ba717a7bda8689bf Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 16 Nov 2022 21:52:29 +0000 Subject: [PATCH 006/282] Change a self.X JS reference to X to account for a symbol resolution discrepancy between vanilla JS and ES6 modules, as explained in [forum:801d8f77e5115141|forum post 801d8f77e5115141]. FossilOrigin-Name: 0590de4da1103d842b9f9f25bcd2e69223b2ea067ae2f320f58dd3763218b39d --- ext/wasm/api/extern-post-js.js | 5 ++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index 84b99b53a6..d933a36265 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -15,7 +15,10 @@ impls which Emscripten installs at some point in the file above this. */ - const originalInit = self.sqlite3InitModule; + const originalInit = + /*Maintenance reminde: DO NOT use `self.` here. It's correct + for non-ES6 Module cases but wrong for ES6 modules because those + resolve this symbol differently! */ sqlite3InitModule; if(!originalInit){ throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); } diff --git a/manifest b/manifest index 1061913bf1..987375b147 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm\sbuild:\sinclude\sFTS5\sinstead\sof\sFTS4,\sper\s/chat\sand\sHN\sdiscussions. -D 2022-11-16T21:39:23.062 +C Change\sa\sself.X\sJS\sreference\sto\sX\sto\saccount\sfor\sa\ssymbol\sresolution\sdiscrepancy\sbetween\svanilla\sJS\sand\sES6\smodules,\sas\sexplained\sin\s[forum:801d8f77e5115141|forum\spost\s801d8f77e5115141]. +D 2022-11-16T21:52:29.439 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 1350088aee90e959ad9a94fab1bb6bcb5e99d4d27f976db389050f54f2640c78 -F ext/wasm/api/extern-post-js.js f3dc4219a2a1f183d98452dcbd55a0c5351b759eccca75480a92473974d8b047 +F ext/wasm/api/extern-post-js.js c197b7567496fc27766842f8c4f4963054bf8c667926ab3df4c91aa548939ce6 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 @@ -2055,8 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5689f0d9ad1be532b274508938b25ff0d63027b8cc31f796dfaa2cca71d53642 -R 1644a92a7b62cf30dbc95dcea2edd9c6 +P 85c3f2ac5a887ba809f236c8c9d3837b2fdb578d4beae61bc78ada7ce03b58e2 +R 7b8987d4eb67ce150345834e9ef9546a U stephan -Z a1a626093eb098d45fed407572cb36cb +Z 50adbd5c6dbf458e11e189272fde1820 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5d3d4cc6d4..1f20330e90 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -85c3f2ac5a887ba809f236c8c9d3837b2fdb578d4beae61bc78ada7ce03b58e2 \ No newline at end of file +0590de4da1103d842b9f9f25bcd2e69223b2ea067ae2f320f58dd3763218b39d \ No newline at end of file From 841c98e14ca76ed96661583c4739629f0b2e876b Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 17 Nov 2022 01:24:06 +0000 Subject: [PATCH 007/282] Raise an error if an attempt is made to compile the CLI with the SQLITE_OMIT_COMPLETE option, since sqlite3_complete() really is necessary for the CLI to operate sanely. FossilOrigin-Name: a119a9e2ade4eac5feb1aa885c15b83e725f87386351de99d3abb49656219d50 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 8 +++----- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 987375b147..22f3b0c245 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\sself.X\sJS\sreference\sto\sX\sto\saccount\sfor\sa\ssymbol\sresolution\sdiscrepancy\sbetween\svanilla\sJS\sand\sES6\smodules,\sas\sexplained\sin\s[forum:801d8f77e5115141|forum\spost\s801d8f77e5115141]. -D 2022-11-16T21:52:29.439 +C Raise\san\serror\sif\san\sattempt\sis\smade\sto\scompile\sthe\sCLI\swith\sthe\nSQLITE_OMIT_COMPLETE\soption,\ssince\ssqlite3_complete()\sreally\sis\snecessary\nfor\sthe\sCLI\sto\soperate\ssanely. +D 2022-11-17T01:24:06.314 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -638,7 +638,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in 458cb3de9d548342fc645b699620b1af3de770d2ceec09ac71f86c19bd244064 +F src/shell.c.in a0ba4a297f8134fef1a6b618ac57604a6c4f1eadeab4f6950b99a8bc151f3c1f F src/sqlite.h.in bdb10b78166f5b735318667eb16c84ac90d9e0de88cc25c193eeb4379a126945 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2055,8 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 85c3f2ac5a887ba809f236c8c9d3837b2fdb578d4beae61bc78ada7ce03b58e2 -R 7b8987d4eb67ce150345834e9ef9546a -U stephan -Z 50adbd5c6dbf458e11e189272fde1820 +P 0590de4da1103d842b9f9f25bcd2e69223b2ea067ae2f320f58dd3763218b39d +R 9a1abb4a6d0ca4f10b6b70f7ecc4eaec +U drh +Z 23b35cc422152da909abe420b023163d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1f20330e90..ab03b00c16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0590de4da1103d842b9f9f25bcd2e69223b2ea067ae2f320f58dd3763218b39d \ No newline at end of file +a119a9e2ade4eac5feb1aa885c15b83e725f87386351de99d3abb49656219d50 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index f164789776..6b038495f5 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10959,13 +10959,11 @@ static int line_is_command_terminator(char *zLine){ } /* -** We need a default sqlite3_complete() implementation to use in case -** the shell is compiled with SQLITE_OMIT_COMPLETE. The default assumes -** any arbitrary text is a complete SQL statement. This is not very -** user-friendly, but it does seem to work. +** The CLI needs a working sqlite3_complete() to work properly. So error +** out of the build if compiling with SQLITE_OMIT_COMPLETE. */ #ifdef SQLITE_OMIT_COMPLETE -#define sqlite3_complete(x) 1 +# error the CLI application is imcompatable with SQLITE_OMIT_COMPLETE. #endif /* From 823872c6d635782030d529ca25f6a02a04fc6487 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 17 Nov 2022 13:58:25 +0000 Subject: [PATCH 008/282] Split out the documentation for sqlite3_value_encoding() into its own page and make it clear that this interface is intended for testing and debugging only. [forum:/forumpost/c9f445453da950ad|Forum thread c9f445453da950ad]. Comment changes only - no changes to code. FossilOrigin-Name: 9048a766ff7dfa0cd91ea74092e462f4501cb3f719033ccb55700bf5e4dfd0d3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 31 +++++++++++++++++++++---------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 22f3b0c245..65bffb3672 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Raise\san\serror\sif\san\sattempt\sis\smade\sto\scompile\sthe\sCLI\swith\sthe\nSQLITE_OMIT_COMPLETE\soption,\ssince\ssqlite3_complete()\sreally\sis\snecessary\nfor\sthe\sCLI\sto\soperate\ssanely. -D 2022-11-17T01:24:06.314 +C Split\sout\sthe\sdocumentation\sfor\ssqlite3_value_encoding()\sinto\sits\sown\npage\sand\smake\sit\sclear\sthat\sthis\sinterface\sis\sintended\sfor\stesting\sand\ndebugging\sonly.\n[forum:/forumpost/c9f445453da950ad|Forum\sthread\sc9f445453da950ad].\nComment\schanges\sonly\s-\sno\schanges\sto\scode. +D 2022-11-17T13:58:25.001 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -639,7 +639,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 F src/shell.c.in a0ba4a297f8134fef1a6b618ac57604a6c4f1eadeab4f6950b99a8bc151f3c1f -F src/sqlite.h.in bdb10b78166f5b735318667eb16c84ac90d9e0de88cc25c193eeb4379a126945 +F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f F src/sqliteInt.h 2c24ba38f78e32fe5d7ec136321a6ad827698b33ca98664970a8b7274d69ef7c @@ -2055,8 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0590de4da1103d842b9f9f25bcd2e69223b2ea067ae2f320f58dd3763218b39d -R 9a1abb4a6d0ca4f10b6b70f7ecc4eaec +P a119a9e2ade4eac5feb1aa885c15b83e725f87386351de99d3abb49656219d50 +R b081909c6ecb1750561be5091dea8e63 U drh -Z 23b35cc422152da909abe420b023163d +Z c8051a67682b7cc2de89c2c47da25213 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ab03b00c16..4a06fb59e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a119a9e2ade4eac5feb1aa885c15b83e725f87386351de99d3abb49656219d50 \ No newline at end of file +9048a766ff7dfa0cd91ea74092e462f4501cb3f719033ccb55700bf5e4dfd0d3 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c2bbc8ee3a..c2fc4e5a6a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5542,16 +5542,6 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), ** then the conversion is performed. Otherwise no conversion occurs. ** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ ** -** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], -** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current encoding -** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) -** returns something other than SQLITE_TEXT, then the return value from -** sqlite3_value_encoding(X) is meaningless. ^Calls to -** sqlite3_value_text(X), sqlite3_value_text16(X), sqlite3_value_text16be(X), -** sqlite3_value_text16le(X), sqlite3_value_bytes(X), or -** sqlite3_value_bytes16(X) might change the encoding of the value X and -** thus change the return from subsequent calls to sqlite3_value_encoding(X). -** ** ^Within the [xUpdate] method of a [virtual table], the ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation @@ -5616,6 +5606,27 @@ int sqlite3_value_type(sqlite3_value*); int sqlite3_value_numeric_type(sqlite3_value*); int sqlite3_value_nochange(sqlite3_value*); int sqlite3_value_frombind(sqlite3_value*); + +/* +** CAPI3REF: Report the internal text encoding state of an sqlite3_value object +** METHOD: sqlite3_value +** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current text encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or +** [sqlite3_value_bytes16(X)] might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** +** This routine is intended for used by applications that test and validate +** the SQLite implementation. This routine is inquiring about the opaque +** internal state of an [sqlite3_value] object. Ordinary applications should +** not need to know what the internal state of an sqlite3_value object is and +** hence should not need to use this interface. +*/ int sqlite3_value_encoding(sqlite3_value*); /* From 3c1572ddb43ba7f4cbae57ee4b019a2cd45ee015 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 17 Nov 2022 14:40:33 +0000 Subject: [PATCH 009/282] Use the log10() and log2() functions from the standard C library to implement the equivalent SQL functions, in the hope that this will prevent reported precision problems. See [forum:/forumpost/cfceb1230bdcfd84|forum post cfceb1230bdcfd84] and the surrounding thread. FossilOrigin-Name: 7c572d02e60a83b36543ba4d9d45f61e9fc111b61fee085410c2d87558c732d6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 8 +++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 65bffb3672..2432894ab6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Split\sout\sthe\sdocumentation\sfor\ssqlite3_value_encoding()\sinto\sits\sown\npage\sand\smake\sit\sclear\sthat\sthis\sinterface\sis\sintended\sfor\stesting\sand\ndebugging\sonly.\n[forum:/forumpost/c9f445453da950ad|Forum\sthread\sc9f445453da950ad].\nComment\schanges\sonly\s-\sno\schanges\sto\scode. -D 2022-11-17T13:58:25.001 +C Use\sthe\slog10()\sand\slog2()\sfunctions\sfrom\sthe\sstandard\sC\slibrary\sto\simplement\nthe\sequivalent\sSQL\sfunctions,\sin\sthe\shope\sthat\sthis\swill\sprevent\sreported\nprecision\sproblems.\nSee\s[forum:/forumpost/cfceb1230bdcfd84|forum\spost\scfceb1230bdcfd84]\sand\sthe\nsurrounding\sthread. +D 2022-11-17T14:40:33.537 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -590,7 +590,7 @@ F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e F src/expr.c 847f87d9df3ede2b2b0a8db088af0b9c1923b21009f8ea1b9b7b28cb0a383170 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 -F src/func.c d25f3c667d59dbac195e65f3539fdbbd8a36c142ce7bcdb45d1baf07446ad13a +F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d F src/global.c e06ff8e0acd85aec13563c9ecb44fbbf38232ccf73594998fd880b92d619594b F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 @@ -2055,8 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a119a9e2ade4eac5feb1aa885c15b83e725f87386351de99d3abb49656219d50 -R b081909c6ecb1750561be5091dea8e63 +P 9048a766ff7dfa0cd91ea74092e462f4501cb3f719033ccb55700bf5e4dfd0d3 +R a034e1e03397012452b518f4075f2764 U drh -Z c8051a67682b7cc2de89c2c47da25213 +Z 49084f227abbaddd45e0c2d870d50ee7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4a06fb59e5..d717acf28e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9048a766ff7dfa0cd91ea74092e462f4501cb3f719033ccb55700bf5e4dfd0d3 \ No newline at end of file +7c572d02e60a83b36543ba4d9d45f61e9fc111b61fee085410c2d87558c732d6 \ No newline at end of file diff --git a/src/func.c b/src/func.c index b42b2588af..3c5a852307 100644 --- a/src/func.c +++ b/src/func.c @@ -2106,17 +2106,15 @@ static void logFunc( } ans = log(x)/b; }else{ - ans = log(x); switch( SQLITE_PTR_TO_INT(sqlite3_user_data(context)) ){ case 1: - /* Convert from natural logarithm to log base 10 */ - ans /= M_LN10; + ans = log10(x); break; case 2: - /* Convert from natural logarithm to log base 2 */ - ans /= M_LN2; + ans = log2(x); break; default: + ans = log(x); break; } } From d8e48fffdff1ebfbf5e0b2ded9e12ce00b1ba427 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 17 Nov 2022 19:24:39 +0000 Subject: [PATCH 010/282] Fix corner cases in UTF8 handling in the REGEXP extension. [forum:/forumpost/3ffe058b04|Forum post 3ffe058b04]. FossilOrigin-Name: abb18f61c5cec0f524acc41453b4c06b61c5af51ff46417588837fc0c3967288 --- ext/misc/regexp.c | 8 ++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- test/regexp1.test | 27 +++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index d0c8ee5cfe..086ef564cb 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -185,7 +185,7 @@ static unsigned re_next_char(ReInput *p){ c = (c&0x0f)<<12 | ((p->z[p->i]&0x3f)<<6) | (p->z[p->i+1]&0x3f); p->i += 2; if( c<=0x7ff || (c>=0xd800 && c<=0xdfff) ) c = 0xfffd; - }else if( (c&0xf8)==0xf0 && p->i+3mx && (p->z[p->i]&0xc0)==0x80 + }else if( (c&0xf8)==0xf0 && p->i+2mx && (p->z[p->i]&0xc0)==0x80 && (p->z[p->i+1]&0xc0)==0x80 && (p->z[p->i+2]&0xc0)==0x80 ){ c = (c&0x07)<<18 | ((p->z[p->i]&0x3f)<<12) | ((p->z[p->i+1]&0x3f)<<6) | (p->z[p->i+2]&0x3f); @@ -712,15 +712,15 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ ** one or more matching characters, enter those matching characters into ** zInit[]. The re_match() routine can then search ahead in the input ** string looking for the initial match without having to run the whole - ** regex engine over the string. Do not worry able trying to match + ** regex engine over the string. Do not worry about trying to match ** unicode characters beyond plane 0 - those are very rare and this is ** just an optimization. */ if( pRe->aOp[0]==RE_OP_ANYSTAR && !noCase ){ for(j=0, i=1; j<(int)sizeof(pRe->zInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){ unsigned x = pRe->aArg[i]; - if( x<=127 ){ + if( x<=0x7f ){ pRe->zInit[j++] = (unsigned char)x; - }else if( x<=0xfff ){ + }else if( x<=0x7ff ){ pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); pRe->zInit[j++] = 0x80 | (x&0x3f); }else if( x<=0xffff ){ diff --git a/manifest b/manifest index 2432894ab6..00bc7b6ef8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\slog10()\sand\slog2()\sfunctions\sfrom\sthe\sstandard\sC\slibrary\sto\simplement\nthe\sequivalent\sSQL\sfunctions,\sin\sthe\shope\sthat\sthis\swill\sprevent\sreported\nprecision\sproblems.\nSee\s[forum:/forumpost/cfceb1230bdcfd84|forum\spost\scfceb1230bdcfd84]\sand\sthe\nsurrounding\sthread. -D 2022-11-17T14:40:33.537 +C Fix\scorner\scases\sin\sUTF8\shandling\sin\sthe\sREGEXP\sextension.\n[forum:/forumpost/3ffe058b04|Forum\spost\s3ffe058b04]. +D 2022-11-17T19:24:39.375 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -316,7 +316,7 @@ F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196 F ext/misc/qpvtab.c 09738419e25f603a35c0ac8bd0a04daab794f48d08a9bc07a6085b9057b99009 -F ext/misc/regexp.c 5abed0ace2d9340b42b9ab1dbe64db9c276e4e8eba38a903232b6253e05ccdaf +F ext/misc/regexp.c 064838f7b31e90d312cce089cfafbb992034e75d359009c48886ca06c0a794b2 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 @@ -1411,7 +1411,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/recover.test fd5199f928757cb308661b5fdca1abc19398a798ff7f24b57c3071e9f8e0471e -F test/regexp1.test 83c631617357150f8054ca1d1fed40a552b0d0f8eb7a7f090c3be02cee9f9913 +F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/releasetest_data.tcl 0db8aee0c348090fd06da47020ab4ed8ec692e0723427b2f3947d4dfb806f3b0 @@ -2055,8 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9048a766ff7dfa0cd91ea74092e462f4501cb3f719033ccb55700bf5e4dfd0d3 -R a034e1e03397012452b518f4075f2764 +P 7c572d02e60a83b36543ba4d9d45f61e9fc111b61fee085410c2d87558c732d6 +R f93830c4b318c6359968570a2d6be5a6 U drh -Z 49084f227abbaddd45e0c2d870d50ee7 +Z 502cd80eb5e32a4170e530e0ee906c79 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d717acf28e..13c0efb61f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c572d02e60a83b36543ba4d9d45f61e9fc111b61fee085410c2d87558c732d6 \ No newline at end of file +abb18f61c5cec0f524acc41453b4c06b61c5af51ff46417588837fc0c3967288 \ No newline at end of file diff --git a/test/regexp1.test b/test/regexp1.test index 102c1280c0..0401b13d72 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -303,6 +303,33 @@ do_execsql_test regexp1-6.7 {SELECT 'xabc' REGEXP '(^abc|def)';} {0} do_execsql_test regexp1-6.8 {SELECT 'def' REGEXP '(^abc|def)';} {1} do_execsql_test regexp1-6.9 {SELECT 'xdef' REGEXP '(^abc|def)';} {1} +# 2022-11-17 +# https://sqlite.org/forum/forumpost/3ffe058b04 +# +do_execsql_test regexp1-7.1 { + SELECT char(0x61,0x7ff,0x62) REGEXP char(0x7ff); +} 1 +do_execsql_test regexp1-7.2 { + SELECT char(0x61,0x800,0x62) REGEXP char(0x800); +} 1 +do_execsql_test regexp1-7.3 { + SELECT char(0x61,0xabc,0x62) REGEXP char(0xabc); +} 1 +do_execsql_test regexp1-7.4 { + SELECT char(0x61,0xfff,0x62) REGEXP char(0xfff); +} 1 +do_execsql_test regexp1-7.5 { + SELECT char(0x61,0x1000,0x62) REGEXP char(0x1000); +} 1 +do_execsql_test regexp1-7.10 { + SELECT char(0x61,0xffff,0x62) REGEXP char(0xffff); +} 1 +do_execsql_test regexp1-7.11 { + SELECT char(0x61,0x10000,0x62) REGEXP char(0x10000); +} 1 +do_execsql_test regexp1-7.12 { + SELECT char(0x61,0x10ffff,0x62) REGEXP char(0x10ffff); +} 1 finish_test From ee026c54794b3196326dc5b1db1c9a714bc84e18 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 18 Nov 2022 02:29:59 +0000 Subject: [PATCH 011/282] Replace use of cpp with the fit-to-purpose c-pp to avoid cpp's C-centric/JS-unfriendly quirks. FossilOrigin-Name: 49d70f071e918d5d095c807575bb7ce2b287a123261e789e938521b3b409429a --- ext/wasm/GNUmakefile | 65 +- ext/wasm/api/pre-js.js | 6 +- ext/wasm/api/sqlite3-api-opfs.js | 6 +- ext/wasm/c-pp.c | 1525 ++++++++++++++++++++++++++++++ manifest | 17 +- manifest.uuid | 2 +- 6 files changed, 1573 insertions(+), 48 deletions(-) create mode 100644 ext/wasm/c-pp.c diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 5fd0f073c1..bd30d306ea 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -235,8 +235,8 @@ $(foreach X,$(SOAP.js) $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js),\ all: $(sqlite3-api.ext.jses) sqlite3-api.js := $(dir.tmp)/sqlite3-api.js -sqlite3-api.pre-cpp.js := $(dir.tmp)/sqlite3-api.pre-cpp.js -$(sqlite3-api.pre-cpp.js): $(sqlite3-api.jses) $(MAKEFILE) +sqlite3-api.c-pp.js := $(dir.tmp)/sqlite3-api.c-pp.js +$(sqlite3-api.c-pp.js): $(sqlite3-api.jses) $(MAKEFILE) @echo "Making $@..." @for i in $(sqlite3-api.jses); do \ echo "/* BEGIN FILE: $$i */"; \ @@ -257,7 +257,6 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) ######################################################################## # --post-js and --pre-js are emcc flags we use to append/prepend JS to # the generated emscripten module file. -pre-js.pre-cpp.js := $(dir.api)/pre-js.pre-cpp.js pre-js.js := $(dir.tmp)/pre-js.js post-js.js := $(dir.tmp)/post-js.js post-jses := \ @@ -291,47 +290,47 @@ $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) } > $@ ######################################################################## -# Transform $(1) to $(2) via cpp -E -CC $(1) ... +# Transform $(1) to $(2) via ./c-pp -f $(1) ... # -# DO NOT use gcc as the preprocessor because it will emit its own -# license header to our output because that's a comment in its -# stdc-predef.h, which we cannot tell it to _not_ include. The only -# workaround to that is to allow gcc -E to strip all comments. The -# wasm build uses clang behind emcc, anyway, so we already have a -# clang dependency. However, the clang cpp refuses to read empty -# strings in the form '', so we have to be sure to use "" in JS code -# for those. +# Historical notes: # -# It's tempting to build a custom mini-cpp-like binary for this -# purpose to avoid these dependencies and quirks. Maybe we could use -# lemon to do the heavy lifting for that, noting that we'd still need -# to tokenize it by hand (but only lines which start with "#" or -# backslash-continued lines, and we could treat all other lines as -# opaque content). +# - We first attempted to use gcc and/or clang to preprocess JS files +# in the same way we would normally do C files, but C-specific quirks +# of each makes that untennable. # -# In this build we may have #ifdef's (and the like) in arbitrary input -# JS files and we need to preprocess those before Emscripten gets -# ahold of them. We cannot simply preprocess the resulting -# Emscripten-generated sqlite3.js because (A) Emscripten may choke on -# C preprocessor directives in the input and (B) Emscripten's output -# may contain things which cpp cannot read (like single-quoted empty -# strings: ''). -bin.cpp ?= clang -ifneq (,$(filter ems,$(MAKECMDGOALS))) +# - We implemented c-pp.c (the C-Minus Pre-processor) as a custom +# generic/file-format-agnostic preprocessor to enable us to pack +# code for different target builds into the same JS files. Most +# notably, some ES6 module (a.k.a. ESM) features cannot legally be +# referenced at all in non-ESM code, e.g. the "import" and "export" +# keywords. This preprocessing step permits us to swap out sections +# of code where necessary for ESM and non-ESM (a.k.a. vanilla JS) +# require different implementations. The alternative to such +# preprocessing, would be to have separate source files for ES6 +# builds, which would have a higher maintenance burden than c-pp.c +# seems likely to. +# +# c-pp.c was written specifically for the sqlite project's JavaScript +# builds but is maintained as a standalone project: +# https://fossil.wanderinghorse.net/r/c-pp +bin.c-pp := ./c-pp +$(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) + $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) +ifneq (,$(filter esm,$(MAKECMDGOALS))) js.cpp.defines ?= -DSQLITE_JS_ESM -ems: $(filter-out ems,$(MAKECMDGOALS)) +esm: $(filter-out esm,$(MAKECMDGOALS)) else js.cpp.defines ?= endif -define CPP_JS +define C-PP_JS # $1 = X.js. $2 = output file to generate by filtering $(1) through # $(bin.cpp) -E -CC. -$(2): $(1) $$(MAKEFILE) - $$(bin.cpp) -E -CC -undef $(js.cpp.defines) -x c $(1) | sed -e '/^#/d' > $$@ +$(2): $(1) $$(MAKEFILE) $$(bin.c-pp) + $$(bin.c-pp) $(js.cpp.defines) -f $(1) -o $$@ CLEAN_FILES += $(2) endef -$(eval $(call CPP_JS,$(dir.tmp)/sqlite3-api.pre-cpp.js,$(dir.tmp)/sqlite3-api.js)) -$(eval $(call CPP_JS,$(dir.api)/pre-js.js,$(dir.tmp)/pre-js.js)) +$(eval $(call C-PP_JS,$(dir.tmp)/sqlite3-api.c-pp.js,$(dir.tmp)/sqlite3-api.js)) +$(eval $(call C-PP_JS,$(dir.api)/pre-js.js,$(dir.tmp)/pre-js.js)) # /end CPP-of-JS bits ######################################################################## diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.js index f959c33072..c6d0683ff4 100644 --- a/ext/wasm/api/pre-js.js +++ b/ext/wasm/api/pre-js.js @@ -29,9 +29,9 @@ sqlite3InitModuleState.debugModule('self.location =',self.location); 4) If none of the above apply, (prefix+path) is returned. */ Module['locateFile'] = function(path, prefix) { -#ifdef SQLITE_JS_ESM +//#if SQLITE_JS_ESM return new URL(path, import.meta.url).href; -#else +//#else 'use strict'; let theFile; const up = this.urlParams; @@ -51,7 +51,7 @@ Module['locateFile'] = function(path, prefix) { "result =", theFile ); return theFile; -#endif /* SQLITE_JS_EMS */ +//#endif /* SQLITE_JS_EMS */ }.bind(sqlite3InitModuleState); /** diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 18582400eb..e35eed64d1 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -167,11 +167,11 @@ const installOpfsVfs = function callee(options){ return promiseReject_(err); }; const W = -#ifdef SQLITE_JS_ESM +//#if SQLITE_JS_ESM new Worker(new URL(options.proxyUri, import.meta.url)); -#else +//#else new Worker(options.proxyUri); -#endif +//#endif W._originalOnError = W.onerror /* will be restored later */; W.onerror = function(err){ // The error object doesn't contain any useful info when the diff --git a/ext/wasm/c-pp.c b/ext/wasm/c-pp.c new file mode 100644 index 0000000000..881c009acb --- /dev/null +++ b/ext/wasm/c-pp.c @@ -0,0 +1,1525 @@ +/* +** 2022-11-12: +** +** In place of a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** +** The C-minus Preprocessor: a truly minimal C-like preprocessor. +** Why? Because C preprocessors _can_ process non-C code but generally make +** quite a mess of it. The purpose of this application is an extremely +** minimal preprocessor with only the most basic functionality of a C +** preprocessor, namely: +** +** - Limited `#if`, where its one argument is a macro name which +** resolves to true if it's defined, false if it's not. Likewise, +** `#ifnot` is the inverse. Includes `#else` and `#elif` and +** `#elifnot`. Such chains are terminated with `#endif`. +** +** - `#define` accepts one or more arguments, the names of +** macros. Each one is implicitly true. +** +** - `#undef` undefine one or more macros. +** +** - `#error` treats the rest of the line as a fatal error message. +** +** - `#include` treats its argument as a filename token (NOT quoted, +** though support for quoting may be added later). Some effort is +** made to prevent recursive inclusion, but that support is both +** somewhat fragile and possibly completely unnecessary. +** +** - `#pragma` is in place for adding "meta-commands", but it does not +** yet have any concrete list of documented commands. +** +* - `#stderr` outputs its file name, line number, and the remaininder +** of that line to stderr. +** +** - `#//` acts as a single-line comment, noting that there must be as +** space after the `//` part because `//` is (despite appearances) +** parsed like a keyword. +** +** Note that "#" above is symbolic. The keyword delimiter is +** configurable and defaults to "##". Define CMPP_DEFAULT_DELIM to a +** string when compiling to define the default at build-time. +** +** This preprocessor does no expansion of content except within the +** bounds of its `#keywords`. +** +** Design note: this code makes use of sqlite3. Though not _strictly_ +** needed in order to implement it, this tool was specifically created +** for potential use with the sqlite3 project's own JavaScript code, +** so there's no reason not to make use of it to do some of the heavy +** lifting. It does not require any cutting-edge sqlite3 features and +** should be usable with any version which supports `WITHOUT ROWID`. +** +** Author(s): +** +** - Stephan Beal +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "sqlite3.h" + +#if defined(_WIN32) || defined(WIN32) +# include +# include +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +#else +# include +#endif + +#ifndef CMPP_DEFAULT_DELIM +#define CMPP_DEFAULT_DELIM "##" +#endif + +#if 1 +# define CMPP_NORETURN __attribute__((noreturn)) +#else +# define CMPP_NORETURN +#endif + +/* Fatally exits the app with the given printf-style message. */ +static CMPP_NORETURN void fatalv(char const *zFmt, va_list); +static CMPP_NORETURN void fatal(char const *zFmt, ...); + +/** Proxy for free(), for symmetry with cmpp_realloc(). */ +static void cmpp_free(void *p); +/** A realloc() proxy which dies fatally on allocation error. */ +static void * cmpp_realloc(void * p, unsigned n); +#if 0 +/** A malloc() proxy which dies fatally on allocation error. */ +static void * cmpp_malloc(unsigned n); +#endif + +/* +** If p is stdin or stderr then this is a no-op, else it is a +** proxy for fclose(). This is a no-op if p is NULL. +*/ +static void FILE_close(FILE *p); +/* +** Works like fopen() but accepts the special name "-" to mean either +** stdin (if zMode indicates a real-only mode) or stdout. Fails +** fatally on error. +*/ +static FILE * FILE_open(char const *zName, const char * zMode); +/* +** Reads the entire contents of the given file, allocating it in a +** buffer which gets assigned to `*pOut`. `*nOut` gets assigned the +** length of the output buffer. Fails fatally on error. +*/ +static void FILE_slurp(FILE *pFile, unsigned char **pOut, + unsigned * nOut); + +/* +** Intended to be passed an sqlite3 result code. If it's non-0 +** then it emits a fatal error message which contains both the +** given string and the sqlite3_errmsg() from the application's +** database instance. +*/ +static void db_affirm_rc(int rc, const char * zMsg); + +/* +** Proxy for sqlite3_str_finish() which fails fatally if that +** routine returns NULL. +*/ +static char * db_str_finish(sqlite3_str *s, int * n); +/* +** Proxy for sqlite3_str_new() which fails fatally if that +** routine returns NULL. +*/ +static sqlite3_str * db_str_new(void); + +/* Proxy for sqlite3_finalize(). */ +static void db_finalize(sqlite3_stmt *pStmt); +/* +** Proxy for sqlite3_step() which fails fatally if the result +** is anything other than SQLITE_ROW or SQLITE_DONE. +*/ +static int db_step(sqlite3_stmt *pStmt); +/* +** Proxy for sqlite3_bind_int() which fails fatally on error. +*/ +static void db_bind_int(sqlite3_stmt *pStmt, int col, int val); +#if 0 +/* +** Proxy for sqlite3_bind_null() which fails fatally on error. +*/ +static void db_bind_null(sqlite3_stmt *pStmt, int col); +#endif +/* +** Proxy for sqlite3_bind_text() which fails fatally on error. +*/ +static void db_bind_text(sqlite3_stmt *pStmt, int col, const char * zStr); +/* +** Proxy for sqlite3_bind_text() which fails fatally on error. +*/ +static void db_bind_textn(sqlite3_stmt *pStmt, int col, const char * zStr, int len); +#if 0 +/* +** Proxy for sqlite3_bind_text() which fails fatally on error. It uses +** sqlite3_str_vappendf() so supports all of its formatting options. +*/ +static void db_bind_textv(sqlite3_stmt *pStmt, int col, const char * zFmt, ...); +#endif +/* +** Proxy for sqlite3_free(), to be passed any memory which is allocated +** by sqlite3_malloc(). +*/ +static void db_free(void *m); +/* +** Adds the given `#define` macro name to the list of macros, ignoring +** any duplicates. Fails fatally on error. +*/ +static void db_define_add(const char * zKey); +/* +** Returns true if the given key is already in the `#define` list, +** else false. Fails fatally on db error. +*/ +static int db_define_has(const char * zName); +/* +** Removes the given `#define` macro name from the list of +** macros. Fails fatally on error. +*/ +static void db_define_rm(const char * zKey); +/* +** Adds the given filename to the list of being-`#include`d files, +** using the given source file name and line number of error reporting +** purposes. If recursion is later detected. +*/ +static void db_including_add(const char * zKey, const char * zSrc, int srcLine); +/* +** Adds the given dir to the list of includes. They are checked in the +** order they are added. +*/ +static void db_include_dir_add(const char * zKey); +/* +** Returns a resolved path of PREFIX+'/'+zKey, where PREFIX is one of +** the `#include` dirs (db_include_dir_add()). If no file match is +** found, NULL is returned. Memory must eventually be passed to +** db_free() to free it. +*/ +static char * db_include_search(const char * zKey); +/* +** Removes the given key from the `#include` list. +*/ +static void db_include_rm(const char * zKey); +/* +** A proxy for sqlite3_prepare() which fails fatally on error. +*/ +static void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...); + +/* +** Opens the given file and processes its contents as c-pp, sending +** all output to the global c-pp output channel. Fails fatally on +** error. +*/ +static void cmpp_process_file(const char * zName); + +/* +** Returns the number newline characters between the given starting +** point and inclusive ending point. Results are undefined if zFrom is +** greater than zTo. +*/ +static unsigned count_lines(unsigned char const * zFrom, + unsigned char const *zTo); + +/* +** Wrapper around a FILE handle. +*/ +struct FileWrapper { + /* File's name. */ + char const *zName; + /* FILE handle. */ + FILE * pFile; + /* Where FileWrapper_slurp() stores the file's contents. */ + unsigned char * zContent; + /* Size of this->zContent, as set by FileWrapper_slurp(). */ + unsigned nContent; +}; +typedef struct FileWrapper FileWrapper; +#define FileWrapper_empty_m {0,0,0,0} +static const FileWrapper FileWrapper_empty = FileWrapper_empty_m; + +/* Proxy for FILE_close(). */ +static void FileWrapper_close(FileWrapper * p); +/* Proxy for FILE_open(). */ +static void FileWrapper_open(FileWrapper * p, const char * zName, const char *zMode); +/* Proxy for FILE_slurp(). */ +static void FileWrapper_slurp(FileWrapper * p); + +/* +** Outputs a printf()-formatted message to stderr. +*/ +static void g_stderr(char const *zFmt, ...); +/* +** Outputs a printf()-formatted message to stderr. +*/ +static void g_stderrv(char const *zFmt, va_list); +#define g_debug(lvl,pfexpr) \ + if(lvl<=g.doDebug) g_stderr("%s @ %s:%d: ",g.zArgv0,__FILE__,__LINE__); \ + if(lvl<=g.doDebug) g_stderr pfexpr + +void fatalv(char const *zFmt, va_list va){ + if(zFmt && *zFmt){ + vfprintf(stderr, zFmt, va); + } + fputc('\n', stderr); + exit(1); +} + +void fatal(char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + fatalv(zFmt, va); + va_end(va); +} + +void cmpp_free(void *p){ + free(p); +} + +void * cmpp_realloc(void * p, unsigned n){ + void * const rc = realloc(p, n); + if(!rc) fatal("realloc(P,%u) failed", n); + return rc; +} + +#if 0 +void * cmpp_malloc(unsigned n){ + void * const rc = malloc(n); + if(!rc) fatal("malloc(%u) failed", n); + return rc; +} +#endif + +FILE * FILE_open(char const *zName, const char * zMode){ + FILE * p; + if('-'==zName[0] && 0==zName[1]){ + p = strstr(zMode,"w") ? stdout : stdin; + }else{ + p = fopen(zName, zMode); + if(!p) fatal("Cannot open file [%s] with mode [%s]", zName, zMode); + } + return p; +} + +void FILE_close(FILE *p){ + if(p && p!=stdout && p!=stderr){ + fclose(p); + } +} + +void FILE_slurp(FILE *pFile, unsigned char **pOut, + unsigned * nOut){ + unsigned char zBuf[1024 * 8]; + unsigned char * pDest = 0; + unsigned nAlloc = 0; + unsigned nOff = 0; + /* Note that this needs to be able to work on non-seekable streams, + ** thus we read in chunks instead of doing a single alloc and + ** filling it in one go. */ + while( !feof(pFile) ){ + size_t const n = fread(zBuf, 1, sizeof(zBuf), pFile); + if(n>0){ + if(nAlloc < nOff + n + 1){ + nAlloc = nOff + n + 1; + pDest = cmpp_realloc(pDest, nAlloc); + } + memcpy(pDest + nOff, zBuf, n); + nOff += n; + } + } + if(pDest) pDest[nOff] = 0; + *pOut = pDest; + *nOut = nOff; +} + +void FileWrapper_close(FileWrapper * p){ + if(p->pFile) FILE_close(p->pFile); + if(p->zContent) cmpp_free(p->zContent); + *p = FileWrapper_empty; +} + +void FileWrapper_open(FileWrapper * p, const char * zName, + const char * zMode){ + FileWrapper_close(p); + p->pFile = FILE_open(zName, zMode); + p->zName = zName; +} + +void FileWrapper_slurp(FileWrapper * p){ + assert(!p->zContent); + assert(p->pFile); + FILE_slurp(p->pFile, &p->zContent, &p->nContent); +} + +unsigned count_lines(unsigned char const * zFrom, unsigned char const *zTo){ + unsigned ln = 0; + unsigned char const *zPos = zFrom; + assert(zFrom && zTo); + assert(zFrom <= zTo); + for(; zPos < zTo; ++zPos){ + switch(*zPos){ + case (unsigned)'\n': ++ln; break; + default: break; + } + } + return ln; +} + +enum CmppParseState { +TS_Start = 1, +TS_If, +TS_IfPassed, +TS_Else, +TS_Error +}; +typedef enum CmppParseState CmppParseState; +enum CmppTokenType { +TT_Invalid = 0, +TT_Comment, +TT_Define, +TT_Elif, +TT_ElifNot, +TT_Else, +TT_EndIf, +TT_Error, +TT_If, +TT_IfNot, +TT_Include, +TT_Line, +TT_Pragma, +TT_Stderr, +TT_Undef +}; +typedef enum CmppTokenType CmppTokenType; + +struct CmppToken { + CmppTokenType ttype; + /* Line number of this token in the source file. */ + unsigned lineNo; + /* Start of the token. */ + unsigned char const * zBegin; + /* One-past-the-end byte of the token. */ + unsigned char const * zEnd; +}; +typedef struct CmppToken CmppToken; +#define CmppToken_empty_m {TT_Invalid,0,0,0} +static const CmppToken CmppToken_empty = CmppToken_empty_m; + +/* +** CmppLevel represents one "level" of tokenization, starting at the +** top of the main input, incrementing once for each level of `#if`, +** and decrementing for each `#endif`. +*/ +typedef struct CmppLevel CmppLevel; +struct CmppLevel { + unsigned short flags; + /* + ** Used for controlling which parts of an if/elif/...endif chain + ** should get output. + */ + unsigned short skipLevel; + /* The token which started this level (an 'if' or 'ifnot'). */ + CmppToken token; + CmppParseState pstate; +}; +#define CmppLevel_empty_m {0U,0U,CmppToken_empty_m,TS_Start} +static const CmppLevel CmppLevel_empty = CmppLevel_empty_m; +enum CmppLevel_Flags { +/* Max depth of nested `#if` constructs in a single tokenizer. */ +CmppLevel_Max = 10, +/* Max number of keyword arguments. */ +CmppArgs_Max = 10, +/* Flag indicating that output for a CmpLevel should be elided. */ +CmppLevel_F_ELIDE = 0x01, +/* +** Mask of CmppLevel::flags which are inherited when CmppLevel_push() +** is used. +*/ +CmppLevel_F_INHERIT_MASK = 0x01 +}; + +typedef struct CmppTokenizer CmppTokenizer; +typedef struct CmppKeyword CmppKeyword; +typedef void (*cmpp_keyword_f)(CmppKeyword const * pKw, CmppTokenizer * t); +struct CmppKeyword { + const char *zName; + unsigned nName; + int bTokenize; + CmppTokenType ttype; + cmpp_keyword_f xCall; +}; + +static CmppKeyword const * CmppKeyword_search(const char *zName); +static void cmpp_process_keyword(CmppTokenizer * const t); + +/* +** Tokenizer for c-pp input files. +*/ +struct CmppTokenizer { + const char * zName; /* Input (file) name for error reporting */ + unsigned const char * zBegin; /* start of input */ + unsigned const char * zEnd; /* one-after-the-end of input */ + unsigned const char * zAnchor; /* start of input or end point of + previous token */ + unsigned const char * zPos; /* current position */ + unsigned int lineNo; /* line # of current pos */ + CmppParseState pstate; + CmppToken token; /* current token result */ + struct { + unsigned ndx; + CmppLevel stack[CmppLevel_Max]; + } level; + /* Args for use in cmpp_keyword_f() impls. */ + struct { + CmppKeyword const * pKw; + int argc; + const unsigned char * argv[CmppArgs_Max]; + unsigned char lineBuf[1024]; + } args; +}; +#define CT_level(t) (t)->level.stack[(t)->level.ndx] +#define CT_pstate(t) CT_level(t).pstate +#define CT_skipLevel(t) CT_level(t).skipLevel +#define CLvl_skip(lvl) ((lvl)->skipLevel || ((lvl)->flags & CmppLevel_F_ELIDE)) +#define CT_skip(t) CLvl_skip(&CT_level(t)) +#define CmppTokenizer_empty_m { \ + 0,0,0,0,0,1U/*lineNo*/, \ + TS_Start, \ + CmppToken_empty_m, \ + {/*level*/0U,{CmppLevel_empty_m}}, \ + {/*args*/0,0,{0},{0}} \ + } +static const CmppTokenizer CmppTokenizer_empty = CmppTokenizer_empty_m; + +static void cmpp_t_out(CmppTokenizer * t, void const *z, unsigned int n); +/*static void cmpp_t_outf(CmppTokenizer * t, char const *zFmt, ...);*/ + +/* +** Pushes a new level into the given tokenizer. Fails fatally if +** it's too deep. +*/ +static void CmppLevel_push(CmppTokenizer * const t); +/* +** Pops a level from the tokenizer. Fails fatally if the top +** level is popped. +*/ +static void CmppLevel_pop(CmppTokenizer * const t); +/* +** Returns the current level object. +*/ +static CmppLevel * CmppLevel_get(CmppTokenizer * const t); + +/* +** Global app state singleton. */ +static struct Global { + /* main()'s argv[0]. */ + const char * zArgv0; + /* + ** Bytes of the keyword delimiter/prefix. Owned + ** elsewhere. + */ + const char * zDelim; + /* Byte length of this->zDelim. */ + unsigned short nDelim; + /* If true, enables certain debugging output. */ + int doDebug; + /* App's db instance. */ + sqlite3 * db; + /* Output channel. */ + FileWrapper out; + struct { + sqlite3_stmt * defIns; + sqlite3_stmt * defDel; + sqlite3_stmt * defHas; + sqlite3_stmt * inclIns; + sqlite3_stmt * inclDel; + sqlite3_stmt * inclHas; + sqlite3_stmt * inclPathAdd; + sqlite3_stmt * inclSearch; + } stmt; +} g = { +"?", +CMPP_DEFAULT_DELIM/*zDelim*/, +(unsigned short) sizeof(CMPP_DEFAULT_DELIM)-1/*nDelim*/, +0/*doDebug*/, +0/*db*/, +FileWrapper_empty_m/*out*/, +{/*stmt*/ + 0/*defIns*/, 0/*defDel*/, 0/*defHas*/, + 0/*inclIns*/, 0/*inclDel*/, 0/*inclHas*/, + 0/*inclPathAdd*/ +} +}; + + +#if 0 +/* +** Outputs a printf()-formatted message to c-pp's global output +** channel. +*/ +static void g_outf(char const *zFmt, ...); +void g_outf(char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + vfprintf(g.out.pFile, zFmt, va); + va_end(va); +} +#endif + +#if 0 +/* Outputs n bytes from z to c-pp's global output channel. */ +static void g_out(void const *z, unsigned int n); +void g_out(void const *z, unsigned int n){ + if(1!=fwrite(z, n, 1, g.out.pFile)){ + int const err = errno; + fatal("fwrite() output failed with errno #%d", err); + } +} +#endif + +void g_stderrv(char const *zFmt, va_list va){ + vfprintf(stderr, zFmt, va); +} + +void g_stderr(char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + g_stderrv(zFmt, va); + va_end(va); +} + +#if 0 +void cmpp_t_outf(CmppTokenizer * t, char const *zFmt, ...){ + if(!CT_skip(t)){ + va_list va; + va_start(va, zFmt); + vfprintf(g.out.pFile, zFmt, va); + va_end(va); + } +} +#endif + +void cmpp_t_out(CmppTokenizer * t, void const *z, unsigned int n){ + if(!CT_skip(t)){ + if(1!=fwrite(z, n, 1, g.out.pFile)){ + int const err = errno; + fatal("fwrite() output failed with errno #%d", err); + } + } +} + +void CmppLevel_push(CmppTokenizer * const t){ + CmppLevel * pPrev; + CmppLevel * p; + if(t->level.ndx+1 == (unsigned)CmppLevel_Max){ + fatal("%sif nesting level is too deep. Max=%d\n", + g.zDelim, CmppLevel_Max); + } + pPrev = &CT_level(t); + p = &t->level.stack[++t->level.ndx]; + *p = CmppLevel_empty; + p->token = t->token; + p->flags = (CmppLevel_F_INHERIT_MASK & pPrev->flags); + if(CLvl_skip(pPrev)) p->flags |= CmppLevel_F_ELIDE; +} + +void CmppLevel_pop(CmppTokenizer * const t){ + if(!t->level.ndx){ + fatal("Internal error: CmppLevel_pop() at the top of the stack"); + } + t->level.stack[t->level.ndx--] = CmppLevel_empty; +} + +CmppLevel * CmppLevel_get(CmppTokenizer * const t){ + return &t->level.stack[t->level.ndx]; +} + + +void db_affirm_rc(int rc, const char * zMsg){ + if(rc){ + fatal("Db error #%d %s: %s", rc, zMsg, sqlite3_errmsg(g.db)); + } +} + +void db_finalize(sqlite3_stmt *pStmt){ + sqlite3_finalize(pStmt); +} + +int db_step(sqlite3_stmt *pStmt){ + int const rc = sqlite3_step(pStmt); + if(SQLITE_ROW!=rc && SQLITE_DONE!=rc){ + db_affirm_rc(rc, "from db_step()"); + } + return rc; +} + +static sqlite3_str * db_str_new(void){ + sqlite3_str * rc = sqlite3_str_new(g.db); + if(!rc) fatal("Alloc failed for sqlite3_str_new()"); + return rc; +} + +static char * db_str_finish(sqlite3_str *s, int * n){ + int const rc = sqlite3_str_errcode(s); + if(rc) fatal("Error #%d from sqlite3_str_errcode()", rc); + if(n) *n = sqlite3_str_length(s); + char * z = sqlite3_str_finish(s); + if(!z) fatal("Alloc failed for sqlite3_str_new()"); + return z; +} + +void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...){ + int rc; + sqlite3_str * str = db_str_new(); + char * z = 0; + int n = 0; + va_list va; + if(!str) fatal("sqlite3_str_new() failed"); + va_start(va, zSql); + sqlite3_str_vappendf(str, zSql, va); + va_end(va); + rc = sqlite3_str_errcode(str); + if(rc) fatal("sqlite3_str_errcode() = %d", rc); + z = db_str_finish(str, &n); + rc = sqlite3_prepare_v2(g.db, z, n, pStmt, 0); + if(rc) fatal("Error #%d (%s) preparing: %s", + rc, sqlite3_errmsg(g.db), z); + sqlite3_free(z); +} + +void db_bind_int(sqlite3_stmt *pStmt, int col, int val){ + int const rc = sqlite3_bind_int(pStmt, col, val); + db_affirm_rc(rc,"from db_bind_int()"); +} + +#if 0 +void db_bind_null(sqlite3_stmt *pStmt, int col){ + int const rc = sqlite3_bind_null(pStmt, col); + db_affirm_rc(rc,"from db_bind_null()"); +} +#endif + +void db_bind_textn(sqlite3_stmt *pStmt, int col, + const char * zStr, int n){ + int const rc = zStr + ? sqlite3_bind_text(pStmt, col, zStr, n, SQLITE_TRANSIENT) + : sqlite3_bind_null(pStmt, col); + db_affirm_rc(rc,"from db_bind_textn()"); +} + +void db_bind_text(sqlite3_stmt *pStmt, int col, + const char * zStr){ + db_bind_textn(pStmt, col, zStr, -1); +} + +#if 0 +void db_bind_textv(sqlite3_stmt *pStmt, int col, + const char * zFmt, ...){ + int rc; + sqlite3_str * str = db_str_new(); + int n = 0; + char * z; + va_list va; + va_start(va,zFmt); + sqlite3_str_vappendf(str, zFmt, va); + va_end(va); + z = db_str_finish(str, &n); + rc = sqlite3_bind_text(pStmt, col, z, n, sqlite3_free); + db_affirm_rc(rc,"from db_bind_textv()"); +} +#endif + +void db_free(void *m){ + sqlite3_free(m); +} + +void db_define_add(const char * zKey){ + int rc; + if(!g.stmt.defIns){ + db_prepare(&g.stmt.defIns, + "INSERT OR REPLACE INTO def(k) VALUES(?)"); + } + db_bind_text(g.stmt.defIns, 1, zKey); + rc = db_step(g.stmt.defIns); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping INSERT on def"); + } + g_debug(2,("define: %s\n",zKey)); + sqlite3_reset(g.stmt.defIns); +} + +int db_define_has(const char * zName){ + int rc; + if(!g.stmt.defHas){ + db_prepare(&g.stmt.defHas, "SELECT 1 FROM def WHERE k=?"); + } + db_bind_text(g.stmt.defHas, 1, zName); + rc = db_step(g.stmt.defHas); + if(SQLITE_ROW == rc){ + rc = 1; + }else{ + assert(SQLITE_DONE==rc); + rc = 0; + } + g_debug(1,("define has [%s] = %d\n",zName, rc)); + sqlite3_clear_bindings(g.stmt.defHas); + sqlite3_reset(g.stmt.defHas); + return rc; +} + + +void db_define_rm(const char * zKey){ + int rc; + int n = 0; + const char *zPos = zKey; + if(!g.stmt.defDel){ + db_prepare(&g.stmt.defDel, "DELETE FROM def WHERE k=?"); + } + for( ; *zPos && '='!=*zPos; ++n, ++zPos) {} + db_bind_text(g.stmt.defDel, 1, zKey); + rc = db_step(g.stmt.defDel); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping DELETE on def"); + } + g_debug(2,("undefine: %.*s\n",n, zKey)); + sqlite3_clear_bindings(g.stmt.defDel); + sqlite3_reset(g.stmt.defDel); +} + +void db_including_add(const char * zKey, const char * zSrc, int srcLine){ + int rc; + if(!g.stmt.inclIns){ + db_prepare(&g.stmt.inclIns, + "INSERT OR FAIL INTO incl(file,srcFile,srcLine) VALUES(?,?,?)"); + } + db_bind_text(g.stmt.inclIns, 1, zKey); + db_bind_text(g.stmt.inclIns, 2, zSrc); + db_bind_int(g.stmt.inclIns, 3, srcLine); + rc = db_step(g.stmt.inclIns); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping INSERT on incl"); + } + g_debug(2,("inclpath add [%s] from [%s]:%d\n", zKey, zSrc, srcLine)); + sqlite3_clear_bindings(g.stmt.inclIns); + sqlite3_reset(g.stmt.inclIns); +} + +void db_include_rm(const char * zKey){ + int rc; + if(!g.stmt.inclDel){ + db_prepare(&g.stmt.inclDel, "DELETE FROM incl WHERE file=?"); + } + db_bind_text(g.stmt.inclDel, 1, zKey); + rc = db_step(g.stmt.inclDel); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping DELETE on incl"); + } + g_debug(2,("inclpath rm [%s]\n", zKey)); + sqlite3_clear_bindings(g.stmt.inclDel); + sqlite3_reset(g.stmt.inclDel); +} + +char * db_include_search(const char * zKey){ + char * zName = 0; + if(!g.stmt.inclSearch){ + db_prepare(&g.stmt.inclSearch, + "SELECT ?1 fn WHERE fileExists(fn) " + "UNION ALL SELECT * FROM (" + "SELECT replace(dir||'/'||?1, '//','/') AS fn " + "FROM inclpath WHERE fileExists(fn) ORDER BY seq" + ")"); + } + db_bind_text(g.stmt.inclSearch, 1, zKey); + if(SQLITE_ROW==db_step(g.stmt.inclSearch)){ + const unsigned char * z = sqlite3_column_text(g.stmt.inclSearch, 0); + zName = z ? sqlite3_mprintf("%s", z) : 0; + if(!zName) fatal("Alloc failed"); + } + sqlite3_clear_bindings(g.stmt.inclSearch); + sqlite3_reset(g.stmt.inclSearch); + return zName; +} + +static int db_including_has(const char * zName){ + int rc; + if(!g.stmt.inclHas){ + db_prepare(&g.stmt.inclHas, "SELECT 1 FROM incl WHERE file=?"); + } + db_bind_text(g.stmt.inclHas, 1, zName); + rc = db_step(g.stmt.inclHas); + if(SQLITE_ROW == rc){ + rc = 1; + }else{ + assert(SQLITE_DONE==rc); + rc = 0; + } + g_debug(2,("inclpath has [%s] = %d\n",zName, rc)); + sqlite3_clear_bindings(g.stmt.inclHas); + sqlite3_reset(g.stmt.inclHas); + return rc; +} + +#if 0 +/* +** Fails fatally if the `#include` list contains the given key. +*/ +static void db_including_check(const char * zKey); +void db_including_check(const char * zName){ + if(db_including_has(zName)){ + fatal("Recursive include detected: %s\n", zName); + } +} +#endif + +void db_include_dir_add(const char * zDir){ + static int seq = 0; + int rc; + if(!g.stmt.inclPathAdd){ + db_prepare(&g.stmt.inclPathAdd, + "INSERT OR FAIL INTO inclpath(seq,dir) VALUES(?,?)"); + } + db_bind_int(g.stmt.inclPathAdd, 1, ++seq); + db_bind_text(g.stmt.inclPathAdd, 2, zDir); + rc = db_step(g.stmt.inclPathAdd); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping INSERT on inclpath"); + } + g_debug(2,("inclpath add #%d: %s\n",seq, zDir)); + sqlite3_clear_bindings(g.stmt.inclPathAdd); + sqlite3_reset(g.stmt.inclPathAdd); +} + +static void cmpp_atexit(void){ +#define FINI(M) if(g.stmt.M) sqlite3_finalize(g.stmt.M) + FINI(defIns); FINI(defDel); FINI(defHas); + FINI(inclIns); FINI(inclDel); FINI(inclHas); + FINI(inclPathAdd); FINI(inclSearch); +#undef FINI + FileWrapper_close(&g.out); + if(g.db) sqlite3_close(g.db); +} + +/* +** sqlite3 UDF which returns true if its argument refers to an +** accessible file, else false. +*/ +static void udf_file_exists( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + (void)(argc); /* Unused parameter */ + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + sqlite3_result_int(context, 0==access(zName, 0)); +} + +/* Initialize g.db, failing fatally on error. */ +static void cmpp_initdb(void){ + int rc; + char * zErr = 0; + const char * zSchema = + "CREATE TABLE def(" + "k TEXT PRIMARY KEY NOT NULL" + /*"v INTEGER DEFAULT 1"*/ + ") WITHOUT ROWID;" + /* ^^^ defines */ + "CREATE TABLE incl(" + "file TEXT PRIMARY KEY NOT NULL," + "srcFile TEXT DEFAULT NULL," + "srcLine INTEGER DEFAULT 0" + ") WITHOUT ROWID;" + /* ^^^ files currently being included */ + "CREATE TABLE inclpath(" + "seq INTEGER UNIQUE, " + "dir TEXT PRIMARY KEY NOT NULL ON CONFLICT IGNORE" + ")" + /* ^^^ include path */ + ; + assert(0==g.db); + if(g.db) return; + rc = sqlite3_open_v2(":memory:", &g.db, SQLITE_OPEN_READWRITE, 0); + if(rc) fatal("Error opening :memory: db."); + rc = sqlite3_exec(g.db, zSchema, 0, 0, &zErr); + if(rc) fatal("Error initializing database: %s", zErr); + rc = sqlite3_create_function(g.db, "fileExists", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + udf_file_exists, 0, 0); + db_affirm_rc(rc, "UDF registration failed."); +} + +/* +** For position zPos, which must be in the half-open range +** [zBegin,zEnd), returns g.nDelim if it is at the start of a line and +** starts with g.zDelim, else returns 0. +*/ +static unsigned short cmpp_is_delim(unsigned char const *zBegin, + unsigned char const *zEnd, + unsigned char const *zPos){ + assert(zEnd>zBegin); + assert(zPos=zBegin); + if(zPos>zBegin && + ('\n'!=*(zPos - 1) + || ((unsigned)(zEnd - zPos) <= g.nDelim))){ + return 0; + }else if(0==memcmp(zPos, g.zDelim, g.nDelim)){ + return g.nDelim; + }else{ + return 0; + } +} + +/* +** Scans t to the next keyword line, emitting all input before that +** which is _not_ a keyword line unless it's elided due to being +** inside a block which elides its content. Returns 0 if no keyword +** line was found, in which case the end of the input has been +** reached, else returns a truthy value and sets up t's state for use +** with cmpp_process_keyword(), which should then be called. +*/ +static int cmpp_next_keyword_line(CmppTokenizer * const t){ + unsigned char const * zStart; + unsigned char const * z; + CmppToken * const tok = &t->token; + unsigned short isDelim = 0; + + assert(t->zBegin); + assert(t->zEnd > t->zBegin); + if(!t->zPos) t->zPos = t->zBegin; + t->zAnchor = t->zPos; + zStart = z = t->zPos; + *tok = CmppToken_empty; + while(zzEnd + && 0==(isDelim = cmpp_is_delim(t->zBegin, t->zEnd, z))){ + ++z; + } + if(z>zStart){ + /* We passed up content */ + cmpp_t_out(t, zStart, (unsigned)(z - zStart)); + } + assert(isDelim==0 || isDelim==g.nDelim); + tok->lineNo = t->lineNo += count_lines(zStart, z); + if(isDelim){ + /* Handle backslash-escaped newlines */ + int isEsc = 0, atEol = 0; + tok->zBegin = z+isDelim; + for( ++z ; zzEnd && 0==atEol; ++z ){ + switch((int)*z){ + case (int)'\\': + isEsc = 0==isEsc; break; + case (int)'\n': + atEol = 0==isEsc; + isEsc = 0; + ++t->lineNo; + break; + default: + break; + } + } + tok->zEnd = atEol ? z-1 : z; + /* Strip leading spaces */ + while(tok->zBegin < tok->zEnd && isspace((char)(*tok->zBegin))){ + ++tok->zBegin; + } + tok->ttype = TT_Line; + g_debug(2,("Keyword @ line %u: [[[%.*s]]]\n", + tok->lineNo, + (int)(tok->zEnd-tok->zBegin), tok->zBegin)); + } + t->zPos = z; + if(isDelim){ + /* Split t->token into arguments for the line's keyword */ + int i, argc = 0, prevChar = 0; + const unsigned tokLen = (unsigned)(tok->zEnd - tok->zBegin); + unsigned char * zKwd; + unsigned char * zEsc; + unsigned char * zz; + + assert(TT_Line==tok->ttype); + if((unsigned)sizeof(t->args.lineBuf) < tokLen + 1){ + fatal("Keyword line is unreasonably long: %.*s", + tokLen, tok->zBegin); + }else if(!tokLen){ + fatal("Line #%u has no keyword after delimiter", tok->lineNo); + } + g_debug(2,("token @ line %u len=%u [[[%.*s]]]\n", + tok->lineNo, tokLen, tokLen, tok->zBegin)); + zKwd = &t->args.lineBuf[0]; + memcpy(zKwd, tok->zBegin, tokLen); + memset(zKwd + tokLen, 0, sizeof(t->args.lineBuf) - tokLen); + for( zEsc = 0, zz = zKwd; *zz; ++zz ){ + /* Convert backslash-escaped newlines to whitespace */ + switch((int)*zz){ + case (int)'\\': + if(zEsc) zEsc = 0; + else zEsc = zz; + break; + case (int)'\n': + assert(zEsc && "Should not have an unescaped newline?"); + if(zEsc==zz-1){ + *zEsc = (unsigned char)' '; + /* FIXME?: memmove() lnBuf content one byte to the left here + ** to collapse backslash and newline into a single + ** byte. Also consider collapsing all leading space on the + ** next line. */ + } + zEsc = 0; + *zz = (unsigned char)' '; + break; + default: + zEsc = 0; + break; + } + } + t->args.argv[argc++] = zKwd; + for( zz = zKwd; *zz; ++zz ){ + if(isspace(*zz)){ + *zz = 0; + break; + } + } + t->args.pKw = CmppKeyword_search((char const *)zKwd); + if(!t->args.pKw){ + fatal("Unknown keyword '%s' at line %u\n", (char const *)zKwd, + tok->lineNo); + } + for( ++zz ; *zz && isspace(*zz); ++zz ){} + if(t->args.pKw->bTokenize){ + for( ; *zz; prevChar = *zz, ++zz ){ + /* Split string into word-shaped tokens. + ** TODO ?= quoted strings, for the sake of the + ** #error keyword. */ + if(isspace(*zz)){ + assert(zz!=zKwd && "Leading space was stripped earlier."); + *zz = 0; + }else{ + if(argc == (int)CmppArgs_Max){ + fatal("Too many arguments @ line %u: %.*s", + tok->lineNo, tokLen, tok->zBegin); + }else if(zz>zKwd && !prevChar){ + t->args.argv[argc++] = zz; + } + } + } + }else{ + /* Treat rest of line as one token */ + if(*zz) t->args.argv[argc++] = zz; + } + tok->ttype = t->args.pKw->ttype; + if(g.doDebug>1){ + for(i = 0; i < argc; ++i){ + g_debug(0,("line %u arg #%d=%s\n", + tok->lineNo, i, + (char const *)t->args.argv[i])); + } + } + t->args.argc = argc; + }else{ + t->args.pKw = 0; + t->args.argc = 0; + } + return isDelim; +} + +static void cmpp_kwd__err_prefix(CmppKeyword const * pKw, CmppTokenizer *t, + char const *zPrefix){ + g_stderr("%s%s%s @ %s line %u: ", + zPrefix ? zPrefix : "", + zPrefix ? ": " : "", + pKw->zName, t->zName, t->token.lineNo); +} + +/* Internal error reporting helper for cmpp_keyword_f() impls. */ +static CMPP_NORETURN void cmpp_kwd__misuse(CmppKeyword const * pKw, + CmppTokenizer *t, + char const *zFmt, ...){ + va_list va; + cmpp_kwd__err_prefix(pKw, t, "Fatal error"); + va_start(va, zFmt); + fatalv(zFmt, va); + va_end(va); +} + +/* No-op cmpp_keyword_f() impl. */ +static void cmpp_kwd_noop(CmppKeyword const * pKw, CmppTokenizer *t){ + if(t || pKw){/*unused*/} +} + +/* #error impl. */ +static void cmpp_kwd_error(CmppKeyword const * pKw, CmppTokenizer *t){ + if(CT_skip(t)) return; + else{ + assert(t->args.argc < 3); + const char *zBegin = t->args.argc>1 + ? (const char *)t->args.argv[1] : 0; + cmpp_kwd__err_prefix(pKw, t, NULL); + fatal("%s", zBegin ? zBegin : "(no additional info)"); + } +} + +/* Impl. for #define, #undef */ +static void cmpp_kwd_define(CmppKeyword const * pKw, CmppTokenizer *t){ + if(CT_skip(t)) return; + if(t->args.argc<2){ + cmpp_kwd__misuse(pKw, t, "Expecting one or more arguments"); + }else{ + int i = 1; + void (*func)(const char *) = TT_Define==pKw->ttype + ? db_define_add : db_define_rm; + for( ; i < t->args.argc; ++i){ + func( (char const *)t->args.argv[i] ); + } + } +} + +/* Impl. for #if, #ifnot, #elif, #elifnot. */ +static void cmpp_kwd_if(CmppKeyword const * pKw, CmppTokenizer *t){ + int buul; + CmppParseState tmpState = TS_Start; + if(t->args.argc!=2){ + cmpp_kwd__misuse(pKw, t, "Expecting exactly 1 argument"); + } + /*g_debug(0,("%s %s level %u pstate=%d\n", pKw->zName, + (char const *)t->args.argv[1], + t->level.ndx, (int)CT_pstate(t)));*/ + switch(pKw->ttype){ + case TT_Elif: + case TT_ElifNot: + switch(CT_pstate(t)){ + case TS_If: break; + case TS_IfPassed: CT_level(t).flags |= CmppLevel_F_ELIDE; return; + default: goto misuse; + } + break; + case TT_If: + case TT_IfNot: + CmppLevel_push(t); + break; + default: + cmpp_kwd__misuse(pKw, t, "Unpexected keyword token type"); + break; + } + buul = db_define_has((char const *)t->args.argv[1]); + if(TT_IfNot==pKw->ttype || TT_ElifNot==pKw->ttype) buul = !buul; + if(buul){ + CT_pstate(t) = tmpState = TS_IfPassed; + CT_skipLevel(t) = 0; + }else{ + CT_pstate(t) = TS_If /* also for TT_IfNot, TT_Elif, TT_ElifNot */; + CT_skipLevel(t) = 1; + } + if(TT_If==pKw->ttype || TT_IfNot==pKw->ttype){ + unsigned const lvlIf = t->level.ndx; + CmppToken const lvlToken = CT_level(t).token; + while(cmpp_next_keyword_line(t)){ + cmpp_process_keyword(t); + if(lvlIf > t->level.ndx){ + assert(TT_EndIf == t->token.ttype); + break; + } + if(TS_IfPassed==tmpState){ + tmpState = TS_Start; + t->level.stack[lvlIf].flags |= CmppLevel_F_ELIDE; + } + } + if(lvlIf <= t->level.ndx){ + cmpp_kwd__err_prefix(pKw, t, NULL); + fatal("Input ended inside an unterminated %sif " + "opened at [%s] line %u", + g.zDelim, t->zName, lvlToken.lineNo); + } + } + return; + misuse: + cmpp_kwd__misuse(pKw, t, "'%s' used out of context", + pKw->zName); +} + +/* Impl. for #else. */ +static void cmpp_kwd_else(CmppKeyword const * pKw, CmppTokenizer *t){ + if(t->args.argc>1){ + cmpp_kwd__misuse(pKw, t, "Expecting no arguments"); + } + switch(CT_pstate(t)){ + case TS_IfPassed: CT_skipLevel(t) = 1; break; + case TS_If: CT_skipLevel(t) = 0; break; + default: + cmpp_kwd__misuse(pKw, t, "'%s' with no matching 'if'", + pKw->zName); + } + /*g_debug(0,("else flags=0x%02x skipLevel=%u\n", + CT_level(t).flags, CT_level(t).skipLevel));*/ + CT_pstate(t) = TS_Else; +} + +/* Impl. for #endif. */ +static void cmpp_kwd_endif(CmppKeyword const * pKw, CmppTokenizer *t){ + /* Maintenance reminder: we ignore all arguments after the endif + ** to allow for constructs like: + ** + ** #endif // foo + ** + ** in a manner which does not require a specific comment style */ + switch(CT_pstate(t)){ + case TS_Else: + case TS_If: + case TS_IfPassed: + break; + default: + cmpp_kwd__misuse(pKw, t, "'%s' with no matching 'if'", + pKw->zName); + } + CmppLevel_pop(t); +} + +/* Impl. for #include. */ +static void cmpp_kwd_include(CmppKeyword const * pKw, CmppTokenizer *t){ + char const * zFile; + char * zResolved; + if(CT_skip(t)) return; + else if(t->args.argc!=2){ + cmpp_kwd__misuse(pKw, t, "Expecting exactly 1 filename argument"); + } + zFile = (const char *)t->args.argv[1]; + if(db_including_has(zFile)){ + /* Note that different spellings of the same filename + ** will elude this check, but that seems okay, as different + ** spellings means that we're not re-running the exact same + ** invocation. We might want some other form of multi-include + ** protection, rather than this, however. There may well be + ** sensible uses for recursion. */ + cmpp_kwd__err_prefix(pKw, t, NULL); + fatal("Recursive include of file: %s", zFile); + } + zResolved = db_include_search(zFile); + if(zResolved){ + db_including_add(zFile, t->zName, t->token.lineNo); + cmpp_process_file(zResolved); + db_include_rm(zFile); + db_free(zResolved); + }else{ + cmpp_kwd__err_prefix(pKw, t, NULL); + fatal("file not found: %s", zFile); + } +} + +/* Impl. for #pragma. */ +static void cmpp_kwd_pragma(CmppKeyword const * pKw, CmppTokenizer *t){ + const char * zArg; + if(CT_skip(t)) return; + else if(t->args.argc!=2){ + cmpp_kwd__misuse(pKw, t, "Expecting one argument"); + } + zArg = (const char *)t->args.argv[1]; +#define M(X) 0==strcmp(zArg,X) + if(M("defines")){ + sqlite3_stmt * q = 0; + db_prepare(&q, "SELECT k FROM def ORDER BY k"); + g_stderr("cmpp defines:\n"); + while(SQLITE_ROW==db_step(q)){ + int const n = sqlite3_column_bytes(q, 0); + const char * z = (const char *)sqlite3_column_text(q, 0); + g_stderr("\t%.*s\n", n, z); + } + db_finalize(q); + }else{ + cmpp_kwd__misuse(pKw, t, "Unknown pragma"); + } +#undef M +} + +/* #stder impl. */ +static void cmpp_kwd_stderr(CmppKeyword const * pKw, CmppTokenizer *t){ + if(CT_skip(t)) return; + else{ + const char *zBegin = t->args.argc>1 + ? (const char *)t->args.argv[1] : 0; + if(zBegin){ + g_stderr("%s:%u: %s\n", t->zName, t->token.lineNo, zBegin); + }else{ + g_stderr("%s:%u: (no %.*s%s argument)\n", + t->zName, t->token.lineNo, + g.nDelim, g.zDelim, pKw->zName); + } + } +} + +#if 0 +/* Impl. for dummy placeholder. */ +static void cmpp_kwd_todo(CmppKeyword const * pKw, CmppTokenizer *t){ + if(t){/*unused*/} + g_debug(0,("TODO: keyword handler for %s\n", pKw->zName)); +} +#endif + +CmppKeyword aKeywords[] = { +/* Keep these sorted by zName */ + {"//", 2, 0, TT_Comment, cmpp_kwd_noop}, + {"define", 6, 1, TT_Define, cmpp_kwd_define}, + {"elif", 4, 1, TT_Elif, cmpp_kwd_if}, + {"elifnot", 7, 1, TT_ElifNot, cmpp_kwd_if}, + {"else", 4, 1, TT_Else, cmpp_kwd_else}, + {"endif", 5, 0, TT_EndIf, cmpp_kwd_endif}, + {"error", 4, 0, TT_Error, cmpp_kwd_error}, + {"if", 2, 1, TT_If, cmpp_kwd_if}, + {"ifnot", 5, 1, TT_IfNot, cmpp_kwd_if}, + {"include", 7, 0, TT_Include, cmpp_kwd_include}, + {"pragma", 6, 1, TT_Pragma, cmpp_kwd_pragma}, + {"stderr", 6, 0, TT_Stderr, cmpp_kwd_stderr}, + {"undef", 5, 1, TT_Undef, cmpp_kwd_define}, + {0,0,TT_Invalid, 0} +}; + +static int cmp_CmppKeyword(const void *p1, const void *p2){ + char const * zName = (const char *)p1; + CmppKeyword const * kw = (CmppKeyword const *)p2; + return strcmp(zName, kw->zName); +} + +CmppKeyword const * CmppKeyword_search(const char *zName){ + return (CmppKeyword const *)bsearch(zName, &aKeywords[0], + sizeof(aKeywords)/sizeof(aKeywords[0]) - 1, + sizeof(aKeywords[0]), + cmp_CmppKeyword); +} + +void cmpp_process_keyword(CmppTokenizer * const t){ + assert(t->args.pKw); + assert(t->args.argc); + t->args.pKw->xCall(t->args.pKw, t); + t->args.pKw = 0; + t->args.argc = 0; +} + +void cmpp_process_file(const char * zName){ + FileWrapper fw = FileWrapper_empty; + CmppTokenizer ct = CmppTokenizer_empty; + + FileWrapper_open(&fw, zName, "r"); + FileWrapper_slurp(&fw); + g_debug(1,("Read %u byte(s) from [%s]\n", fw.nContent, fw.zName)); + ct.zName = zName; + ct.zBegin = fw.zContent; + ct.zEnd = fw.zContent + fw.nContent; + while(cmpp_next_keyword_line(&ct)){ + cmpp_process_keyword(&ct); + } + FileWrapper_close(&fw); + if(0!=ct.level.ndx){ + CmppLevel * const lv = CmppLevel_get(&ct); + fatal("Input ended inside an unterminated nested construct" + "opened at [%s] line %u", zName, lv->token.lineNo); + } +} + +static void usage(int isErr){ + FILE * const fOut = isErr ? stderr : stdout; + fprintf(fOut, + "Usage: %s [flags] [infile]\n" + "Flags:\n", + g.zArgv0); +#define arg(F,D) fprintf(fOut," %s\n %s\n",F, D) + arg("-f|--file FILE","Read input from FILE (default=- (stdin)).\n" + " Alternately, the first non-flag argument is assumed to " + "be the input file."); + arg("-o|--outfile FILE","Send output to FILE (default=- (stdout))"); + arg("-DXYZ","Define XYZ to true"); + arg("-UXYZ","Undefine XYZ (equivalent to false)"); + arg("-IXYZ","Add dir XYZ to include path"); + arg("-d|--delimiter VALUE", "Set keyword delimiter to VALUE " + "(default=" CMPP_DEFAULT_DELIM ")"); +#undef arg + fputs("",fOut); +} + +int main(int argc, char const * const * argv){ + int rc = 0; + int i; + int inclCount = 0; + const char * zInfile = 0; +#define M(X) (0==strcmp(X,zArg)) +#define ISFLAG(X) else if(M(X)) +#define ISFLAG2(X,Y) else if(M(X) || M(Y)) +#define ARGVAL \ + if(i+1>=argc) fatal("Missing value for flag '%s'", zArg); \ + zArg = argv[++i] + g.zArgv0 = argv[0]; + atexit(cmpp_atexit); + cmpp_initdb(); + for(i = 1; i < argc; ++i){ + char const * zArg = argv[i]; + while('-'==*zArg) ++zArg; + if(M("?") || M("help")) { + usage(0); + goto end; + }else if('D'==*zArg){ + ++zArg; + if(!*zArg) fatal("Missing key for -D"); + db_define_add(zArg); + }else if('U'==*zArg){ + ++zArg; + if(!*zArg) fatal("Missing key for -U"); + db_define_rm(zArg); + }else if('I'==*zArg){ + ++zArg; + if(!*zArg) fatal("Missing directory for -I"); + db_include_dir_add(zArg); + ++inclCount; + } + ISFLAG2("o","outfile"){ + ARGVAL; + if(g.out.zName) fatal("Cannot use -o more than once."); + g.out.zName = zArg; + } + ISFLAG2("f","file"){ + ARGVAL; + do_infile: + if(zInfile) fatal("Cannot use -i more than once."); + zInfile = zArg; + } + ISFLAG2("d","delimiter"){ + ARGVAL; + g.zDelim = zArg; + g.nDelim = (unsigned short)strlen(zArg); + if(!g.nDelim) fatal("Keyword delimiter may not be empty."); + } + ISFLAG("debug"){ + ++g.doDebug; + }else if(!zInfile){ + goto do_infile; + }else{ + fatal("Unhandled flag: %s", argv[i]); + } + } + if(!zInfile) zInfile = "-"; + if(!g.out.zName) g.out.zName = "-"; + if(!inclCount) db_include_dir_add("."); + FileWrapper_open(&g.out, g.out.zName, "w"); + cmpp_process_file(zInfile); + FileWrapper_close(&g.out); + end: + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} + +#undef CT_level +#undef CT_pstate +#undef CT_skipLevel +#undef CT_skip +#undef CLvl_skip diff --git a/manifest b/manifest index 6821e2b994..8b2b8bd429 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sinto\sjs-cpp\sbranch. -D 2022-11-17T15:21:49.967 +C Replace\suse\sof\scpp\swith\sthe\sfit-to-purpose\sc-pp\sto\savoid\scpp's\sC-centric/JS-unfriendly\squirks. +D 2022-11-18T02:29:59.533 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,7 +488,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 5034e5c0ebbf15479887d7253dcb2efb292906fb78d3e69701191c6257c45a9c +F ext/wasm/GNUmakefile 3c97824ce76c7678344844e581a07ba75901a06def4d0046a2a410d9dd635a83 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 @@ -498,11 +498,11 @@ F ext/wasm/api/extern-post-js.js c197b7567496fc27766842f8c4f4963054bf8c667926ab3 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 -F ext/wasm/api/pre-js.js 9620327120abb15b3af96f72ef9efbcf69e78d90e501328521108b93547a8eb8 +F ext/wasm/api/pre-js.js 749bbbac2f1a2192eaba80cf5e00a3219da78b3c0a84e3019b5eef30e0bc9b88 F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js 1def5fd676142ebe69594c77c5cf8523be1ca0880846a441bf5f981ff529672b +F ext/wasm/api/sqlite3-api-opfs.js 4368a30586df3e11339a72082c77bdef670d619c6185c2dd1ebf5208c3ab0a5c F ext/wasm/api/sqlite3-api-prologue.js fd526fa017fa2578673ca18158354515c719e719a5d93f2f6d0e43f39170430e F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -513,6 +513,7 @@ F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52af F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 F ext/wasm/batch-runner.js 49609e89aaac9989d6c1ad3fae268e4878e1ad7bc5fd3e5c2f44959660780b2e +F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c0779 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/common/testing.css 35889709547d89a6109ff83b25c11bbc91d8dd43aab8722e428655ca98880a06 @@ -2055,8 +2056,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 718a6d371e61359d73c8f80afdb248e3d9b4d8df4c4e5c122ac884344e31035b 7c572d02e60a83b36543ba4d9d45f61e9fc111b61fee085410c2d87558c732d6 -R dcd3d37cda21e5021973ee2a87f95321 +P e047b33d1fb7d6a32e967f03f9952249cd2da4d21dc301fe92bd7baa0da5d6a9 +R f08c58c3d67343dbb1484f5dc6d0d557 U stephan -Z 83259d3e3081896d2aa5795af34840f7 +Z df00239aed9bf555d5579b7e3a06d443 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a3eda4f1e8..bef9d42ccd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e047b33d1fb7d6a32e967f03f9952249cd2da4d21dc301fe92bd7baa0da5d6a9 \ No newline at end of file +49d70f071e918d5d095c807575bb7ce2b287a123261e789e938521b3b409429a \ No newline at end of file From 26e0f2e916c2513dcee68e17642285c1d019e678 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 18 Nov 2022 17:57:19 +0000 Subject: [PATCH 012/282] Update the version number to 3.41.0 to begin the next development cycle. FossilOrigin-Name: 5c669f5f399fe89998b9edba6486f2a6fe5fca789ed82e8711349c8736b293d9 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 17 ++++++++--------- manifest.uuid | 2 +- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/VERSION b/VERSION index 7e16c94210..371986f6df 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.40.0 +3.41.0 diff --git a/configure b/configure index 4a32d5e561..75ca8935ff 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.40.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.41.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.40.0' -PACKAGE_STRING='sqlite 3.40.0' +PACKAGE_VERSION='3.41.0' +PACKAGE_STRING='sqlite 3.41.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1468,7 +1468,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.40.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.41.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1533,7 +1533,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.40.0:";; + short | recursive ) echo "Configuration of sqlite 3.41.0:";; esac cat <<\_ACEOF @@ -1661,7 +1661,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.40.0 +sqlite configure 3.41.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2080,7 +2080,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.40.0, which was +It was created by sqlite $as_me 3.41.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12390,7 +12390,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.40.0, which was +This file was extended by sqlite $as_me 3.41.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12456,7 +12456,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.40.0 +sqlite config.status 3.41.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/manifest b/manifest index e49970e29b..e17c2e03a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C shell.c.in:\son\snon-Windows\splatforms,\scheck\sfor\s$XDG_CONFIG_HOME/sqlite3/sqliterc\sbefore\s~/.sqliterc,\sper\srequest\sin\s[forum:7a16582b1e403c81|forum\spost\s7a16582b1e403c81]. -D 2022-11-18T15:22:45.180 +C Update\sthe\sversion\snumber\sto\s3.41.0\sto\sbegin\sthe\snext\sdevelopment\scycle. +D 2022-11-18T17:57:19.737 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -7,7 +7,7 @@ F Makefile.in 78e4c4916f2c3993a8a454018745ff02094a8029d449d0c22db98f1cf8260420 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc e7a564ceec71f0d9666031d5638cf0d4f88b050b44e8df5d32125137cd259ac0 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e -F VERSION 8868ddfa6e1eee218286021a94b3e22d13e550c76c72d878857547ca001de24a +F VERSION 413ec94920a487ae32c9a2a8819544d690662d6f7c7ce025c0d0b8a1e74fa9db F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 @@ -33,7 +33,7 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034 F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure b93755fe94b3e9b2015f3c2d6c63928e8899b0a7c54e042deb843498094179da x +F configure 9cbf6a15a4b5f2b3551a466420e5f51ce04c40aaed7f90c886402bf0b66ec110 x F configure.ac 48dc6bfee293eef05910faf085760f2fd79b680aa47b50e8e6a22ca40bb026bb F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd @@ -2055,9 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P abb18f61c5cec0f524acc41453b4c06b61c5af51ff46417588837fc0c3967288 49c6e438a83b9ff40ebadd3dfd5f58e6eea053575e15335909f5ee59a6dba82c -R b7d6e814ba737a6b7a457711e5f6de4b -T +closed 49c6e438a83b9ff40ebadd3dfd5f58e6eea053575e15335909f5ee59a6dba82c Closed\sby\sintegrate-merge. -U stephan -Z 43c8e43e0f98659e4d07cefba98fa2de +P 17065d095d26a814acf1e13f5cc18b21fecc58eb8c9da100458029bb139fcd35 +R 22a528d37f506919a8fa02d3dfd9adb1 +U drh +Z fb95072ca0b99e5cdd6cbf0ffc52d763 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 77b2a474df..1f7bef4760 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17065d095d26a814acf1e13f5cc18b21fecc58eb8c9da100458029bb139fcd35 \ No newline at end of file +5c669f5f399fe89998b9edba6486f2a6fe5fca789ed82e8711349c8736b293d9 \ No newline at end of file From cf3107c7d47f3da737f2008f85965a4497d84314 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 19 Nov 2022 00:08:35 +0000 Subject: [PATCH 013/282] Databases created using sqlite3_deserialize() should report their filename as an empty string, not as "x". Fix for ticket [53043c9793715f08]. FossilOrigin-Name: ff494449efd475878c549728cc22ee9b12d13674068781747fc042a0c1bd09c8 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/memdb.c | 7 +++++++ src/pager.c | 6 +++++- src/sqliteInt.h | 3 +++ 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index e17c2e03a8..4ead71ff69 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sversion\snumber\sto\s3.41.0\sto\sbegin\sthe\snext\sdevelopment\scycle. -D 2022-11-18T17:57:19.737 +C Databases\screated\susing\ssqlite3_deserialize()\sshould\sreport\stheir\sfilename\nas\san\sempty\sstring,\snot\sas\s"x".\s\sFix\sfor\sticket\s[53043c9793715f08]. +D 2022-11-19T00:08:35.872 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -607,7 +607,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c 5a3dbd8ac8a6501152a4fc1fcae9b0900c2d7eb0589c4ec7456fdde15725a26c -F src/memdb.c c2dc88f97c410eb68a24468344b65526685e18354ddfd15906750c1eaf9dc2dd +F src/memdb.c a42248bc04dc1f80da3511618a78956ed77b65058b73e251bde39061204e9f97 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 @@ -624,7 +624,7 @@ F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d87210 F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 6176d9752eb580419e8fef4592dc417a6b00ddfd43ee22f818819bf8840ceee8 +F src/pager.c d3122cf67f327f1e2df12d06236a3473a8099542071e257067552f42917f172d F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197 @@ -642,7 +642,7 @@ F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 2c24ba38f78e32fe5d7ec136321a6ad827698b33ca98664970a8b7274d69ef7c +F src/sqliteInt.h 1a2cec0f7682da69975eb203fef2058949051ff500445b997d8047cbd8813315 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2055,8 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17065d095d26a814acf1e13f5cc18b21fecc58eb8c9da100458029bb139fcd35 -R 22a528d37f506919a8fa02d3dfd9adb1 +P 5c669f5f399fe89998b9edba6486f2a6fe5fca789ed82e8711349c8736b293d9 +R 06e897a36d76f0b40d32a92b9dcc8859 U drh -Z fb95072ca0b99e5cdd6cbf0ffc52d763 +Z 7c255a26bf6fe056b55fba3394c16cef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1f7bef4760..668e2dc4ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c669f5f399fe89998b9edba6486f2a6fe5fca789ed82e8711349c8736b293d9 \ No newline at end of file +ff494449efd475878c549728cc22ee9b12d13674068781747fc042a0c1bd09c8 \ No newline at end of file diff --git a/src/memdb.c b/src/memdb.c index 31b2324b93..0aecb71022 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -857,6 +857,13 @@ end_deserialize: return rc; } +/* +** Return true if the VFS is the memvfs. +*/ +int sqlite3IsMemdb(const sqlite3_vfs *pVfs){ + return pVfs==&memdb_vfs; +} + /* ** This routine is called when the extension is loaded. ** Register the new VFS. diff --git a/src/pager.c b/src/pager.c index 883e6532cf..c859f50285 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7009,7 +7009,11 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ */ const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; - return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename; + if( nullIfMemDb && (pPager->memDb || sqlite3IsMemdb(pPager->pVfs)) ){ + return &zFake[4]; + }else{ + return pPager->zFilename; + } } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e4b74f6d0b..469ce9bba1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -5003,6 +5003,9 @@ const char *sqlite3ErrName(int); #ifndef SQLITE_OMIT_DESERIALIZE int sqlite3MemdbInit(void); +int sqlite3IsMemdb(const sqlite3_vfs*); +#else +# define sqlite3IsMemdb(X) 0 #endif const char *sqlite3ErrStr(int); From 85728a21c63e8feb1c13d2cf81962bd34b023a08 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 19 Nov 2022 00:22:12 +0000 Subject: [PATCH 014/282] Small performance optimization in btree.c. FossilOrigin-Name: f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 38 +++++++++++++++++++------------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 4ead71ff69..1fd11cbf21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Databases\screated\susing\ssqlite3_deserialize()\sshould\sreport\stheir\sfilename\nas\san\sempty\sstring,\snot\sas\s"x".\s\sFix\sfor\sticket\s[53043c9793715f08]. -D 2022-11-19T00:08:35.872 +C Small\sperformance\soptimization\sin\sbtree.c. +D 2022-11-19T00:22:12.991 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -576,7 +576,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 6321ff29261bf9726e6b231058ff21b1ccf9f441a0b718b76c37341b16fa14ce +F src/btree.c e8fae9a95ea9561aebc41e467a9ee9ba9150ca373031e65773d62ff02d8250d2 F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2055,8 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5c669f5f399fe89998b9edba6486f2a6fe5fca789ed82e8711349c8736b293d9 -R 06e897a36d76f0b40d32a92b9dcc8859 +P ff494449efd475878c549728cc22ee9b12d13674068781747fc042a0c1bd09c8 +R 8e4332decceeea7e1973b8e5ac88e8a4 U drh -Z 7c255a26bf6fe056b55fba3394c16cef +Z 80da1bc9f69008afc02dd88003bee918 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 668e2dc4ed..7e33296fde 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff494449efd475878c549728cc22ee9b12d13674068781747fc042a0c1bd09c8 \ No newline at end of file +f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index df24b7b33c..1c6c14d4b6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5495,9 +5495,25 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. */ +static SQLITE_NOINLINE int btreeLast(BtCursor *pCur, int *pRes){ + int rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ - int rc; - assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); @@ -5518,23 +5534,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ *pRes = 0; return SQLITE_OK; } - - rc = moveToRoot(pCur); - if( rc==SQLITE_OK ){ - assert( pCur->eState==CURSOR_VALID ); - *pRes = 0; - rc = moveToRightmost(pCur); - if( rc==SQLITE_OK ){ - pCur->curFlags |= BTCF_AtLast; - }else{ - pCur->curFlags &= ~BTCF_AtLast; - } - }else if( rc==SQLITE_EMPTY ){ - assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); - *pRes = 1; - rc = SQLITE_OK; - } - return rc; + return btreeLast(pCur, pRes); } /* Move the cursor so that it points to an entry in a table (a.k.a INTKEY) From ff4917e9a537deeb0bdbbec7c9ea3849e8a095c6 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 19 Nov 2022 02:32:26 +0000 Subject: [PATCH 015/282] Create new branch named "base_convert" FossilOrigin-Name: 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index 1fd11cbf21..dabd88714c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\soptimization\sin\sbtree.c. -D 2022-11-19T00:22:12.991 +C Create\snew\sbranch\snamed\s"base_convert" +D 2022-11-19T02:32:26.249 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2055,8 +2055,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff494449efd475878c549728cc22ee9b12d13674068781747fc042a0c1bd09c8 +P f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff R 8e4332decceeea7e1973b8e5ac88e8a4 -U drh -Z 80da1bc9f69008afc02dd88003bee918 +T *branch * base_convert +T *sym-base_convert * +T -sym-trunk * +U larrybr +Z 21ba949f5c7ccf39d76b47dd8fe1f905 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e33296fde..bc34d0feed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff \ No newline at end of file +0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea \ No newline at end of file From fb2e0bfa8b540fb6e7be5a74beaadd06b60570c1 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 19 Nov 2022 02:39:16 +0000 Subject: [PATCH 016/282] New extensions for base85 and base64 conversion UDFs FossilOrigin-Name: 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 --- ext/misc/base64.c | 233 +++++++++++++++++++++++++++++ ext/misc/base85.c | 366 ++++++++++++++++++++++++++++++++++++++++++++++ manifest | 15 +- manifest.uuid | 2 +- 4 files changed, 607 insertions(+), 9 deletions(-) create mode 100644 ext/misc/base64.c create mode 100644 ext/misc/base85.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c new file mode 100644 index 0000000000..1ed1a4f174 --- /dev/null +++ b/ext/misc/base64.c @@ -0,0 +1,233 @@ +/* +** 2022-11-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a SQLite extension for converting in either direction +** between a (binary) blob and base64 text. Base64 can transit a +** sane ASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals, +** and can be used unmodified in XML-like documents. +** +** This is an independent implementation of conversions specified in +** RFC 4648, done on the above date by the author (Larry Brasfield) +** who thereby has the right to put this into the public domain. +** +** The conversions meet RFC 4648 requirements, provided that this +** C source specifies that line-feeds are included in the encoded +** data to limit visible line lengths to 72 characters. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Generated base64 sequences, with their line-feeds included, +** can be concatenated; the result converted back to binary will +** be the concatenation of the represented binary sequences. +** +** This SQLite3 extension creates a function, base64(x), which +** either: converts text x containing base64 to a returned blob; +** or converts a blob x to returned text containing base64. An +** error will be thrown for other input argument types. +** +** This code relies on UTF-8 encoding only with respect to the +** meaning of the first 128 (7-bit) codes being the same as ASCII. +** It will fail miserably if somehow made to try to convert EBCDIC. +** Because it is table-driven, it could be enhanced to handle that. +** But the world and SQLite have moved on from that anachronism. +** +** To build the extension: +** Set shell variable SQDIR= +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ + +typedef unsigned char ubyte; + +static const ubyte b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80) +#define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[dv]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define DARK_MAX 72 + +/* Encode a byte buffer into base64 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + *pOut = 0; + while( nbIn > 0 ){ + static signed char ncio[] = { 0, 2, 3, 4 }; + int nbi = (nbIn > 3)? 3 : nbIn; + signed char nc; + int nbe; + unsigned long qv = (ubyte)*pIn++; + for( nbe=1; nbe<3; ++nbe ){ + ubyte b = (nbe=0; --nbe ){ + char ce = (nbe0 && *pIn!='=' ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + if( ncIn<=0 ) break; + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + nbo = nboi[nti]; + while( nti>0 ){ + char c = *pIn++; + ubyte bdp = BX_DV_PROTO(c); + --ncIn; + switch( bdp ){ + case WS: + case ND: + nti = 0; + break; + case PC: + bdp = 0; + /* fall thru */ + default: /* It's the digit value. */ + qv = qv<<6 | bdp; + --nti; + break; + } + } + while( nbo-- > 0 ){ + *pOut++ = (qv >> (8*nbo))&0xff; + } + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + ubyte *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*(nv+2/3); /* quads needed */ + nc += (nc+(DARK_MAX-1))/DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big.", -1); + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + bBuf = (ubyte*)sqlite3_value_blob(av[0]); + nc = (int)(toBase64(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big.", -1); + }else if( nb<1 ){ + nb = 1; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + cBuf = (char *)sqlite3_value_text(av[0]); + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text.", -1); + break; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} diff --git a/ext/misc/base85.c b/ext/misc/base85.c new file mode 100644 index 0000000000..365db80f2b --- /dev/null +++ b/ext/misc/base85.c @@ -0,0 +1,366 @@ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane ASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit ASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +/* Classify c according to interval within ASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static unsigned char b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static unsigned char base85DigitValue( char c ){ + unsigned char dv = (unsigned char)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +static char * skipNonB85( char *s ){ + char c; + while( (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +static char base85Numeral( unsigned char b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} + +static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + *pOut = 0; + while( nbIn > 0 ){ + static signed char ncio[] = { 0, 2, 3, 4, 5 }; + int nbi = (nbIn > 4)? 4 : nbIn; + unsigned long qv = 0L; + int nbe = 0; + signed char nco; + while( nbe++ < nbi ){ + qv = (qv<<8) | *pIn++; + } + nco = ncio[nbi]; + nbIn -= nbi; + while( nco > 0 ){ + unsigned char dv = (unsigned char)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += ncio[nbi]; + if( pSep && ((nCol += ncio[nbi])>=80 || nbIn<=0) ){ + char *p = pSep; + while( *p ) *pOut++ = *p++; + nCol = 0; + } + *pOut = 0; + } + return pOut; +} + +static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + if( ncIn==0 ) break; + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + while( nti>0 ){ + char c = *pIn++; + unsigned char cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + c - cdo; + --nti; + } + nbo -= nti; + while( nbo-- > 0 ){ + *pOut++ = (qv >> (8*nbo))&0xff; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +# ifndef OMIT_BASE85_CHECKER +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL.", -1); + break; + } +} +# endif + +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + unsigned char *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big.", -1); + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + bBuf = (unsigned char*)sqlite3_value_blob(av[0]); + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big.", -1); + }else if( nb<1 ){ + nb = 1; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + cBuf = (char *)sqlite3_value_text(av[0]); + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + break; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +# ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +# endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + unsigned char bBuf[64]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +# ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +# endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif diff --git a/manifest b/manifest index dabd88714c..faf2f3f49e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\snew\sbranch\snamed\s"base_convert" -D 2022-11-19T02:32:26.249 +C New\sextensions\sfor\sbase85\sand\sbase64\sconversion\sUDFs +D 2022-11-19T02:39:16.296 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,6 +289,8 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 +F ext/misc/base64.c e4d3f13bb59aa903734b557e1400f3c5961f0a5abc40400e0e2aa3ad362c0fd7 +F ext/misc/base85.c 3e07aea038129ee646b129ac2ed736344ea03859ff447019e67250b80722c528 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -2055,11 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff -R 8e4332decceeea7e1973b8e5ac88e8a4 -T *branch * base_convert -T *sym-base_convert * -T -sym-trunk * +P 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea +R 4ecad003b3a6f60dd6b11d354f8edbe2 U larrybr -Z 21ba949f5c7ccf39d76b47dd8fe1f905 +Z d182470ca73df70df09238ac5a5e4382 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bc34d0feed..b47c8a0962 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea \ No newline at end of file +5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 \ No newline at end of file From f7d98ac7649f6b0ad5cd0597e02e0fd96df437dc Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 19 Nov 2022 02:51:41 +0000 Subject: [PATCH 017/282] More work towards creation of a ES6 JS module. FossilOrigin-Name: 6b826e700f6849eebfbba38e5948b96be245994e3e03ea30743114d3f5689c42 --- ext/wasm/GNUmakefile | 14 ++++++++------ ext/wasm/api/extern-post-js.js | 9 +++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index bd30d306ea..1b85fa5e08 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -225,13 +225,13 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js sqlite3-worker1.js := $(dir.api)/sqlite3-worker1.js sqlite3-worker1-promiser.js := $(dir.api)/sqlite3-worker1-promiser.js -define CP_XAPI +define COPY_XAPI sqlite3-api.ext.jses += $$(dir.dout)/$$(notdir $(1)) $$(dir.dout)/$$(notdir $(1)): $(1) $$(MAKEFILE) cp $$< $$@ endef $(foreach X,$(SOAP.js) $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js),\ - $(eval $(call CP_XAPI,$(X)))) + $(eval $(call COPY_XAPI,$(X)))) all: $(sqlite3-api.ext.jses) sqlite3-api.js := $(dir.tmp)/sqlite3-api.js @@ -270,7 +270,8 @@ $(post-js.js): $(post-jses) $(MAKEFILE) cat $$i; \ echo "/* END FILE: $$i */"; \ done > $@ -extern-post-js.js := $(dir.api)/extern-post-js.js +extern-post-js.js.in := $(dir.api)/extern-post-js.js +extern-post-js.js := $(dir.tmp)/extern-post-js.js extern-pre-js.js := $(dir.api)/extern-pre-js.js pre-post-common.flags := \ --post-js=$(post-js.js) \ @@ -322,15 +323,16 @@ esm: $(filter-out esm,$(MAKECMDGOALS)) else js.cpp.defines ?= endif -define C-PP_JS +define C-PP.JS # $1 = X.js. $2 = output file to generate by filtering $(1) through # $(bin.cpp) -E -CC. $(2): $(1) $$(MAKEFILE) $$(bin.c-pp) $$(bin.c-pp) $(js.cpp.defines) -f $(1) -o $$@ CLEAN_FILES += $(2) endef -$(eval $(call C-PP_JS,$(dir.tmp)/sqlite3-api.c-pp.js,$(dir.tmp)/sqlite3-api.js)) -$(eval $(call C-PP_JS,$(dir.api)/pre-js.js,$(dir.tmp)/pre-js.js)) +$(eval $(call C-PP.JS,$(dir.tmp)/sqlite3-api.c-pp.js,$(sqlite3-api.js))) +$(eval $(call C-PP.JS,$(dir.api)/pre-js.js,$(dir.tmp)/pre-js.js)) +$(eval $(call C-PP.JS,$(extern-post-js.js.in),$(extern-post-js.js))) # /end CPP-of-JS bits ######################################################################## diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index d933a36265..fe9e4392be 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -4,6 +4,9 @@ most of the associated JS code, runs outside of the Emscripten-generated module init scope, in the current global scope. */ +//#if SQLITE_JS_ESM +const toexport = +//#endif (function(){ /** In order to hide the sqlite3InitModule()'s resulting Emscripten @@ -103,4 +106,10 @@ exports["sqlite3InitModule"] = sqlite3InitModule; /* AMD modules get injected in a way we cannot override, so we can't handle those here. */ +//#if SQLITE_JS_ESM + return self.sqlite3InitModule; +//#endif })(); +//#if SQLITE_JS_ESM +export default toexport; +//#endif diff --git a/manifest b/manifest index 8b2b8bd429..c7e27a945f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\suse\sof\scpp\swith\sthe\sfit-to-purpose\sc-pp\sto\savoid\scpp's\sC-centric/JS-unfriendly\squirks. -D 2022-11-18T02:29:59.533 +C More\swork\stowards\screation\sof\sa\sES6\sJS\smodule. +D 2022-11-19T02:51:41.611 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,13 +488,13 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 3c97824ce76c7678344844e581a07ba75901a06def4d0046a2a410d9dd635a83 +F ext/wasm/GNUmakefile 6a0d03e8d0b52b6851a364c1faaa0df8a07be1e8eb8aa9f87432aad74005a04e F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 1350088aee90e959ad9a94fab1bb6bcb5e99d4d27f976db389050f54f2640c78 -F ext/wasm/api/extern-post-js.js c197b7567496fc27766842f8c4f4963054bf8c667926ab3df4c91aa548939ce6 +F ext/wasm/api/extern-post-js.js 824f37f1957e15b150bb36e98621b3bf91b55f6af7055cedc831331129b4883d F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 @@ -2056,8 +2056,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e047b33d1fb7d6a32e967f03f9952249cd2da4d21dc301fe92bd7baa0da5d6a9 -R f08c58c3d67343dbb1484f5dc6d0d557 +P 49d70f071e918d5d095c807575bb7ce2b287a123261e789e938521b3b409429a +R bb5a2de30f99128427e97b8a82e5762a U stephan -Z df00239aed9bf555d5579b7e3a06d443 +Z e18a8c88533141f60b929eb3534d76f0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bef9d42ccd..50b9b44af1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49d70f071e918d5d095c807575bb7ce2b287a123261e789e938521b3b409429a \ No newline at end of file +6b826e700f6849eebfbba38e5948b96be245994e3e03ea30743114d3f5689c42 \ No newline at end of file From 27a67968af60dfa5e2146b618df149c0bcea4997 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 19 Nov 2022 05:26:45 +0000 Subject: [PATCH 018/282] Add build of sqlite3.mjs (ES6 module), add a test app for it, and include it in the dist build. FossilOrigin-Name: 2e783670e10b59e67c14b0db7f4803b41790cc7730de221d54fa2d4483cfba33 --- ext/wasm/GNUmakefile | 229 ++++++++++++++++++------------- ext/wasm/api/extern-post-js.js | 14 +- ext/wasm/api/pre-js.js | 2 +- ext/wasm/api/sqlite3-api-opfs.js | 2 +- ext/wasm/dist.make | 4 +- ext/wasm/fiddle.make | 6 +- ext/wasm/index-dist.html | 4 + ext/wasm/index.html | 5 +- ext/wasm/tester1-esm.html | 30 ++++ ext/wasm/wasmfs.make | 9 +- manifest | 30 ++-- manifest.uuid | 2 +- 12 files changed, 213 insertions(+), 124 deletions(-) create mode 100644 ext/wasm/tester1-esm.html diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 1b85fa5e08..03b7689f2d 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -168,7 +168,6 @@ emcc_opt_full := $(emcc_opt) -g3 # (Much later: -O2 consistently gives the best speeds.) ######################################################################## - $(sqlite3.c) $(sqlite3.h): $(MAKE) -C $(dir.top) sqlite3.c @@ -199,6 +198,48 @@ $(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) $(CC) -o $@ $< DISTCLEAN_FILES += $(bin.stripccomments) + +######################################################################## +# Transform $(1) to $(2) via ./c-pp -f $(1) ... +# +# Historical notes: +# +# - We first attempted to use gcc and/or clang to preprocess JS files +# in the same way we would normally do C files, but C-specific quirks +# of each makes that untennable. +# +# - We implemented c-pp.c (the C-Minus Pre-processor) as a custom +# generic/file-format-agnostic preprocessor to enable us to pack +# code for different target builds into the same JS files. Most +# notably, some ES6 module (a.k.a. ESM) features cannot legally be +# referenced at all in non-ESM code, e.g. the "import" and "export" +# keywords. This preprocessing step permits us to swap out sections +# of code where necessary for ESM and non-ESM (a.k.a. vanilla JS) +# require different implementations. The alternative to such +# preprocessing, would be to have separate source files for ES6 +# builds, which would have a higher maintenance burden than c-pp.c +# seems likely to. +# +# c-pp.c was written specifically for the sqlite project's JavaScript +# builds but is maintained as a standalone project: +# https://fossil.wanderinghorse.net/r/c-pp +bin.c-pp := ./c-pp +$(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) + $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) +define C-PP.JS +# $1 c-pp -D... flags +# $2 = c-pp -f X.js +# $3 = c-pp -o X.js +$(3): $(2) $$(MAKEFILE) $$(bin.c-pp) + $$(bin.c-pp) -f $(2) -o $$@ $(1) +CLEAN_FILES += $(3) +endef +c-pp.D.vanilla ?= +c-pp.D.esm ?= -Dsqlite3-es6-module-build +# /end CPP-of-JS bits +######################################################################## + + EXPORTED_FUNCTIONS.api.in := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api $(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) @@ -234,9 +275,8 @@ $(foreach X,$(SOAP.js) $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js),\ $(eval $(call COPY_XAPI,$(X)))) all: $(sqlite3-api.ext.jses) -sqlite3-api.js := $(dir.tmp)/sqlite3-api.js -sqlite3-api.c-pp.js := $(dir.tmp)/sqlite3-api.c-pp.js -$(sqlite3-api.c-pp.js): $(sqlite3-api.jses) $(MAKEFILE) +sqlite3-api.js.in := $(dir.tmp)/sqlite3-api.c-pp.js +$(sqlite3-api.js.in): $(sqlite3-api.jses) $(MAKEFILE) @echo "Making $@..." @for i in $(sqlite3-api.jses); do \ echo "/* BEGIN FILE: $$i */"; \ @@ -257,28 +297,52 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) ######################################################################## # --post-js and --pre-js are emcc flags we use to append/prepend JS to # the generated emscripten module file. -pre-js.js := $(dir.tmp)/pre-js.js -post-js.js := $(dir.tmp)/post-js.js -post-jses := \ +pre-js.js.in := $(dir.api)/pre-js.js +pre-js.js.esm := $(dir.tmp)/pre-js.esm.js +pre-js.js.vanilla := $(dir.tmp)/pre-js.vanilla.js +$(eval $(call C-PP.JS,$(c-pp.D.vanilla),$(pre-js.js.in),$(pre-js.js.vanilla))) +$(eval $(call C-PP.JS,$(c-pp.D.esm),$(pre-js.js.in),$(pre-js.js.esm))) +post-js.js.in := $(dir.tmp)/post-js.js +post-js.js.vanilla := $(dir.tmp)/post-js.vanilla.js +post-js.js.esm := $(dir.tmp)/post-js.esm.js +post-jses.js := \ $(dir.api)/post-js-header.js \ - $(sqlite3-api.js) \ + $(sqlite3-api.js.in) \ $(dir.api)/post-js-footer.js -$(post-js.js): $(post-jses) $(MAKEFILE) +$(post-js.js.in): $(post-jses.js) $(MAKEFILE) @echo "Making $@..." - @for i in $(post-jses); do \ + @for i in $(post-jses.js); do \ echo "/* BEGIN FILE: $$i */"; \ cat $$i; \ echo "/* END FILE: $$i */"; \ done > $@ +$(eval $(call C-PP.JS,$(c-pp.D.vanilla),$(post-js.js.in),$(post-js.js.vanilla))) +$(eval $(call C-PP.JS,$(c-pp.D.esm),$(post-js.js.in),$(post-js.js.esm))) + extern-post-js.js.in := $(dir.api)/extern-post-js.js -extern-post-js.js := $(dir.tmp)/extern-post-js.js +extern-post-js.js.vanilla := $(dir.tmp)/extern-post-js.vanilla.js +extern-post-js.js.esm := $(dir.tmp)/extern-post-js.esm.js +$(eval $(call C-PP.JS,$(c-pp.D.vanilla),$(extern-post-js.js.in),$(extern-post-js.js.vanilla))) +$(eval $(call C-PP.JS,$(c-pp.D.esm),$(extern-post-js.js.in),$(extern-post-js.js.esm))) extern-pre-js.js := $(dir.api)/extern-pre-js.js + +# Emscripten flags for --[extern-][pre|post]-js=... pre-post-common.flags := \ - --post-js=$(post-js.js) \ - --extern-post-js=$(extern-post-js.js) \ --extern-pre-js=$(sqlite3-license-version.js) -pre-post-jses.deps := $(post-js.js) \ - $(extern-post-js.js) $(extern-pre-js.js) $(sqlite3-license-version.js) +pre-post-common.flags.vanilla := \ + $(pre-post-common.flags) \ + --post-js=$(post-js.js.vanilla) \ + --extern-post-js=$(extern-post-js.js.vanilla) +pre-post-common.flags.esm := \ + $(pre-post-common.flags) \ + --post-js=$(post-js.js.esm) \ + --extern-post-js=$(extern-post-js.js.esm) + +pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) +pre-post-jses.deps.vanilla := $(pre-post-jses.deps.common) \ + $(post-js.js.vanilla) $(extern-post-js.js.vanilla) +pre-post-jses.deps.esm := $(pre-post-jses.deps.common) \ + $(post-js.js.esm) $(extern-post-js.js.esm) $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) $(MAKEFILE) @echo "Making $@..."; { \ cat $(sqlite3-license-version-header.js); \ @@ -290,66 +354,25 @@ $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) echo '*/'; \ } > $@ -######################################################################## -# Transform $(1) to $(2) via ./c-pp -f $(1) ... -# -# Historical notes: -# -# - We first attempted to use gcc and/or clang to preprocess JS files -# in the same way we would normally do C files, but C-specific quirks -# of each makes that untennable. -# -# - We implemented c-pp.c (the C-Minus Pre-processor) as a custom -# generic/file-format-agnostic preprocessor to enable us to pack -# code for different target builds into the same JS files. Most -# notably, some ES6 module (a.k.a. ESM) features cannot legally be -# referenced at all in non-ESM code, e.g. the "import" and "export" -# keywords. This preprocessing step permits us to swap out sections -# of code where necessary for ESM and non-ESM (a.k.a. vanilla JS) -# require different implementations. The alternative to such -# preprocessing, would be to have separate source files for ES6 -# builds, which would have a higher maintenance burden than c-pp.c -# seems likely to. -# -# c-pp.c was written specifically for the sqlite project's JavaScript -# builds but is maintained as a standalone project: -# https://fossil.wanderinghorse.net/r/c-pp -bin.c-pp := ./c-pp -$(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) - $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) -ifneq (,$(filter esm,$(MAKECMDGOALS))) -js.cpp.defines ?= -DSQLITE_JS_ESM -esm: $(filter-out esm,$(MAKECMDGOALS)) -else -js.cpp.defines ?= -endif -define C-PP.JS -# $1 = X.js. $2 = output file to generate by filtering $(1) through -# $(bin.cpp) -E -CC. -$(2): $(1) $$(MAKEFILE) $$(bin.c-pp) - $$(bin.c-pp) $(js.cpp.defines) -f $(1) -o $$@ -CLEAN_FILES += $(2) -endef -$(eval $(call C-PP.JS,$(dir.tmp)/sqlite3-api.c-pp.js,$(sqlite3-api.js))) -$(eval $(call C-PP.JS,$(dir.api)/pre-js.js,$(dir.tmp)/pre-js.js)) -$(eval $(call C-PP.JS,$(extern-post-js.js.in),$(extern-post-js.js))) -# /end CPP-of-JS bits -######################################################################## - ######################################################################## # call-make-pre-js creates rules for pre-js-$(1).js. $1 = the base -# name of the JS file on whose behalf this pre-js is for. +# name of the JS file on whose behalf this pre-js is for. $2 is the +# build mode: one of (vanilla, esm). define call-make-pre-js -pre-post-$(1).flags ?= -$$(dir.tmp)/pre-js-$(1).js: $$(pre-js.js) $$(MAKEFILE) - cp $$(pre-js.js) $$@ +pre-post-$(1).flags.$(2) ?= +$$(dir.tmp)/pre-js-$(1)-$(2).js: $$(pre-js.js.$(2)) $$(MAKEFILE) + cp $$(pre-js.js.$(2)) $$@ @if [ sqlite3-wasmfs = $(1) ]; then \ echo "delete Module[xNameOfInstantiateWasm] /*for WASMFS build*/;"; \ elif [ sqlite3 != $(1) ]; then \ echo "Module[xNameOfInstantiateWasm].uri = '$(1).wasm';"; \ fi >> $$@ -pre-post-$(1).deps := $$(pre-post-jses.deps) $$(dir.tmp)/pre-js-$(1).js -pre-post-$(1).flags += --pre-js=$$(dir.tmp)/pre-js-$(1).js +pre-post-$(1).deps.$(2) := \ + $$(pre-post-jses.deps.$(2)) \ + $$(dir.tmp)/pre-js-$(1)-$(2).js +pre-post-$(1).flags.$(2) += \ + $$(pre-post-common.flags.$(2)) \ + --pre-js=$$(dir.tmp)/pre-js-$(1)-$(2).js endef #$(error $(call call-make-pre-js,sqlite3-wasmfs)) # /post-js and pre-js @@ -466,20 +489,11 @@ emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) # debugging info, _huge_. ######################################################################## -######################################################################## -# AN EXPERIMENT: undocumented Emscripten feature: if the target file -# extension is "mjs", it defaults to ES6 module builds: +sqlite3.js := $(dir.dout)/sqlite3.js +sqlite3.mjs := $(dir.dout)/sqlite3.mjs +# Undocumented Emscripten feature: if the target file extension is +# "mjs", it defaults to ES6 module builds: # https://github.com/emscripten-core/emscripten/issues/14383 -ifeq (,$(filter esm,$(MAKECMDGOALS))) -sqlite3.js.ext := js -else -esm.deps := $(filter-out esm,$(MAKECMDGOALS)) -esm: $(if $(esm.deps),$(esm.deps),all) -sqlite3.js.ext := mjs -endif -# /esm -######################################################################## -sqlite3.js := $(dir.dout)/sqlite3.$(sqlite3.js.ext) sqlite3.wasm := $(dir.dout)/sqlite3.wasm sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c # sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter @@ -487,22 +501,52 @@ sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c # enough to the target speed requirements that the 500ms makes a # difference. Thus we build all binaries against sqlite3-wasm.c # instead of building a shared copy of sqlite3-wasm.o. -$(eval $(call call-make-pre-js,sqlite3)) +$(eval $(call call-make-pre-js,sqlite3,vanilla)) +$(eval $(call call-make-pre-js,sqlite3,esm)) $(sqlite3.js): -$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \ - $(EXPORTED_FUNCTIONS.api) \ - $(pre-post-sqlite3.deps) +$(sqlite3.js) $(sqlite3.mjs): $(MAKEFILE) $(sqlite3.wasm.obj) \ + $(EXPORTED_FUNCTIONS.api) +$(sqlite3.js): $(pre-post-sqlite3.deps.vanilla) +$(sqlite3.mjs): $(pre-post-sqlite3.deps.esm) +# SQLITE3.xJS.RECIPE = Recipe body for $(sqlite3.js) and +# $(sqlite3.mjs). $1 = one of (vanilla, esm). +define SQLITE3.xJS.RECIPE @echo "Building $@ ..." $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ - $(emcc.jsflags) $(pre-post-common.flags) $(pre-post-sqlite3.flags) \ + $(emcc.jsflags) \ + $(pre-post-sqlite3.flags.$(1)) $(emcc.flags.sqlite3.$(1)) \ $(cflags.common) $(SQLITE_OPT) $(sqlite3-wasm.c) + if [ esm = $(1) ]; then \ + sed -i -e '0,/^export default/{/^export default/d}' $@; \ + fi # work around an Emscripten annoyance. See emcc.flags.esm chmod -x $(sqlite3.wasm) $(maybe-wasm-strip) $(sqlite3.wasm) @ls -la $@ $(sqlite3.wasm) +endef +emcc.flags.sqlite3.vanilla := +emcc.flags.sqlite3.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META +# Reminder: even if we use -sEXPORT_ES6=0, emcc _still_ adds: +# +# export default $(sqlite3.js.init-func); +# +# when building *.mjs, which is bad because we need to export an +# overwritten version of that function and cannot "export default" +# twice. Because of this, we have to sed $(sqlite3.mjs) to remove the +# first instance of /^export default/. +$(sqlite3.js): + $(call SQLITE3.xJS.RECIPE,vanilla) +$(sqlite3.mjs): + $(call SQLITE3.xJS.RECIPE,esm) $(sqlite3.wasm): $(sqlite3.js) -CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm) -all: $(sqlite3.js) -wasm: $(sqlite3.js) +$(sqlite3.mjs): $(sqlite3.js) +# We have to ensure that we do not build both $(sqlite3.js) and +# $(sqlite3.mjs) in parallel because both result in the build of +# $(sqlite3.wasm). We have no way to build just the .mjs file without +# also building the .wasm file. i.e. we're building $(sqlite3.wasm) +# twice, but that's unavoidable. +CLEAN_FILES += $(sqlite3.js) $(sqlite3.mjs) $(sqlite3.wasm) +all: $(sqlite3.mjs) +wasm: $(sqlite3.mjs) # End main Emscripten-based module build ######################################################################## @@ -552,7 +596,6 @@ speedtest1-common.eflags += -sDYNAMIC_EXECUTION=0 speedtest1-common.eflags += --minify 0 speedtest1-common.eflags += -sEXPORT_NAME=$(sqlite3.js.init-func) speedtest1-common.eflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) -speedtest1-common.eflags += $(pre-post-common.flags) speedtest1.exit-runtime0 := -sEXIT_RUNTIME=0 speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 # Re -sEXIT_RUNTIME=1 vs 0: if it's 1 and speedtest1 crashes, we get @@ -576,17 +619,17 @@ $(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) @echo "Making $@ ..." @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ speedtest1.js := $(dir.dout)/speedtest1.js -speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js)) +speedtest1.wasm := $(dir.dout)/speedtest1.wasm speedtest1.cflags := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.c) -$(eval $(call call-make-pre-js,speedtest1)) +$(eval $(call call-make-pre-js,speedtest1,vanilla)) $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ - $(pre-post-speedtest1.deps) \ + $(pre-post-speedtest1.deps.vanilla) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ $(speedtest1.eflags) $(speedtest1-common.eflags) $(speedtest1.cflags) \ - $(pre-post-speedtest1.flags) \ + $(pre-post-speedtest1.flags.vanilla) \ $(SQLITE_OPT) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cses) -lm diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index fe9e4392be..3dc61ae050 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -1,11 +1,15 @@ + +/* ^^^^ ACHTUNG: blank line at the start is necessary because + Emscripten will not add a newline in some cases and we need + a blank line for a sed-based kludge for the ES6 build. */ /* extern-post-js.js must be appended to the resulting sqlite3.js file. It gets its name from being used as the value for the --extern-post-js=... Emscripten flag. Note that this code, unlike most of the associated JS code, runs outside of the Emscripten-generated module init scope, in the current global scope. */ -//#if SQLITE_JS_ESM -const toexport = +//#if sqlite3-es6-module-build +const toExportForES6 = //#endif (function(){ /** @@ -106,10 +110,10 @@ const toexport = exports["sqlite3InitModule"] = sqlite3InitModule; /* AMD modules get injected in a way we cannot override, so we can't handle those here. */ -//#if SQLITE_JS_ESM +//#if sqlite3-es6-module-build return self.sqlite3InitModule; //#endif })(); -//#if SQLITE_JS_ESM -export default toexport; +//#if sqlite3-es6-module-build +export default toExportForES6; //#endif diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.js index c6d0683ff4..41d13d6853 100644 --- a/ext/wasm/api/pre-js.js +++ b/ext/wasm/api/pre-js.js @@ -29,7 +29,7 @@ sqlite3InitModuleState.debugModule('self.location =',self.location); 4) If none of the above apply, (prefix+path) is returned. */ Module['locateFile'] = function(path, prefix) { -//#if SQLITE_JS_ESM +//#if sqlite3-es6-module-build return new URL(path, import.meta.url).href; //#else 'use strict'; diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index e35eed64d1..31fe45eda0 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -167,7 +167,7 @@ const installOpfsVfs = function callee(options){ return promiseReject_(err); }; const W = -//#if SQLITE_JS_ESM +//#if sqlite3-es6-module-build new Worker(new URL(options.proxyUri, import.meta.url)); //#else new Worker(options.proxyUri); diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index 5aee8af779..07d289ddbf 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -42,7 +42,7 @@ dist-dir.jswasm := $(dist-dir.top)/$(notdir $(dir.dout)) dist-dir.common := $(dist-dir.top)/common dist.top.extras := \ demo-123.html demo-123-worker.html demo-123.js \ - tester1.html tester1-worker.html tester1.js \ + tester1.html tester1-worker.html tester1-esm.html tester1.js \ demo-jsstorage.html demo-jsstorage.js \ demo-worker1.html demo-worker1.js \ demo-worker1-promiser.html demo-worker1-promiser.js @@ -78,6 +78,8 @@ dist: \ @cp -p $(dist.jswasm.extras) $(dist-dir.jswasm) @$(bin.stripccomments) -k -k < $(sqlite3.js) \ > $(dist-dir.jswasm)/$(notdir $(sqlite3.js)) + @$(bin.stripccomments) -k -k < $(sqlite3.mjs) \ + > $(dist-dir.jswasm)/$(notdir $(sqlite3.mjs)) @cp -p $(dist.common.extras) $(dist-dir.common) @set -e; \ vnum=$$($(bin.version-info) --download-version); \ diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 43e6941f55..71602114a9 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -58,12 +58,12 @@ fiddle.SOAP.js := $(dir.fiddle)/$(notdir $(SOAP.js)) $(fiddle.SOAP.js): $(SOAP.js) cp $< $@ -$(eval $(call call-make-pre-js,fiddle-module)) +$(eval $(call call-make-pre-js,fiddle-module,vanilla)) $(fiddle-module.js): $(MAKEFILE) $(MAKEFILE.fiddle) \ $(EXPORTED_FUNCTIONS.fiddle) \ - $(fiddle.cses) $(pre-post-fiddle-module.deps) $(fiddle.SOAP.js) + $(fiddle.cses) $(pre-post-fiddle-module.deps.vanilla) $(fiddle.SOAP.js) $(emcc.bin) -o $@ $(fiddle.emcc-flags) \ - $(pre-post-common.flags) $(pre-post-fiddle-module.flags) \ + $(pre-post-fiddle-module.flags.vanilla) \ $(fiddle.cses) $(maybe-wasm-strip) $(fiddle-module.wasm) gzip < $@ > $@.gz diff --git a/ext/wasm/index-dist.html b/ext/wasm/index-dist.html index 6b038c82a9..2333190d95 100644 --- a/ext/wasm/index-dist.html +++ b/ext/wasm/index-dist.html @@ -56,6 +56,10 @@ utility code.
  • tester1-worker: same thing but running in a Worker.
    • +
  • tester1-esm: same thing + but loaded in the main thread via an ES6 module. Note that + not all browsers permit loading modules in Worker threads. +
  • Higher-level apps and demos... diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 044cd1360d..0aca0661c2 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -46,7 +46,10 @@ regression tests for the various APIs and surrounding utility code.
  • tester1-worker: same thing - but running in a Worker.
    • + but running in a Worker. +
  • tester1-esm: same thing + but loaded in the main thread via an ES6 module. Note that + not all browsers permit loading modules in Worker threads.
  • High-level apps and demos... diff --git a/ext/wasm/tester1-esm.html b/ext/wasm/tester1-esm.html new file mode 100644 index 0000000000..dc63f7cbe6 --- /dev/null +++ b/ext/wasm/tester1-esm.html @@ -0,0 +1,30 @@ + + + + + + + + + sqlite3 tester ESM #1 (UI thread) + + + +

    sqlite3 WASM/JS tester ESM #1 (UI thread)

    +
    + + +
    +
    + + + + diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make index 81b41870c8..33f812aac6 100644 --- a/ext/wasm/wasmfs.make +++ b/ext/wasm/wasmfs.make @@ -67,14 +67,17 @@ sqlite3-wasmfs.jsflags += $(sqlite3-wasmfs.fsflags) # USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, # see https://github.com/WebAssembly/design/issues/1271 [-Wpthreads-mem-growth] sqlite3-wasmfs.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) -$(eval $(call call-make-pre-js,sqlite3-wasmfs)) -sqlite3-wasmfs.jsflags += $(pre-post-common.flags) $(pre-post-sqlite3-wasmfs.flags) +$(eval $(call call-make-pre-js,sqlite3-wasmfs,vanilla)) +sqlite3-wasmfs.jsflags += \ + $(pre-post-common.flags.vanilla) \ + $(pre-post-sqlite3-wasmfs.flags.vanilla) $(sqlite3-wasmfs.js): $(sqlite3-wasm.c) \ $(EXPORTED_FUNCTIONS.api) $(MAKEFILE) $(MAKEFILE.wasmfs) \ - $(pre-post-sqlite3-wasmfs.deps) + $(pre-post-sqlite3-wasmfs.deps.vanilla) @echo "Building $@ ..." $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ $(sqlite3-wasmfs.cflags) $(sqlite3-wasmfs.jsflags) \ + $(pre-post-sqlite3-wasm.flags.vanilla) \ $(sqlite3-wasm.c) chmod -x $(sqlite3-wasmfs.wasm) $(maybe-wasm-strip) $(sqlite3-wasmfs.wasm) diff --git a/manifest b/manifest index 8267d53859..3df90b2d8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Integrate\sa\scustom\spreprocessor\sto\sthe\sJS\sbuild\sprocess\sto\sfacilitate\screation\sof\sboth\svanilla\sJS\sand\sES6\sModule\sbuilds\sfrom\sthe\ssame\ssource\sfiles.\sThere\sis\sstill\ssome\sbuild-level\sreworking\spending\sto\smake\san\sESM\sbuild\sa\sfirst-class\sdeliverable. -D 2022-11-19T02:58:03.867 +C Add\sbuild\sof\ssqlite3.mjs\s(ES6\smodule),\sadd\sa\stest\sapp\sfor\sit,\sand\sinclude\sit\sin\sthe\sdist\sbuild. +D 2022-11-19T05:26:45.763 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,21 +488,21 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 6a0d03e8d0b52b6851a364c1faaa0df8a07be1e8eb8aa9f87432aad74005a04e +F ext/wasm/GNUmakefile 1e38a4f7147d621bd2138d13938ef34157bcf47908325baa6b06cd02c5e3ef89 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 29276a845e57004e82efba61fa5866fd05f9137380a1dc26dc4c6d65264cd81c -F ext/wasm/api/extern-post-js.js 824f37f1957e15b150bb36e98621b3bf91b55f6af7055cedc831331129b4883d +F ext/wasm/api/extern-post-js.js 015121df2c903cf12d51507227b756ab3626036d8e9d610a2a2c15b3f54afe4d F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 -F ext/wasm/api/pre-js.js 749bbbac2f1a2192eaba80cf5e00a3219da78b3c0a84e3019b5eef30e0bc9b88 +F ext/wasm/api/pre-js.js 1156a7fb9de817bb1cb39ad90b76aa93fbb9dcf950a1f2d6f547e5976872be36 F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js 4368a30586df3e11339a72082c77bdef670d619c6185c2dd1ebf5208c3ab0a5c +F ext/wasm/api/sqlite3-api-opfs.js 9f115a37dafe8067bce8812996d2deff45741c6e39f7aad7b48f5fbbd822dba5 F ext/wasm/api/sqlite3-api-prologue.js fd526fa017fa2578673ca18158354515c719e719a5d93f2f6d0e43f39170430e F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -527,14 +527,14 @@ F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98 F ext/wasm/demo-worker1-promiser.js b85a2bb1b918db4f09dfa24419241cb3edad7791389425c2505092e9b715017d F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa -F ext/wasm/dist.make 481289899a07958439d07ee4302ff86235fa0fbb72f17ea05db2be90a94abf90 -F ext/wasm/fiddle.make e570ec1bfc7d803507a2e514fe32f673fe001b2114b85c73c3964a462ba8bcfc +F ext/wasm/dist.make 4b55c8a7926bbab4936adab6a08eca524085fc47bc3b08f41918df5b4665da3d +F ext/wasm/fiddle.make 2812c44c9bafb5be9c8767963d1b9f374d77af7795fcaa06483c03e7059dea74 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 -F ext/wasm/index-dist.html cb0da16cba0f21cda2c25724c5869102d48eb0af04446acd3cd0ca031f80ed19 -F ext/wasm/index.html ce6a68a75532b47e3c0adb83381a06d15de8c0ac0331fb7bf31d33f8e7c77dc4 +F ext/wasm/index-dist.html 6bfb3591e40f7c23626730df587f533e983e996d4d1fb67244fb6a88fe6cf9a6 +F ext/wasm/index.html 49f58dddc29f6394b6e8a93e42768de59380c258454b68b9182e1946d13a4a4b F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e F ext/wasm/module-symbols.html b8eebafef8e536624bbe5f7a3da40c07a9062b843dfd3161a0bb72cbb6763dc5 @@ -549,11 +549,12 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac +F ext/wasm/tester1-esm.html 8d226a21b20707dbd66d68a3990141f0392fc781a281291d3dc59f38a3555887 F ext/wasm/tester1-worker.html 51bf39e2b87f974ae3d5bc3086e2fb36d258f3698c54f6e21ba4b3b99636fa27 F ext/wasm/tester1.html 624ec41cd9f78a1f2b6d7df70aaa7a6394396b1f2455ecbd6de5775c1275b121 F ext/wasm/tester1.js bff806de454de115922d78c056f11d523ec7ed9ed3839d4e21433a9f72558b88 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 -F ext/wasm/wasmfs.make edfd60691d10fd19ada4c061280fd7fbe4cf5f6bf6b913268e8ebedfccea6ab5 +F ext/wasm/wasmfs.make 8aa7565f9de8dd3c291ad8c3ceb1a2c67a3eb31a8e531070b25c6c6b1f0278bf F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 @@ -2056,9 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff 6b826e700f6849eebfbba38e5948b96be245994e3e03ea30743114d3f5689c42 -R a5784f7e95f9f4ca471c3d20c7b7d6ea -T +closed 6b826e700f6849eebfbba38e5948b96be245994e3e03ea30743114d3f5689c42 Closed\sby\sintegrate-merge. +P 10c723d96d61d2e552ec1102563d58f1eb11bc3d30e03606fd8e0279c5a9043a +R 222130501cabad70ae0d5e4c47519919 U stephan -Z 22024ccae1db68511ad1978bbc10208a +Z bbe51f75fec2c52a531f67e9a6ac36f6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4b46dcbad7..78bd7a46af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -10c723d96d61d2e552ec1102563d58f1eb11bc3d30e03606fd8e0279c5a9043a \ No newline at end of file +2e783670e10b59e67c14b0db7f4803b41790cc7730de221d54fa2d4483cfba33 \ No newline at end of file From bf9b994f52f076fa955c2d9aabe1c6970f30d205 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 19 Nov 2022 13:09:03 +0000 Subject: [PATCH 019/282] Streamline the decodeFlags() routine in btree.c for a small performance increase. FossilOrigin-Name: 4cb285210b4a2b14c80962bf2ecb35be310d3444c329c15d86b3073096455704 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/btree.c | 85 +++++++++++++++++++++++++++------------------------ 3 files changed, 53 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 3df90b2d8a..4507d9d171 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sbuild\sof\ssqlite3.mjs\s(ES6\smodule),\sadd\sa\stest\sapp\sfor\sit,\sand\sinclude\sit\sin\sthe\sdist\sbuild. -D 2022-11-19T05:26:45.763 +C Streamline\sthe\sdecodeFlags()\sroutine\sin\sbtree.c\sfor\sa\ssmall\sperformance\nincrease. +D 2022-11-19T13:09:03.004 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -578,7 +578,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c e8fae9a95ea9561aebc41e467a9ee9ba9150ca373031e65773d62ff02d8250d2 +F src/btree.c 405994e58bdf8f58adb5b238cda4b428de2bfe67e8dac5577082f2fd70faeade F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 10c723d96d61d2e552ec1102563d58f1eb11bc3d30e03606fd8e0279c5a9043a -R 222130501cabad70ae0d5e4c47519919 -U stephan -Z bbe51f75fec2c52a531f67e9a6ac36f6 +P 2e783670e10b59e67c14b0db7f4803b41790cc7730de221d54fa2d4483cfba33 +R b0bb099cec1bed805505fb24b4006305 +U drh +Z 40c7b0608af0db932c6182b2b48b07d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 78bd7a46af..e953848055 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e783670e10b59e67c14b0db7f4803b41790cc7730de221d54fa2d4483cfba33 \ No newline at end of file +4cb285210b4a2b14c80962bf2ecb35be310d3444c329c15d86b3073096455704 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 1c6c14d4b6..569b596c57 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1900,62 +1900,67 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ ** Only the following combinations are supported. Anything different ** indicates a corrupt database files: ** -** PTF_ZERODATA -** PTF_ZERODATA | PTF_LEAF -** PTF_LEAFDATA | PTF_INTKEY -** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF +** PTF_ZERODATA (0x02, 2) +** PTF_LEAFDATA | PTF_INTKEY (0x05, 5) +** PTF_ZERODATA | PTF_LEAF (0x0a, 10) +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF (0x0d, 13) */ static int decodeFlags(MemPage *pPage, int flagByte){ BtShared *pBt; /* A copy of pPage->pBt */ assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); - flagByte &= ~PTF_LEAF; - pPage->childPtrSize = 4-4*pPage->leaf; pBt = pPage->pBt; - if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an - ** interior table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); - /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a - ** leaf table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); - pPage->intKey = 1; - if( pPage->leaf ){ + pPage->max1bytePayload = pBt->max1bytePayload; + if( flagByte>=(PTF_ZERODATA | PTF_LEAF) ){ + pPage->childPtrSize = 0; + pPage->leaf = 1; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF) ){ pPage->intKeyLeaf = 1; pPage->xCellSize = cellSizePtrTableLeaf; pPage->xParseCell = btreeParseCellPtr; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==(PTF_ZERODATA | PTF_LEAF) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + return SQLITE_CORRUPT_PAGE(pPage); + } + }else{ + pPage->childPtrSize = 4; + pPage->leaf = 0; + if( flagByte==(PTF_ZERODATA) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + }else if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ pPage->intKeyLeaf = 0; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + return SQLITE_CORRUPT_PAGE(pPage); } - pPage->maxLocal = pBt->maxLeaf; - pPage->minLocal = pBt->minLeaf; - }else if( flagByte==PTF_ZERODATA ){ - /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an - ** interior index b-tree page. */ - assert( (PTF_ZERODATA)==2 ); - /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a - ** leaf index b-tree page. */ - assert( (PTF_ZERODATA|PTF_LEAF)==10 ); - pPage->intKey = 0; - pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; - pPage->xParseCell = btreeParseCellPtrIndex; - pPage->maxLocal = pBt->maxLocal; - pPage->minLocal = pBt->minLocal; - }else{ - /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is - ** an error. */ - pPage->intKey = 0; - pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; - pPage->xParseCell = btreeParseCellPtrIndex; - return SQLITE_CORRUPT_PAGE(pPage); } - pPage->max1bytePayload = pBt->max1bytePayload; return SQLITE_OK; } From 1e62057436a904f02e710adb264f539c51a0b66c Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 19 Nov 2022 14:18:48 +0000 Subject: [PATCH 020/282] Improved comments on the pageFreeArray() routine of btree.c. No changes to code. FossilOrigin-Name: 0c2adc6d3547b07e950ae49f07f688f71a21b3ad5a51f16f0e8d49ab91564582 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4507d9d171..9d1924cb8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Streamline\sthe\sdecodeFlags()\sroutine\sin\sbtree.c\sfor\sa\ssmall\sperformance\nincrease. -D 2022-11-19T13:09:03.004 +C Improved\scomments\son\sthe\spageFreeArray()\sroutine\sof\sbtree.c.\s\sNo\schanges\sto\ncode. +D 2022-11-19T14:18:48.226 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -578,7 +578,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 405994e58bdf8f58adb5b238cda4b428de2bfe67e8dac5577082f2fd70faeade +F src/btree.c c98cf491cbb8aa930bc9deb2cf794d2a32005c97c9cde01b7b8a10ef7c3d8451 F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2e783670e10b59e67c14b0db7f4803b41790cc7730de221d54fa2d4483cfba33 -R b0bb099cec1bed805505fb24b4006305 +P 4cb285210b4a2b14c80962bf2ecb35be310d3444c329c15d86b3073096455704 +R 08dfc1f1d36db05c00bfb06ac1037121 U drh -Z 40c7b0608af0db932c6182b2b48b07d6 +Z 7815819acfb2ddb8c0f5e3f3b0dc41f8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e953848055..b2733321bc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4cb285210b4a2b14c80962bf2ecb35be310d3444c329c15d86b3073096455704 \ No newline at end of file +0c2adc6d3547b07e950ae49f07f688f71a21b3ad5a51f16f0e8d49ab91564582 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 569b596c57..33762cc906 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7448,8 +7448,8 @@ static int pageFreeArray( int nRet = 0; int i; int iEnd = iFirst + nCell; - u8 *pFree = 0; - int szFree = 0; + u8 *pFree = 0; /* \__ Parameters for pending call to */ + int szFree = 0; /* / freeSpace() */ for(i=iFirst; iapCell[i]; @@ -7470,6 +7470,9 @@ static int pageFreeArray( return 0; } }else{ + /* The current cell is adjacent to and before the pFree cell. + ** Combine the two regions into one to reduce the number of calls + ** to freeSpace(). */ pFree = pCell; szFree += sz; } From 8d0d409876b2bfb6ebe6163ebedefeb8fb9b337f Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 19 Nov 2022 16:16:40 +0000 Subject: [PATCH 021/282] Get tester1.js working via an ES6 worker module and add that variant to the dist zipfile. FossilOrigin-Name: 90480586f1b2ad82118e19536b095431b8457f294c0afaa9b4f883f184cc804c --- ext/wasm/GNUmakefile | 36 ++++++++++++++++-------- ext/wasm/dist.make | 3 +- ext/wasm/index-dist.html | 12 ++++++-- ext/wasm/index.html | 13 +++++++-- ext/wasm/tester1-esm.html | 4 +-- ext/wasm/tester1-worker.html | 19 +++++++++++-- ext/wasm/{tester1.js => tester1.c-pp.js} | 25 ++++++++++++++-- manifest | 26 ++++++++--------- manifest.uuid | 2 +- 9 files changed, 99 insertions(+), 41 deletions(-) rename ext/wasm/{tester1.js => tester1.c-pp.js} (99%) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 03b7689f2d..3d952c2615 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -227,12 +227,13 @@ bin.c-pp := ./c-pp $(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) define C-PP.JS -# $1 c-pp -D... flags -# $2 = c-pp -f X.js -# $3 = c-pp -o X.js -$(3): $(2) $$(MAKEFILE) $$(bin.c-pp) - $$(bin.c-pp) -f $(2) -o $$@ $(1) -CLEAN_FILES += $(3) +# Create $2 from $1 using $(bin.c-pp) +# $1 = Input file: c-pp -f X.js +# $2 = Output file: c-pp -o X.js +# $3 = optional c-pp -D... flags +$(2): $(1) $$(MAKEFILE) $$(bin.c-pp) + $$(bin.c-pp) -f $(1) -o $$@ $(3) +CLEAN_FILES += $(2) endef c-pp.D.vanilla ?= c-pp.D.esm ?= -Dsqlite3-es6-module-build @@ -300,8 +301,8 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) pre-js.js.in := $(dir.api)/pre-js.js pre-js.js.esm := $(dir.tmp)/pre-js.esm.js pre-js.js.vanilla := $(dir.tmp)/pre-js.vanilla.js -$(eval $(call C-PP.JS,$(c-pp.D.vanilla),$(pre-js.js.in),$(pre-js.js.vanilla))) -$(eval $(call C-PP.JS,$(c-pp.D.esm),$(pre-js.js.in),$(pre-js.js.esm))) +$(eval $(call C-PP.JS,$(pre-js.js.in),$(pre-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.JS,$(pre-js.js.in),$(pre-js.js.esm),$(c-pp.D.esm))) post-js.js.in := $(dir.tmp)/post-js.js post-js.js.vanilla := $(dir.tmp)/post-js.vanilla.js post-js.js.esm := $(dir.tmp)/post-js.esm.js @@ -316,14 +317,14 @@ $(post-js.js.in): $(post-jses.js) $(MAKEFILE) cat $$i; \ echo "/* END FILE: $$i */"; \ done > $@ -$(eval $(call C-PP.JS,$(c-pp.D.vanilla),$(post-js.js.in),$(post-js.js.vanilla))) -$(eval $(call C-PP.JS,$(c-pp.D.esm),$(post-js.js.in),$(post-js.js.esm))) +$(eval $(call C-PP.JS,$(post-js.js.in),$(post-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.JS,$(post-js.js.in),$(post-js.js.esm),$(c-pp.D.esm))) extern-post-js.js.in := $(dir.api)/extern-post-js.js extern-post-js.js.vanilla := $(dir.tmp)/extern-post-js.vanilla.js extern-post-js.js.esm := $(dir.tmp)/extern-post-js.esm.js -$(eval $(call C-PP.JS,$(c-pp.D.vanilla),$(extern-post-js.js.in),$(extern-post-js.js.vanilla))) -$(eval $(call C-PP.JS,$(c-pp.D.esm),$(extern-post-js.js.in),$(extern-post-js.js.esm))) +$(eval $(call C-PP.JS,$(extern-post-js.js.in),$(extern-post-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.JS,$(extern-post-js.js.in),$(extern-post-js.js.esm),$(c-pp.D.esm))) extern-pre-js.js := $(dir.api)/extern-pre-js.js # Emscripten flags for --[extern-][pre|post]-js=... @@ -642,6 +643,17 @@ CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) # end speedtest1.js ######################################################################## +######################################################################## +# tester1 code: +# tester1.js: for main thread and worker thread +# tester1-esm.js: to be loaded from an ES6 Worker Module thread +$(eval $(call C-PP.JS,tester1.c-pp.js,tester1.js)) +$(eval $(call C-PP.JS,tester1.c-pp.js,tester1-esm.js,-Dtester1-esm-worker)) +tester1.js: $(sqlite3.js) +tester1-esm.js: $(sqlite3.mjs) +tester1: tester1.js tester1-esm.js +all: tester1 + ######################################################################## # Convenience rules to rebuild with various -Ox levels. Much # experimentation shows -O2 to be the clear winner in terms of speed. diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index 07d289ddbf..f33711a19d 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -42,7 +42,8 @@ dist-dir.jswasm := $(dist-dir.top)/$(notdir $(dir.dout)) dist-dir.common := $(dist-dir.top)/common dist.top.extras := \ demo-123.html demo-123-worker.html demo-123.js \ - tester1.html tester1-worker.html tester1-esm.html tester1.js \ + tester1.html tester1-worker.html tester1-esm.html \ + tester1.js tester1-esm.js \ demo-jsstorage.html demo-jsstorage.js \ demo-worker1.html demo-worker1.js \ demo-worker1-promiser.html demo-worker1-promiser.js diff --git a/ext/wasm/index-dist.html b/ext/wasm/index-dist.html index 2333190d95..29891c91e9 100644 --- a/ext/wasm/index-dist.html +++ b/ext/wasm/index-dist.html @@ -56,9 +56,15 @@ utility code.
  • tester1-worker: same thing but running in a Worker.
    • -
  • tester1-esm: same thing - but loaded in the main thread via an ES6 module. Note that - not all browsers permit loading modules in Worker threads. +
  • tester1-esm: same as + tester1 but loads sqlite3 in the main thread via + an ES6 module. +
    • +
  • tester1-worker?esm: + same as tester1-esm but loads a Worker Module which + then loads the sqlite3 API via an ES6 module. Note that + not all browsers permit loading modules in Worker + threads.
    • diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 0aca0661c2..0a6cd0a620 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -47,9 +47,16 @@ utility code.
  • tester1-worker: same thing but running in a Worker.
    • -
  • tester1-esm: same thing - but loaded in the main thread via an ES6 module. Note that - not all browsers permit loading modules in Worker threads. +
  • tester1-esm: same as + tester1 but loads sqlite3 in the main thread via + an ES6 module. +
    • +
  • tester1-worker?esm: + same as tester1-esm but loads a Worker Module which + then loads the sqlite3 API via an ES6 module. Note that + not all browsers permit loading modules in Worker + threads. +
  • High-level apps and demos... diff --git a/ext/wasm/tester1-esm.html b/ext/wasm/tester1-esm.html index dc63f7cbe6..118ab676bd 100644 --- a/ext/wasm/tester1-esm.html +++ b/ext/wasm/tester1-esm.html @@ -6,7 +6,7 @@ - sqlite3 tester ESM #1 (UI thread) + sqlite3 tester #1: ES6 Module in UI thread -

    sqlite3 WASM/JS tester ESM #1 (UI thread)

    +

    sqlite3 tester #1: ES6 Module in UI thread

    diff --git a/ext/wasm/tester1-worker.html b/ext/wasm/tester1-worker.html index 4d2df0c8d1..b750df9169 100644 --- a/ext/wasm/tester1-worker.html +++ b/ext/wasm/tester1-worker.html @@ -6,7 +6,7 @@ - sqlite3 tester #1 (Worker thread) + sqlite3 tester #1: Worker thread -

    sqlite3 WASM/JS tester #1 (Worker thread)

    +

    sqlite3 tester #1: Worker thread

    See tester1.html for the UI-thread variant.
    @@ -40,7 +40,20 @@ }; cbReverse.addEventListener('change',cbReverseIt,true); cbReverseIt(); - const w = new Worker("tester1.js?sqlite3.dir=jswasm"); + const urlParams = new URL(self.location.href).searchParams; + const workerArgs = []; + if(urlParams.has('esm')){ + logHtml('warning',"Attempting to run an ES6 Worker Module, "+ + "which is not supported by all browsers! "+ + "e.g. Firefox (as of 2022-11) cannot do this."); + workerArgs.push("tester1-esm.js",{type:"module"}); + document.querySelectorAll('title,#color-target').forEach((e)=>{ + e.innerText = "sqlite3 tester #1: ES6 Worker Module"; + }); + }else{ + workerArgs.push("tester1.js?sqlite3.dir=jswasm"); + } + const w = new Worker(...workerArgs); w.onmessage = function({data}){ switch(data.type){ case 'log': diff --git a/ext/wasm/tester1.js b/ext/wasm/tester1.c-pp.js similarity index 99% rename from ext/wasm/tester1.js rename to ext/wasm/tester1.c-pp.js index 99fb5b3184..7aa8ba4a65 100644 --- a/ext/wasm/tester1.js +++ b/ext/wasm/tester1.c-pp.js @@ -29,8 +29,25 @@ a db in an early test and close it in a later test. Each test gets passed the sqlite3 namespace object as its only argument. */ +/* + This file is intended to be processed by c-pp to inject (or not) + code specific to ES6 modules which is illegal in non-module code. + + Non-ES6 module build and ES6 module for the main-thread: + + ./c-pp -f tester1.c-pp.js -o tester1.js + + ES6 worker module build: + + ./c-pp -f tester1.c-pp.js -o tester1-esm.js -Dtester1-esm-worker +*/ +//#if tester1-esm-worker +import {default as sqlite3InitModule} from './jswasm/sqlite3.mjs'; +self.sqlite3InitModule = sqlite3InitModule; +//#else 'use strict'; -(function(){ +//#endif +(function(self){ /** Set up our output channel differently depending on whether we are running in a worker thread or @@ -1817,7 +1834,8 @@ //////////////////////////////////////////////////////////////////////// log("Loading and initializing sqlite3 WASM module..."); - if(!isUIThread()){ + if(!self.sqlite3InitModule && !isUIThread()){ + /* Vanilla worker, as opposed to an ES6 module worker */ /* If sqlite3.js is in a directory other than this script, in order to get sqlite3.js to resolve sqlite3.wasm properly, we have to @@ -1861,4 +1879,5 @@ } TestUtil.runTests(sqlite3); }); -})(); +})(self); + diff --git a/manifest b/manifest index 9d1924cb8e..40a5b2d04c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\son\sthe\spageFreeArray()\sroutine\sof\sbtree.c.\s\sNo\schanges\sto\ncode. -D 2022-11-19T14:18:48.226 +C Get\stester1.js\sworking\svia\san\sES6\sworker\smodule\sand\sadd\sthat\svariant\sto\sthe\sdist\szipfile. +D 2022-11-19T16:16:40.998 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,7 +488,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 1e38a4f7147d621bd2138d13938ef34157bcf47908325baa6b06cd02c5e3ef89 +F ext/wasm/GNUmakefile 616c2a3e944483f6f5f66a6f77885eb7a7d3fef1a599014d746a792b6ef7b6b6 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 @@ -527,14 +527,14 @@ F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98 F ext/wasm/demo-worker1-promiser.js b85a2bb1b918db4f09dfa24419241cb3edad7791389425c2505092e9b715017d F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa -F ext/wasm/dist.make 4b55c8a7926bbab4936adab6a08eca524085fc47bc3b08f41918df5b4665da3d +F ext/wasm/dist.make d00562b499096704304d9879f834a76475c8dd7ac58aaa3f039625ecc299aa3d F ext/wasm/fiddle.make 2812c44c9bafb5be9c8767963d1b9f374d77af7795fcaa06483c03e7059dea74 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 -F ext/wasm/index-dist.html 6bfb3591e40f7c23626730df587f533e983e996d4d1fb67244fb6a88fe6cf9a6 -F ext/wasm/index.html 49f58dddc29f6394b6e8a93e42768de59380c258454b68b9182e1946d13a4a4b +F ext/wasm/index-dist.html c4337617c4d6d4d0796827cec28ac81d128c6f911dcf888a290a32ad50890408 +F ext/wasm/index.html 8289252ca8b25a4a76b8e5a86e84c42f5c7ca3302f2b5b1dbdf269fa96c28d84 F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e F ext/wasm/module-symbols.html b8eebafef8e536624bbe5f7a3da40c07a9062b843dfd3161a0bb72cbb6763dc5 @@ -549,10 +549,10 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac -F ext/wasm/tester1-esm.html 8d226a21b20707dbd66d68a3990141f0392fc781a281291d3dc59f38a3555887 -F ext/wasm/tester1-worker.html 51bf39e2b87f974ae3d5bc3086e2fb36d258f3698c54f6e21ba4b3b99636fa27 +F ext/wasm/tester1-esm.html 821c7a38b4eb753f6f3d7b5cbfda5b36f85466763977f96bff8372621b6b9eb2 +F ext/wasm/tester1-worker.html 84d56db05bcea2b294a89ca13c21b76fa0521ca4ac240f0055f1819934c713fc +F ext/wasm/tester1.c-pp.js f4b96977a48cdfc13d477ede6d2f754865e9bcd1a23ce09359c31de41b24ba0d w ext/wasm/tester1.js F ext/wasm/tester1.html 624ec41cd9f78a1f2b6d7df70aaa7a6394396b1f2455ecbd6de5775c1275b121 -F ext/wasm/tester1.js bff806de454de115922d78c056f11d523ec7ed9ed3839d4e21433a9f72558b88 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8aa7565f9de8dd3c291ad8c3ceb1a2c67a3eb31a8e531070b25c6c6b1f0278bf F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4cb285210b4a2b14c80962bf2ecb35be310d3444c329c15d86b3073096455704 -R 08dfc1f1d36db05c00bfb06ac1037121 -U drh -Z 7815819acfb2ddb8c0f5e3f3b0dc41f8 +P 0c2adc6d3547b07e950ae49f07f688f71a21b3ad5a51f16f0e8d49ab91564582 +R 22f7660d86bcdb1d5d00005b75d1164a +U stephan +Z 9963e0b4fa0faa6cb28eaed2062c3627 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b2733321bc..5f460cd815 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c2adc6d3547b07e950ae49f07f688f71a21b3ad5a51f16f0e8d49ab91564582 \ No newline at end of file +90480586f1b2ad82118e19536b095431b8457f294c0afaa9b4f883f184cc804c \ No newline at end of file From 47de1f9e198c609193fdf0162aba52e171b8c954 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 19 Nov 2022 18:17:40 +0000 Subject: [PATCH 022/282] Small performance improvement and size reduction in btree. FossilOrigin-Name: daa07149c0a0fcb6a6a1ace6020ca68802588ed309f5aaaf99c871088bc46908 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 8 +++++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 40a5b2d04c..41ca62e5cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\stester1.js\sworking\svia\san\sES6\sworker\smodule\sand\sadd\sthat\svariant\sto\sthe\sdist\szipfile. -D 2022-11-19T16:16:40.998 +C Small\sperformance\simprovement\sand\ssize\sreduction\sin\sbtree. +D 2022-11-19T18:17:40.978 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -551,7 +551,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-esm.html 821c7a38b4eb753f6f3d7b5cbfda5b36f85466763977f96bff8372621b6b9eb2 F ext/wasm/tester1-worker.html 84d56db05bcea2b294a89ca13c21b76fa0521ca4ac240f0055f1819934c713fc -F ext/wasm/tester1.c-pp.js f4b96977a48cdfc13d477ede6d2f754865e9bcd1a23ce09359c31de41b24ba0d w ext/wasm/tester1.js +F ext/wasm/tester1.c-pp.js f4b96977a48cdfc13d477ede6d2f754865e9bcd1a23ce09359c31de41b24ba0d F ext/wasm/tester1.html 624ec41cd9f78a1f2b6d7df70aaa7a6394396b1f2455ecbd6de5775c1275b121 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8aa7565f9de8dd3c291ad8c3ceb1a2c67a3eb31a8e531070b25c6c6b1f0278bf @@ -578,7 +578,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c c98cf491cbb8aa930bc9deb2cf794d2a32005c97c9cde01b7b8a10ef7c3d8451 +F src/btree.c 80f29390eedbdac7f0fd3312df1a2df5ecd3c5b55c5e22dab4fc3fc52d5584dd F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c2adc6d3547b07e950ae49f07f688f71a21b3ad5a51f16f0e8d49ab91564582 -R 22f7660d86bcdb1d5d00005b75d1164a -U stephan -Z 9963e0b4fa0faa6cb28eaed2062c3627 +P 90480586f1b2ad82118e19536b095431b8457f294c0afaa9b4f883f184cc804c +R 66ae91579f9086a7ec56fccf1b9beb6e +U drh +Z c7f53017341eeb656837188f25a22942 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5f460cd815..172b29b5b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90480586f1b2ad82118e19536b095431b8457f294c0afaa9b4f883f184cc804c \ No newline at end of file +daa07149c0a0fcb6a6a1ace6020ca68802588ed309f5aaaf99c871088bc46908 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 33762cc906..efa83dd520 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7239,14 +7239,16 @@ struct CellArray { ** computed. */ static void populateCellCache(CellArray *p, int idx, int N){ + MemPage *pRef = p->pRef; + u16 *szCell = p->szCell; assert( idx>=0 && idx+N<=p->nCell ); while( N>0 ){ assert( p->apCell[idx]!=0 ); - if( p->szCell[idx]==0 ){ - p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + if( szCell[idx]==0 ){ + szCell[idx] = pRef->xCellSize(pRef, p->apCell[idx]); }else{ assert( CORRUPT_DB || - p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + szCell[idx]==pRef->xCellSize(pRef, p->apCell[idx]) ); } idx++; N--; From 16635fa777cbd83eecb1c6a293ff7f4e4fa341ea Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 19 Nov 2022 18:43:32 +0000 Subject: [PATCH 023/282] Performance improvement and size reduction in balance_nonroot(). FossilOrigin-Name: 715bc81eb833ad4834d139a04085e0386c54c30d7395207e48972c4dfe5879c1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 41ca62e5cc..e9ef88380c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sand\ssize\sreduction\sin\sbtree. -D 2022-11-19T18:17:40.978 +C Performance\simprovement\sand\ssize\sreduction\sin\sbalance_nonroot(). +D 2022-11-19T18:43:32.814 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -578,7 +578,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 80f29390eedbdac7f0fd3312df1a2df5ecd3c5b55c5e22dab4fc3fc52d5584dd +F src/btree.c b9e66886f09b33cfa5058a6a3f3eddae92b5b9d566ba65bea26928f0b9860754 F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 90480586f1b2ad82118e19536b095431b8457f294c0afaa9b4f883f184cc804c -R 66ae91579f9086a7ec56fccf1b9beb6e +P daa07149c0a0fcb6a6a1ace6020ca68802588ed309f5aaaf99c871088bc46908 +R 486fbd2f4ae2f2943d0cbb110a637a8e U drh -Z c7f53017341eeb656837188f25a22942 +Z 537c9962ecec2516b80c7b85babce28a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 172b29b5b7..d910060e29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -daa07149c0a0fcb6a6a1ace6020ca68802588ed309f5aaaf99c871088bc46908 \ No newline at end of file +715bc81eb833ad4834d139a04085e0386c54c30d7395207e48972c4dfe5879c1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index efa83dd520..965897f894 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8244,15 +8244,17 @@ static int balance_nonroot( d = r + 1 - leafData; (void)cachedCellSize(&b, d); do{ + int szR, szD; assert( d szLeft-(b.szCell[r]+(i==k-1?0:2)))){ + && (bBulk || szRight+szD+2 > szLeft-(szR+(i==k-1?0:2)))){ break; } - szRight += b.szCell[d] + 2; - szLeft -= b.szCell[r] + 2; + szRight += szD + 2; + szLeft -= szR + 2; cntNew[i-1] = r; r--; d--; From 5a1d659d79604ff96f550e8ebcd2144e3cb665c3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 19 Nov 2022 19:37:26 +0000 Subject: [PATCH 024/282] Performance optimization in sqlite3BtreeInsert(). FossilOrigin-Name: 9776fa31758161970a50995a487b6543ed71e9610460b7324304ef21d9248707 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e9ef88380c..740a9fb5eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sand\ssize\sreduction\sin\sbalance_nonroot(). -D 2022-11-19T18:43:32.814 +C Performance\soptimization\sin\ssqlite3BtreeInsert(). +D 2022-11-19T19:37:26.733 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -578,7 +578,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c b9e66886f09b33cfa5058a6a3f3eddae92b5b9d566ba65bea26928f0b9860754 +F src/btree.c 52e71b216c4c8e661f01f70840d2cfe64677b07dd301e2ae684b8785328e7870 F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P daa07149c0a0fcb6a6a1ace6020ca68802588ed309f5aaaf99c871088bc46908 -R 486fbd2f4ae2f2943d0cbb110a637a8e +P 715bc81eb833ad4834d139a04085e0386c54c30d7395207e48972c4dfe5879c1 +R e1e21840abf2d0e625480a11eaf71aa4 U drh -Z 537c9962ecec2516b80c7b85babce28a +Z 802aa656f8405b91947fb6fe179ec5a2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d910060e29..650f1faa10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -715bc81eb833ad4834d139a04085e0386c54c30d7395207e48972c4dfe5879c1 \ No newline at end of file +9776fa31758161970a50995a487b6543ed71e9610460b7324304ef21d9248707 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 965897f894..a79e81734e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9182,12 +9182,13 @@ int sqlite3BtreeInsert( if( info.nPayload!=info.nLocal ){ Pgno ovfl = get4byte(&newCell[szNew-4]); ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + if( NEVER(rc) ) goto end_insert; } } }else{ rc = fillInCell(pPage, newCell, pX, &szNew); + if( rc ) goto end_insert; } - if( rc ) goto end_insert; assert( szNew==pPage->xCellSize(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(pBt) ); idx = pCur->ix; From ecba10730d032485d1067a819c7a3cc966db3d9e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 19 Nov 2022 20:10:55 +0000 Subject: [PATCH 025/282] New assert()s confirm that BTREE_PREFORMAT must be the same value as OPFLAG_PREFORMAT. FossilOrigin-Name: f40bf2c642643ae61d331e8d4815f601224fa258ab34344c6756966163a89f4a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 1 + src/vdbe.c | 1 + 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 740a9fb5eb..d9beffb9d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\ssqlite3BtreeInsert(). -D 2022-11-19T19:37:26.733 +C New\sassert()s\sconfirm\sthat\sBTREE_PREFORMAT\smust\sbe\sthe\ssame\svalue\sas\nOPFLAG_PREFORMAT. +D 2022-11-19T20:10:55.364 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -578,7 +578,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 52e71b216c4c8e661f01f70840d2cfe64677b07dd301e2ae684b8785328e7870 +F src/btree.c 308474d81388fa8b4fb92da7b2c73cae864c1ac468a7a5ffab61670af4b488f0 F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -711,7 +711,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 0be191521ff6d2805995f4910f0b6231b42843678b2efdc1abecaf39929a673f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 0c7cb1b934ad8611e14e7efaf2c3a95df7dd3f7964d63ea07fef42a23df86131 +F src/vdbe.c c2d6d0c0c343d8ebffef996c73cbb69e337225f757fea7fe5c0e3ea14662adec F src/vdbe.h 58675f47dcf3105bab182c3ad3726efd60ffd003e954386904ac9107d0d2b743 F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f F src/vdbeapi.c 1e8713d0b653acb43cd1bdf579c40e005c4844ea90f414f065946a83db3c27fb @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 715bc81eb833ad4834d139a04085e0386c54c30d7395207e48972c4dfe5879c1 -R e1e21840abf2d0e625480a11eaf71aa4 +P 9776fa31758161970a50995a487b6543ed71e9610460b7324304ef21d9248707 +R bee5cd2196754b460638bc4389e4c820 U drh -Z 802aa656f8405b91947fb6fe179ec5a2 +Z 72dd876552688ca64c7de697dec658ae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 650f1faa10..83c0e52e7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9776fa31758161970a50995a487b6543ed71e9610460b7324304ef21d9248707 \ No newline at end of file +f40bf2c642643ae61d331e8d4815f601224fa258ab34344c6756966163a89f4a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a79e81734e..114136bc50 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9172,6 +9172,7 @@ int sqlite3BtreeInsert( assert( pPage->isInit || CORRUPT_DB ); newCell = pBt->pTmpSpace; assert( newCell!=0 ); + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); if( flags & BTREE_PREFORMAT ){ rc = SQLITE_OK; szNew = pBt->nPreformatSize; diff --git a/src/vdbe.c b/src/vdbe.c index 4eb1c03271..a1c2afbb41 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5579,6 +5579,7 @@ case OP_Insert: { x.nZero = 0; } x.pKey = 0; + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), seekResult From 0a94ef8b52fac73d7c200adc876eb38a88664b4e Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 20 Nov 2022 01:15:22 +0000 Subject: [PATCH 026/282] Minor cleanups in the ESM-related preprocessor filtering. FossilOrigin-Name: 205884a273128bb666b496b659b4fa9f031ebdbbc1aa704fdeb4b7e015740098 --- ext/wasm/GNUmakefile | 43 +++++++++++++++++++------------- ext/wasm/api/extern-post-js.js | 8 +++--- ext/wasm/api/pre-js.js | 2 +- ext/wasm/api/sqlite3-api-opfs.js | 2 +- ext/wasm/tester1-esm.html | 7 +----- ext/wasm/tester1.c-pp.js | 4 +-- manifest | 24 +++++++++--------- manifest.uuid | 2 +- 8 files changed, 47 insertions(+), 45 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 3d952c2615..1c74531403 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -226,17 +226,17 @@ DISTCLEAN_FILES += $(bin.stripccomments) bin.c-pp := ./c-pp $(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) -define C-PP.JS +define C-PP.FILTER # Create $2 from $1 using $(bin.c-pp) -# $1 = Input file: c-pp -f X.js -# $2 = Output file: c-pp -o X.js +# $1 = Input file: c-pp -f $(1).js +# $2 = Output file: c-pp -o $(2).js # $3 = optional c-pp -D... flags $(2): $(1) $$(MAKEFILE) $$(bin.c-pp) $$(bin.c-pp) -f $(1) -o $$@ $(3) CLEAN_FILES += $(2) endef c-pp.D.vanilla ?= -c-pp.D.esm ?= -Dsqlite3-es6-module-build +c-pp.D.esm ?= -Dtarget=es6-module # /end CPP-of-JS bits ######################################################################## @@ -301,8 +301,8 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) pre-js.js.in := $(dir.api)/pre-js.js pre-js.js.esm := $(dir.tmp)/pre-js.esm.js pre-js.js.vanilla := $(dir.tmp)/pre-js.vanilla.js -$(eval $(call C-PP.JS,$(pre-js.js.in),$(pre-js.js.vanilla),$(c-pp.D.vanilla))) -$(eval $(call C-PP.JS,$(pre-js.js.in),$(pre-js.js.esm),$(c-pp.D.esm))) +$(eval $(call C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.esm),$(c-pp.D.esm))) post-js.js.in := $(dir.tmp)/post-js.js post-js.js.vanilla := $(dir.tmp)/post-js.vanilla.js post-js.js.esm := $(dir.tmp)/post-js.esm.js @@ -317,14 +317,14 @@ $(post-js.js.in): $(post-jses.js) $(MAKEFILE) cat $$i; \ echo "/* END FILE: $$i */"; \ done > $@ -$(eval $(call C-PP.JS,$(post-js.js.in),$(post-js.js.vanilla),$(c-pp.D.vanilla))) -$(eval $(call C-PP.JS,$(post-js.js.in),$(post-js.js.esm),$(c-pp.D.esm))) +$(eval $(call C-PP.FILTER,$(post-js.js.in),$(post-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.FILTER,$(post-js.js.in),$(post-js.js.esm),$(c-pp.D.esm))) extern-post-js.js.in := $(dir.api)/extern-post-js.js extern-post-js.js.vanilla := $(dir.tmp)/extern-post-js.vanilla.js extern-post-js.js.esm := $(dir.tmp)/extern-post-js.esm.js -$(eval $(call C-PP.JS,$(extern-post-js.js.in),$(extern-post-js.js.vanilla),$(c-pp.D.vanilla))) -$(eval $(call C-PP.JS,$(extern-post-js.js.in),$(extern-post-js.js.esm),$(c-pp.D.esm))) +$(eval $(call C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.esm),$(c-pp.D.esm))) extern-pre-js.js := $(dir.api)/extern-pre-js.js # Emscripten flags for --[extern-][pre|post]-js=... @@ -644,13 +644,22 @@ CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) ######################################################################## ######################################################################## -# tester1 code: -# tester1.js: for main thread and worker thread -# tester1-esm.js: to be loaded from an ES6 Worker Module thread -$(eval $(call C-PP.JS,tester1.c-pp.js,tester1.js)) -$(eval $(call C-PP.JS,tester1.c-pp.js,tester1-esm.js,-Dtester1-esm-worker)) -tester1.js: $(sqlite3.js) -tester1-esm.js: $(sqlite3.mjs) +# tester1 is the main unit and regression test application and needs to +# be able to run in 4 separate modes to cover the primary use cases: +# +# 1) Load sqlite3 in the main UI thread of a conventional script. +# 2) Load sqlite3 in a conventional Worker thread. +# 3) Load sqlite3 as an ES6 module (ESM) in the main thread. +# 4) Load sqlite3 as an ESM worker. (Not all browsers support this.) +# +# To that end, we require two separate builds of tester1.js: +# +# tester1.js: cases 1 and 2 +# tester1-esm.js: cases 3 and 4 +# +# To create those, we filter tester1.c-pp.js with $(bin.c-pp)... +$(eval $(call C-PP.FILTER,tester1.c-pp.js,tester1.js)) +$(eval $(call C-PP.FILTER,tester1.c-pp.js,tester1-esm.js,$(c-pp.D.esm))) tester1: tester1.js tester1-esm.js all: tester1 diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index 3dc61ae050..acf7068fef 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -8,7 +8,7 @@ most of the associated JS code, runs outside of the Emscripten-generated module init scope, in the current global scope. */ -//#if sqlite3-es6-module-build +//#if target=es6-module const toExportForES6 = //#endif (function(){ @@ -110,10 +110,8 @@ const toExportForES6 = exports["sqlite3InitModule"] = sqlite3InitModule; /* AMD modules get injected in a way we cannot override, so we can't handle those here. */ -//#if sqlite3-es6-module-build - return self.sqlite3InitModule; -//#endif + return self.sqlite3InitModule /* required for ESM */; })(); -//#if sqlite3-es6-module-build +//#if target=es6-module export default toExportForES6; //#endif diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.js index 41d13d6853..2e2fe66bc9 100644 --- a/ext/wasm/api/pre-js.js +++ b/ext/wasm/api/pre-js.js @@ -29,7 +29,7 @@ sqlite3InitModuleState.debugModule('self.location =',self.location); 4) If none of the above apply, (prefix+path) is returned. */ Module['locateFile'] = function(path, prefix) { -//#if sqlite3-es6-module-build +//#if target=es6-module return new URL(path, import.meta.url).href; //#else 'use strict'; diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 31fe45eda0..a3f73cc7b2 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -167,7 +167,7 @@ const installOpfsVfs = function callee(options){ return promiseReject_(err); }; const W = -//#if sqlite3-es6-module-build +//#if target=es6-module new Worker(new URL(options.proxyUri, import.meta.url)); //#else new Worker(options.proxyUri); diff --git a/ext/wasm/tester1-esm.html b/ext/wasm/tester1-esm.html index 118ab676bd..498a1ab6a7 100644 --- a/ext/wasm/tester1-esm.html +++ b/ext/wasm/tester1-esm.html @@ -20,11 +20,6 @@
    - - + diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 7aa8ba4a65..2a5da8407c 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -39,9 +39,9 @@ ES6 worker module build: - ./c-pp -f tester1.c-pp.js -o tester1-esm.js -Dtester1-esm-worker + ./c-pp -f tester1.c-pp.js -o tester1-esm.js -Dtarget=es6-module */ -//#if tester1-esm-worker +//#if target=es6-module import {default as sqlite3InitModule} from './jswasm/sqlite3.mjs'; self.sqlite3InitModule = sqlite3InitModule; //#else diff --git a/manifest b/manifest index d9beffb9d9..ae619fc516 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sassert()s\sconfirm\sthat\sBTREE_PREFORMAT\smust\sbe\sthe\ssame\svalue\sas\nOPFLAG_PREFORMAT. -D 2022-11-19T20:10:55.364 +C Minor\scleanups\sin\sthe\sESM-related\spreprocessor\sfiltering. +D 2022-11-20T01:15:22.201 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,21 +488,21 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 616c2a3e944483f6f5f66a6f77885eb7a7d3fef1a599014d746a792b6ef7b6b6 +F ext/wasm/GNUmakefile 3de890d61bccab21dce204848a9aaa4b009acee6e399604a1c9ecc5d1b418e7b F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 29276a845e57004e82efba61fa5866fd05f9137380a1dc26dc4c6d65264cd81c -F ext/wasm/api/extern-post-js.js 015121df2c903cf12d51507227b756ab3626036d8e9d610a2a2c15b3f54afe4d +F ext/wasm/api/extern-post-js.js 5a92c0afe8edbdfffc6831e60d7fd108594e37c4a364507035e7d3296589b153 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 -F ext/wasm/api/pre-js.js 1156a7fb9de817bb1cb39ad90b76aa93fbb9dcf950a1f2d6f547e5976872be36 +F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js 9f115a37dafe8067bce8812996d2deff45741c6e39f7aad7b48f5fbbd822dba5 +F ext/wasm/api/sqlite3-api-opfs.js b4ece97f94aacd408b37fbe5f6d6bb2cbfbed484ce700b17d1d446a55e6b7e81 F ext/wasm/api/sqlite3-api-prologue.js fd526fa017fa2578673ca18158354515c719e719a5d93f2f6d0e43f39170430e F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -549,9 +549,9 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac -F ext/wasm/tester1-esm.html 821c7a38b4eb753f6f3d7b5cbfda5b36f85466763977f96bff8372621b6b9eb2 +F ext/wasm/tester1-esm.html aef2e711655660ece4f726ff88332238da2811b9fe7e4987d621f35f9f41d6b6 F ext/wasm/tester1-worker.html 84d56db05bcea2b294a89ca13c21b76fa0521ca4ac240f0055f1819934c713fc -F ext/wasm/tester1.c-pp.js f4b96977a48cdfc13d477ede6d2f754865e9bcd1a23ce09359c31de41b24ba0d +F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tester1.html 624ec41cd9f78a1f2b6d7df70aaa7a6394396b1f2455ecbd6de5775c1275b121 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8aa7565f9de8dd3c291ad8c3ceb1a2c67a3eb31a8e531070b25c6c6b1f0278bf @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9776fa31758161970a50995a487b6543ed71e9610460b7324304ef21d9248707 -R bee5cd2196754b460638bc4389e4c820 -U drh -Z 72dd876552688ca64c7de697dec658ae +P f40bf2c642643ae61d331e8d4815f601224fa258ab34344c6756966163a89f4a +R bd56fdaa00a2ddbbb971d4117ea354ba +U stephan +Z faec45069379feabec016e0ee445a4ae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 83c0e52e7b..4f65f59d3a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f40bf2c642643ae61d331e8d4815f601224fa258ab34344c6756966163a89f4a \ No newline at end of file +205884a273128bb666b496b659b4fa9f031ebdbbc1aa704fdeb4b7e015740098 \ No newline at end of file From 8e1bd8264d53a4042447a1ed4208f5db0b33891d Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 20 Nov 2022 04:13:16 +0000 Subject: [PATCH 027/282] Add 'snapshot' target to create distinctly-named snapshot builds of the wasm deliverables zip file. FossilOrigin-Name: f7620aa09e4893971e00cdee5c6f1fe15c3bd21f985bec90fbd90fdfc457ac31 --- ext/wasm/dist.make | 51 +++++++++++++++++++++++----------------------- manifest | 12 +++++------ manifest.uuid | 2 +- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index f33711a19d..b64663c105 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -10,31 +10,30 @@ MAKEFILE.dist := $(lastword $(MAKEFILE_LIST)) ######################################################################## -# Chicken/egg situation: we need $(bin.version-info) to get the version -# info for the archive name, but that binary may not yet be built, and -# won't be built until we expand the dependencies. We have to use a -# temporary name for the archive. -dist-name = sqlite-wasm-TEMP -#ifeq (0,1) -# $(info WARNING *******************************************************************) -# $(info ** Be sure to create the desired build configuration before creating the) -# $(info ** distribution archive. Use one of the following targets to do so:) -# $(info **) -# $(info ** o2: builds with -O2, resulting in the fastest builds) -# $(info ** oz: builds with -Oz, resulting in the smallest builds) -# $(info /WARNING *******************************************************************) -#endif +# Chicken/egg situation: we need $(bin.version-info) to get the +# version info for the archive name, but that binary may not yet be +# built, and won't be built until we expand the dependencies. Thus we +# have to use a temporary name for the archive until we can get +# that binary built. +ifeq (,$(filter snapshot,$(MAKECMDGOALS))) +dist-name-prefix := sqlite-wasm +else +dist-name-prefix := sqlite-wasm-snapshot-$(shell /usr/bin/date +%Y%m%d) +endif +dist-name := $(dist-name-prefix)-TEMP ######################################################################## -# dist.build must be the name of a target which triggers the -# build of the files to be packed into the dist archive. The -# intention is that it be one of (o0, o1, o2, o3, os, oz), each of -# which uses like-named -Ox optimization level flags. The o2 target -# provides the best overall runtime speeds. The oz target provides -# slightly slower speeds (roughly 10%) with significantly smaller WASM -# file sizes. Note that -O2 (the o2 target) results in faster binaries -# than both -O3 and -Os (the o3 and os targets) in all tests run to -# date. +# dist.build must be the name of a target which triggers the build of +# the files to be packed into the dist archive. The intention is that +# it be one of (o0, o1, o2, o3, os, oz), each of which uses like-named +# -Ox optimization level flags. The o2 target provides the best +# overall runtime speeds. The oz target provides slightly slower +# speeds (roughly 10%) with significantly smaller WASM file +# sizes. Note that -O2 (the o2 target) results in faster binaries than +# both -O3 and -Os (the o3 and os targets) in all tests run to +# date. Our general policy is that we want the smallest binaries for +# dist zip files, so use the oz build unless there is a compelling +# reason not to. dist.build ?= oz dist-dir.top := $(dist-name) @@ -52,7 +51,7 @@ dist.common.extras := \ $(wildcard $(dir.common)/*.css) \ $(dir.common)/SqliteTestUtil.js -.PHONY: dist +.PHONY: dist snapshot ######################################################################## # dist: create the end-user deliverable archive. # @@ -84,7 +83,7 @@ dist: \ @cp -p $(dist.common.extras) $(dist-dir.common) @set -e; \ vnum=$$($(bin.version-info) --download-version); \ - vdir=sqlite-wasm-$$vnum; \ + vdir=$(dist-name-prefix)-$$vnum; \ arczip=$$vdir.zip; \ echo "Making $$arczip ..."; \ rm -fr $$arczip $$vdir; \ @@ -94,7 +93,7 @@ dist: \ ls -la $$arczip; \ set +e; \ unzip -lv $$arczip || echo "Missing unzip app? Not fatal." - +snapshot: dist # We need a separate `clean` rule to account for weirdness in # a sub-make, where we get a copy of the $(dist-name) dir # copied into the new $(dist-name) dir. diff --git a/manifest b/manifest index ae619fc516..03022549e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanups\sin\sthe\sESM-related\spreprocessor\sfiltering. -D 2022-11-20T01:15:22.201 +C Add\s'snapshot'\starget\sto\screate\sdistinctly-named\ssnapshot\sbuilds\sof\sthe\swasm\sdeliverables\szip\sfile. +D 2022-11-20T04:13:16.594 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -527,7 +527,7 @@ F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98 F ext/wasm/demo-worker1-promiser.js b85a2bb1b918db4f09dfa24419241cb3edad7791389425c2505092e9b715017d F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa -F ext/wasm/dist.make d00562b499096704304d9879f834a76475c8dd7ac58aaa3f039625ecc299aa3d +F ext/wasm/dist.make ff970852dbf879c8e29a3b060b4451d54ea309cc5373feb746bce96a256cfce8 F ext/wasm/fiddle.make 2812c44c9bafb5be9c8767963d1b9f374d77af7795fcaa06483c03e7059dea74 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f40bf2c642643ae61d331e8d4815f601224fa258ab34344c6756966163a89f4a -R bd56fdaa00a2ddbbb971d4117ea354ba +P 205884a273128bb666b496b659b4fa9f031ebdbbc1aa704fdeb4b7e015740098 +R 68583681205ba1b84e3374ab2ab4b57b U stephan -Z faec45069379feabec016e0ee445a4ae +Z 0d4fea2843b4049473725debb938394b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4f65f59d3a..d32aa7bfc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -205884a273128bb666b496b659b4fa9f031ebdbbc1aa704fdeb4b7e015740098 \ No newline at end of file +f7620aa09e4893971e00cdee5c6f1fe15c3bd21f985bec90fbd90fdfc457ac31 \ No newline at end of file From b2873a30140d495df32a20a4a6e0769e45974b1b Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 20 Nov 2022 04:14:00 +0000 Subject: [PATCH 028/282] Generic cleanups and doc additions in the wasm build files. FossilOrigin-Name: d10f385e36ee7fe3077d80d8d6e7ce55732d20ef73e2a63533d8d2932ec8bf62 --- ext/wasm/GNUmakefile | 327 ++++++++++++++++++++++++++----------------- ext/wasm/wasmfs.make | 2 +- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 206 insertions(+), 139 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 1c74531403..f803696c6e 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -20,6 +20,13 @@ # above-listed o? target names. # # clean = clean up +# +# Required tools beyond those needed for the canonical builds: +# +# - Emscripten SDK: https://emscripten.org/docs/getting_started/downloads.html +# - GNU make, GNU sed, GNU awk, GNU grep +# - wasm-strip for release builds: https://github.com/WebAssembly/wabt +# - InfoZip for 'dist' zip file ######################################################################## SHELL := $(shell which bash 2>/dev/null) MAKEFILE := $(lastword $(MAKEFILE_LIST)) @@ -71,17 +78,19 @@ dir.jacc := jaccwabyt dir.common := common dir.fiddle := fiddle dir.tool := $(dir.top)/tool +CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ $(dir.fiddle)/*~ + ######################################################################## # dir.dout = output dir for deliverables. # -# MAINTENANCE REMINDER: the output .js and .wasm files of emcc must be -# in _this_ dir, rather than a subdir, or else parts of the generated -# code get confused and cannot load property. Specifically, when X.js -# loads X.wasm, whether or not X.js uses the correct path for X.wasm -# depends on how it's loaded: an HTML script tag will resolve it -# intuitively, whereas a Worker's call to importScripts() will not. -# That's a fundamental incompatibility with how URL resolution in -# JS happens between those two contexts. See: +# MAINTENANCE REMINDER: the output .js and .wasm files of certain emcc +# buildables must be in _this_ dir, rather than a subdir, or else +# parts of the generated code get confused and cannot load +# property. Specifically, when X.js loads X.wasm, whether or not X.js +# uses the correct path for X.wasm depends on how it's loaded: an HTML +# script tag will resolve it intuitively, whereas a Worker's call to +# importScripts() will not. That's a fundamental incompatibility with +# how URL resolution in JS happens between those two contexts. See: # # https://zzz.buzz/2017/03/14/relative-uris-in-web-development/ # @@ -104,11 +113,10 @@ ifeq (,$(wildcard $(dir.tmp))) dir._tmp := $(shell mkdir -p $(dir.tmp)) endif -cflags.common := -I. -I.. -I$(dir.top) -CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ -emcc.WASM_BIGINT ?= 1 sqlite3.c := $(dir.top)/sqlite3.c sqlite3.h := $(dir.top)/sqlite3.h +# Most SQLITE_OPT flags are set in sqlite3-wasm.c but we need them +# made explicit here for building speedtest1.c. SQLITE_OPT = \ -DSQLITE_ENABLE_FTS5 \ -DSQLITE_ENABLE_RTREE \ @@ -130,43 +138,6 @@ SQLITE_OPT = \ '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ -DSQLITE_USE_URI=1 \ -DSQLITE_WASM_ENABLE_C_TESTS -# ^^^ most flags are set in sqlite3-wasm.c but we need them -# made explicit here for building speedtest1.c. - -ifneq (,$(filter release,$(MAKECMDGOALS))) -emcc_opt ?= -Oz -flto -else -emcc_opt ?= -O0 -# ^^^^ build times for -O levels higher than 0 are painful at -# dev-time. -endif -# When passing emcc_opt from the CLI, += and re-assignment have no -# effect, so emcc_opt+=-g3 doesn't work. So... -emcc_opt_full := $(emcc_opt) -g3 -# ^^^ ALWAYS use -g3. See below for why. -# -# ^^^ -flto improves runtime speed at -O0 considerably but doubles -# build time. -# -# ^^^^ -O3, -Oz, -Os minify symbol names and there appears to be no -# way around that except to use -g3, but -g3 causes the binary file -# size to absolutely explode (approx. 5x larger). This minification -# utterly breaks the resulting module, making it unsable except as -# self-contained/self-referential-only code, as ALL of the exported -# symbols get minified names. -# -# However, we have an option for using -Oz or -Os: -# -# Build with (-Os -g3) or (-Oz -g3) then use wasm-strip, from the wabt -# tools package (https://github.com/WebAssembly/wabt), to strip the -# debugging symbols. That results in a small build with unmangled -# symbol names. -Oz gives ever-so-slightly better compression than -# -Os: not quite 1% in some completely unscientific tests. Runtime -# speed for the unit tests is all over the place either way so it's -# difficult to say whether -Os gives any speed benefit over -Oz. -# -# (Much later: -O2 consistently gives the best speeds.) -######################################################################## $(sqlite3.c) $(sqlite3.h): $(MAKE) -C $(dir.top) sqlite3.c @@ -186,13 +157,20 @@ else $(info Development build. Use '$(MAKE) release' for a smaller release build.) endif +# bin.version-info = binary to output various sqlite3 version info for +# embedding in the JS files and in building the distribution zip file. +# It must NOT be in $(dir.tmp) because we need it to survive the +# cleanup process for the dist build to work properly. bin.version-info := $(dir.wasm)/version-info -# ^^^^ NOT in $(dir.tmp) because we need it to survive the cleanup -# process for the dist build to work properly. $(bin.version-info): $(dir.wasm)/version-info.c $(sqlite3.h) $(MAKEFILE) $(CC) -O0 -I$(dir.top) -o $@ $< DISTCLEAN_FILES += $(bin.version-info) +# bin.stripcomments is used for stripping C/C++-style comments from JS +# files. The JS files contain large chunks of documentation which we +# don't need for all builds. That app's -k flag is of particular +# importance here, as it allows us to retain the opening comment +# blocks, which contain the license header and version info. bin.stripccomments := $(dir.tool)/stripccomments $(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) $(CC) -o $@ $< @@ -200,7 +178,8 @@ DISTCLEAN_FILES += $(bin.stripccomments) ######################################################################## -# Transform $(1) to $(2) via ./c-pp -f $(1) ... +# C-PP.FILTER: a $(call)able to transform $(1) to $(2) via ./c-pp -f +# $(1) ... # # Historical notes: # @@ -237,17 +216,67 @@ CLEAN_FILES += $(2) endef c-pp.D.vanilla ?= c-pp.D.esm ?= -Dtarget=es6-module -# /end CPP-of-JS bits +# /end C-PP.FILTER ######################################################################## +# cflags.common = C compiler flags for all builds +cflags.common := -I. -I.. -I$(dir.top) +# emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API +# disables certain features if BigInt is not enabled and such builds +# _are not tested_ on any regular basis. +emcc.WASM_BIGINT ?= 1 + +# emcc_opt = optimization-related flags. These are primarily used by +# the various oX targets. build times for -O levels higher than 0 are +# painful at dev-time. +emcc_opt ?= -O0 + +# When passing emcc_opt from the CLI, += and re-assignment have no +# effect, so emcc_opt+=-g3 doesn't work. So... +emcc_opt_full := $(emcc_opt) -g3 +# ^^^ ALWAYS use -g3. See below for why. +# +# ^^^ -flto improves runtime speed at -O0 considerably but doubles +# build time. +# +# ^^^^ -O3, -Oz, -Os minify symbol names and there appears to be no +# way around that except to use -g3, but -g3 causes the binary file +# size to absolutely explode (approx. 5x larger). This minification +# utterly breaks the resulting module, making it unsable except as +# self-contained/self-referential-only code, as ALL of the exported +# symbols get minified names. +# +# However, we have an option for using -Oz or -Os: +# +# Build with (-Os -g3) or (-Oz -g3) then use wasm-strip, from the wabt +# tools package (https://github.com/WebAssembly/wabt), to strip the +# debugging symbols. That results in a small build with unmangled +# symbol names. -Oz gives ever-so-slightly better compression than +# -Os: not quite 1% in some completely unscientific tests. Runtime +# speed for the unit tests is all over the place either way so it's +# difficult to say whether -Os gives any speed benefit over -Oz. +# +# Much practice has demonstrated that -O2 consistently gives the best +# runtime speeds, but not by a large enough factor to rule out use of +# -Oz when small deliverable size is a priority. +######################################################################## + +# EXPORTED_FUNCTIONS.* = files for use with Emscripten's +# -sEXPORTED_FUNCTION flag. EXPORTED_FUNCTIONS.api.in := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api $(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) - cat $(EXPORTED_FUNCTIONS.api.in) > $@ + cp $(EXPORTED_FUNCTIONS.api.in) $@ +# sqlite3-license-version.js = generated JS file with the license +# header and version info. sqlite3-license-version.js := $(dir.tmp)/sqlite3-license-version.js +# sqlite3-license-version-header.js = JS file containing only the +# license header. sqlite3-license-version-header.js := $(dir.api)/sqlite3-license-version-header.js +# sqlite3-api-build-version.js = generated JS file which populates the +# sqlite3.version object using $(bin.version-info). sqlite3-api-build-version.js := $(dir.tmp)/sqlite3-api-build-version.js # sqlite3-api.jses = the list of JS files which make up $(sqlite3-api.js), in # the order they need to be assembled. @@ -267,6 +296,8 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js sqlite3-worker1.js := $(dir.api)/sqlite3-worker1.js sqlite3-worker1-promiser.js := $(dir.api)/sqlite3-worker1-promiser.js +# COPY_XAPI = a $(call)able function to copy $1 to $(dir.dout), where +# $1 must be one of the "external" JS API files. define COPY_XAPI sqlite3-api.ext.jses += $$(dir.dout)/$$(notdir $(1)) $$(dir.dout)/$$(notdir $(1)): $(1) $$(MAKEFILE) @@ -276,6 +307,9 @@ $(foreach X,$(SOAP.js) $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js),\ $(eval $(call COPY_XAPI,$(X)))) all: $(sqlite3-api.ext.jses) +# sqlite3-api.js.in = the generated sqlite3-api.js before it gets +# preprocessed. It contains all of $(sqlite3-api.jses) but none of the +# Emscripten-specific headers and footers. sqlite3-api.js.in := $(dir.tmp)/sqlite3-api.c-pp.js $(sqlite3-api.js.in): $(sqlite3-api.jses) $(MAKEFILE) @echo "Making $@..." @@ -294,10 +328,22 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) echo ';'; \ echo '});'; \ } > $@ +$(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) \ + $(MAKEFILE) + @echo "Making $@..."; { \ + cat $(sqlite3-license-version-header.js); \ + echo '/*'; \ + echo '** This code was built from sqlite3 version...'; \ + echo "** "; \ + awk -e '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' \ + -e '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ + echo '*/'; \ + } > $@ ######################################################################## # --post-js and --pre-js are emcc flags we use to append/prepend JS to -# the generated emscripten module file. +# the generated emscripten module file. The following rules generate +# various versions of those files for the vanilla and ESM builds. pre-js.js.in := $(dir.api)/pre-js.js pre-js.js.esm := $(dir.tmp)/pre-js.esm.js pre-js.js.vanilla := $(dir.tmp)/pre-js.vanilla.js @@ -320,6 +366,9 @@ $(post-js.js.in): $(post-jses.js) $(MAKEFILE) $(eval $(call C-PP.FILTER,$(post-js.js.in),$(post-js.js.vanilla),$(c-pp.D.vanilla))) $(eval $(call C-PP.FILTER,$(post-js.js.in),$(post-js.js.esm),$(c-pp.D.esm))) +# extern-post-js* and extern-pre-js* are files for use with +# Emscripten's --extern-pre-js and --extern-post-js flags. These +# rules make different copies for the vanilla and ESM builds. extern-post-js.js.in := $(dir.api)/extern-post-js.js extern-post-js.js.vanilla := $(dir.tmp)/extern-post-js.vanilla.js extern-post-js.js.esm := $(dir.tmp)/extern-post-js.esm.js @@ -327,7 +376,8 @@ $(eval $(call C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.vanilla),$ $(eval $(call C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.esm),$(c-pp.D.esm))) extern-pre-js.js := $(dir.api)/extern-pre-js.js -# Emscripten flags for --[extern-][pre|post]-js=... +# Emscripten flags for --[extern-][pre|post]-js=... for the +# various builds. pre-post-common.flags := \ --extern-pre-js=$(sqlite3-license-version.js) pre-post-common.flags.vanilla := \ @@ -339,26 +389,21 @@ pre-post-common.flags.esm := \ --post-js=$(post-js.js.esm) \ --extern-post-js=$(extern-post-js.js.esm) +# pre-post-jses.deps.* = a list of dependencies for the +# --[extern-][pre/post]-js files. pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) pre-post-jses.deps.vanilla := $(pre-post-jses.deps.common) \ $(post-js.js.vanilla) $(extern-post-js.js.vanilla) pre-post-jses.deps.esm := $(pre-post-jses.deps.common) \ $(post-js.js.esm) $(extern-post-js.js.esm) -$(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) $(MAKEFILE) - @echo "Making $@..."; { \ - cat $(sqlite3-license-version-header.js); \ - echo '/*'; \ - echo '** This code was built from sqlite3 version...'; \ - echo "** "; \ - awk -e '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' \ - -e '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ - echo '*/'; \ - } > $@ ######################################################################## -# call-make-pre-js creates rules for pre-js-$(1).js. $1 = the base -# name of the JS file on whose behalf this pre-js is for. $2 is the -# build mode: one of (vanilla, esm). +# call-make-pre-js is a $(call)able which creates rules for +# pre-js-$(1).js. $1 = the base name of the JS file on whose behalf +# this pre-js is for. $2 is the build mode: one of (vanilla, esm). +# This sets up --[extern-][pre/post]-js flags in +# $(pre-post-$(1).flags.$(2)) and dependencies in +# $(pre-post-$(1).deps.$(2)). define call-make-pre-js pre-post-$(1).flags.$(2) ?= $$(dir.tmp)/pre-js-$(1)-$(2).js: $$(pre-js.js.$(2)) $$(MAKEFILE) @@ -375,29 +420,30 @@ pre-post-$(1).flags.$(2) += \ $$(pre-post-common.flags.$(2)) \ --pre-js=$$(dir.tmp)/pre-js-$(1)-$(2).js endef -#$(error $(call call-make-pre-js,sqlite3-wasmfs)) # /post-js and pre-js ######################################################################## ######################################################################## # emcc flags for .c/.o/.wasm/.js. emcc.flags := -#emcc.flags += -v # _very_ loud but also informative about what it's doing -# -g3 is needed to keep -O2 and higher from creating broken JS via -# minification. +ifeq (1,$(verbose)) +emcc.flags += -v +# -v is _very_ loud but also informative about what it's doing +endif ######################################################################## # emcc flags for .c/.o. emcc.cflags := emcc.cflags += -std=c99 -fPIC -# -------------^^^^^^^^ we currently need c99 for WASM-specific sqlite3 APIs. +# -------------^^^^^^^^ we need c99 for $(sqlite3-wasm.c). emcc.cflags += -I. -I$(dir.top) ######################################################################## -# emcc flags specific to building the final .js/.wasm file... +# emcc flags specific to building .js/.wasm files... emcc.jsflags := -fPIC emcc.jsflags += --minify 0 emcc.jsflags += --no-entry +emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) emcc.jsflags += -sMODULARIZE emcc.jsflags += -sSTRICT_JS emcc.jsflags += -sDYNAMIC_EXECUTION=0 @@ -405,7 +451,9 @@ emcc.jsflags += -sNO_POLYFILL emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.api) emcc.exportedRuntimeMethods := \ -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory - # FS ==> stdio/POSIX I/O proxies + # FS ==> stdio/POSIX I/O proxies. Currently used explicitly only + # by the fiddle app, and it must never be exposed to client code + # via our APIs. # wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 @@ -439,32 +487,35 @@ emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) emcc.jsflags += $(emcc.environment) #emcc.jsflags += -sTOTAL_STACK=4194304 - -sqlite3.js.init-func := sqlite3InitModule -# ^^^^ $(sqlite3.js.init-func) symbol name is hard-coded in +######################################################################## +# $(sqlite3.js.init-func) is the name Emscripten assigns our exported +# module init/load function. This symbol name is hard-coded in # $(extern-post-js.js) as well as in numerous docs. If changed, it # needs to be globally modified in *.js and all related documentation. - +# Note that changing it will break client applications, so never +# change it unless you're creating a custom deliverable. +sqlite3.js.init-func := sqlite3InitModule emcc.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. #emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() #emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS #emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API -#emcc.jsflags += -sABORTING_MALLOC +#emcc.jsflags += -sABORTING_MALLOC # only for experimentation emcc.jsflags += -sALLOW_TABLE_GROWTH -# -sALLOW_TABLE_GROWTH is required for installing new SQL UDFs +# ^^^^ -sALLOW_TABLE_GROWTH is required for installing new SQL UDFs emcc.jsflags += -Wno-limited-postlink-optimizations -# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. -#emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why -# https://lld.llvm.org/WebAssembly.html -emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 +# ^^^^ emcc likes to warn when we have "limited optimizations" via the +# -g3 flag. +# emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why. + +# Re. undefined symbol handling, see: https://lld.llvm.org/WebAssembly.html +emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=1 emcc.jsflags += -sLLD_REPORT_UNDEFINED #emcc.jsflags += --allow-undefined #emcc.jsflags += --import-undefined #emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic #emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined #emcc.jsflags += --unresolved-symbols=ignore-all -emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) ######################################################################## # -sMEMORY64=1 fails to load, erroring with: @@ -476,18 +527,21 @@ emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) # new Uint8Array(wasm.heap8u().buffer, ptr, n) # # because ptr is now a BigInt, so is invalid for passing to arguments -# which have strict must-be-a-Number requirements. +# which have strict must-be-a-Number requirements. That aspect will +# make any eventual port to 64-bit address space extremely painful, as +# such constructs are found all over the place in the source code. ######################################################################## ######################################################################## # -sSINGLE_FILE: -# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js#L1704 -# -sSINGLE_FILE=1 would be really nice but we have to build with -g3 +# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js +# +# -sSINGLE_FILE=1 would be _really_ nice but we have to build with -g3 # for -O2 and higher to work (else minification breaks the code) and # cannot wasm-strip the binary before it gets encoded into the JS -# file. The result is that the generated JS file is, because of the -g3 -# debugging info, _huge_. +# file. The result is that the generated JS file is, because of the +# -g3 debugging info, _huge_. ######################################################################## sqlite3.js := $(dir.dout)/sqlite3.js @@ -504,36 +558,41 @@ sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c # instead of building a shared copy of sqlite3-wasm.o. $(eval $(call call-make-pre-js,sqlite3,vanilla)) $(eval $(call call-make-pre-js,sqlite3,esm)) -$(sqlite3.js): $(sqlite3.js) $(sqlite3.mjs): $(MAKEFILE) $(sqlite3.wasm.obj) \ $(EXPORTED_FUNCTIONS.api) $(sqlite3.js): $(pre-post-sqlite3.deps.vanilla) $(sqlite3.mjs): $(pre-post-sqlite3.deps.esm) -# SQLITE3.xJS.RECIPE = Recipe body for $(sqlite3.js) and -# $(sqlite3.mjs). $1 = one of (vanilla, esm). +# SQLITE3.xJS.RECIPE = the $(call)able recipe body for $(sqlite3.js) +# and $(sqlite3.mjs). $1 = one of (vanilla, esm). define SQLITE3.xJS.RECIPE @echo "Building $@ ..." $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ $(emcc.jsflags) \ $(pre-post-sqlite3.flags.$(1)) $(emcc.flags.sqlite3.$(1)) \ $(cflags.common) $(SQLITE_OPT) $(sqlite3-wasm.c) - if [ esm = $(1) ]; then \ - sed -i -e '0,/^export default/{/^export default/d}' $@; \ - fi # work around an Emscripten annoyance. See emcc.flags.esm + @if [ esm = $(1) ]; then \ + echo "Fragile workaround for an Emscripten annoyance. See emcc.flags.sqlite3.esm."; \ + sed -i -e '0,/^export default/{/^export default/d}' $@ || exit $$?; \ + if ! grep -q '^export default' $@; then \ + echo "Cannot find export default." 1>&2; \ + exit 1; \ + fi; \ + fi chmod -x $(sqlite3.wasm) $(maybe-wasm-strip) $(sqlite3.wasm) @ls -la $@ $(sqlite3.wasm) endef emcc.flags.sqlite3.vanilla := emcc.flags.sqlite3.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META -# Reminder: even if we use -sEXPORT_ES6=0, emcc _still_ adds: +# Reminder for ESM build: even if we use -sEXPORT_ES6=0, emcc _still_ +# adds: # # export default $(sqlite3.js.init-func); # # when building *.mjs, which is bad because we need to export an # overwritten version of that function and cannot "export default" # twice. Because of this, we have to sed $(sqlite3.mjs) to remove the -# first instance of /^export default/. +# _first_ instance (only) of /^export default/. $(sqlite3.js): $(call SQLITE3.xJS.RECIPE,vanilla) $(sqlite3.mjs): @@ -542,17 +601,18 @@ $(sqlite3.wasm): $(sqlite3.js) $(sqlite3.mjs): $(sqlite3.js) # We have to ensure that we do not build both $(sqlite3.js) and # $(sqlite3.mjs) in parallel because both result in the build of -# $(sqlite3.wasm). We have no way to build just the .mjs file without -# also building the .wasm file. i.e. we're building $(sqlite3.wasm) -# twice, but that's unavoidable. +# $(sqlite3.wasm). We have no(?) way to build just the .mjs file +# without also building the .wasm file. i.e. we're building +# $(sqlite3.wasm) twice, but that's apparently unavoidable. CLEAN_FILES += $(sqlite3.js) $(sqlite3.mjs) $(sqlite3.wasm) all: $(sqlite3.mjs) wasm: $(sqlite3.mjs) -# End main Emscripten-based module build +# End main $(sqlite3.js) build ######################################################################## ######################################################################## -# batch-runner.js... +# batch-runner.js is part of one of the test apps which reads in SQL +# dumps generated by $(speedtest1) and executes them. dir.sql := sql speedtest1 := ../../speedtest1 speedtest1.c := ../../test/speedtest1.c @@ -574,29 +634,30 @@ batch: batch-runner.list all: batch # end batch-runner.js ######################################################################## -# speedtest1.js... -# speedtest1-common.eflags = emcc flags used by multiple builds of speedtest1 +# Wasmified speedtest1 is our primary benchmarking tool. +# +# speedtest1.eflags.common = emcc flags used by multiple builds of speedtest1 # speedtest1.eflags = emcc flags used by main build of speedtest1 -speedtest1-common.eflags := $(emcc_opt_full) +speedtest1.eflags.common := $(emcc_opt_full) speedtest1.eflags := speedtest1.eflags += -sENVIRONMENT=web speedtest1.eflags += -sALLOW_MEMORY_GROWTH speedtest1.eflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) -speedtest1-common.eflags += -sINVOKE_RUN=0 -speedtest1-common.eflags += --no-entry -#speedtest1-common.eflags += -flto -speedtest1-common.eflags += -sABORTING_MALLOC -speedtest1-common.eflags += -sSTRICT_JS -speedtest1-common.eflags += -sMODULARIZE -speedtest1-common.eflags += -Wno-limited-postlink-optimizations +speedtest1.eflags.common += -sINVOKE_RUN=0 +speedtest1.eflags.common += --no-entry +#speedtest1.eflags.common += -flto +speedtest1.eflags.common += -sABORTING_MALLOC +speedtest1.eflags.common += -sSTRICT_JS +speedtest1.eflags.common += -sMODULARIZE +speedtest1.eflags.common += -Wno-limited-postlink-optimizations EXPORTED_FUNCTIONS.speedtest1 := $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) -speedtest1-common.eflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.speedtest1) -speedtest1-common.eflags += $(emcc.exportedRuntimeMethods) -speedtest1-common.eflags += -sALLOW_TABLE_GROWTH -speedtest1-common.eflags += -sDYNAMIC_EXECUTION=0 -speedtest1-common.eflags += --minify 0 -speedtest1-common.eflags += -sEXPORT_NAME=$(sqlite3.js.init-func) -speedtest1-common.eflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) +speedtest1.eflags.common += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.speedtest1) +speedtest1.eflags.common += $(emcc.exportedRuntimeMethods) +speedtest1.eflags.common += -sALLOW_TABLE_GROWTH +speedtest1.eflags.common += -sDYNAMIC_EXECUTION=0 +speedtest1.eflags.common += --minify 0 +speedtest1.eflags.common += -sEXPORT_NAME=$(sqlite3.js.init-func) +speedtest1.eflags.common += -sWASM_BIGINT=$(emcc.WASM_BIGINT) speedtest1.exit-runtime0 := -sEXIT_RUNTIME=0 speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 # Re -sEXIT_RUNTIME=1 vs 0: if it's 1 and speedtest1 crashes, we get @@ -629,8 +690,8 @@ $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ - $(speedtest1.eflags) $(speedtest1-common.eflags) $(speedtest1.cflags) \ - $(pre-post-speedtest1.flags.vanilla) \ + $(speedtest1.eflags) $(speedtest1.eflags.common) \ + $(speedtest1.cflags) $(pre-post-speedtest1.flags.vanilla) \ $(SQLITE_OPT) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cses) -lm @@ -644,8 +705,9 @@ CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) ######################################################################## ######################################################################## -# tester1 is the main unit and regression test application and needs to -# be able to run in 4 separate modes to cover the primary use cases: +# tester1 is the main unit and regression test application and needs +# to be able to run in 4 separate modes to cover the primary +# client-side use cases: # # 1) Load sqlite3 in the main UI thread of a conventional script. # 2) Load sqlite3 in a conventional Worker thread. @@ -672,8 +734,8 @@ all: tester1 .PHONY: o0 o1 o2 o3 os oz o-xtra := -flto # ^^^^ -flto can have a considerably performance boost at -O0 but -# doubles the build time and seems to have negligible effect on -# higher optimization levels. +# doubles the build time and seems to have negligible, if any, effect +# on higher optimization levels. o0: clean $(MAKE) -e "emcc_opt=-O0" o1: clean @@ -691,16 +753,21 @@ oz: clean ######################################################################## # Sub-makes... +# sqlite.org/fiddle application... include fiddle.make # Only add wasmfs if wasmfs.enable=1 or we're running (dist)clean +ifneq (,$(filter wasmfs,$(MAKECMDGOALS))) +wasmfs.enable ?= 1 +else wasmfs.enable ?= $(if $(filter %clean,$(MAKECMDGOALS)),1,0) +endif ifeq (1,$(wasmfs.enable)) # wasmfs build disabled 2022-10-19 per /chat discussion. # OPFS-over-wasmfs was initially a stopgap measure and a convenient # point of comparison for the OPFS sqlite3_vfs's performance, but it # currently doubles our deliverables and build maintenance burden for -# little, if any, benefit. +# little benefit. # ######################################################################## # Some platforms do not support the WASMFS build. Raspberry Pi OS is one @@ -720,8 +787,8 @@ endif ######################################################################## ######################################################################## -# Create deliverables: -ifneq (,$(filter dist,$(MAKECMDGOALS))) +# Create main client downloadable zip file: +ifneq (,$(filter dist snapshot,$(MAKECMDGOALS))) include dist.make endif diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make index 33f812aac6..e4d72059a4 100644 --- a/ext/wasm/wasmfs.make +++ b/ext/wasm/wasmfs.make @@ -101,7 +101,7 @@ $(speedtest1-wasmfs.js): $(speedtest1.cses) $(sqlite3-wasmfs.js) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ - $(speedtest1-wasmfs.eflags) $(speedtest1-common.eflags) \ + $(speedtest1-wasmfs.eflags) $(speedtest1.eflags.common) \ $(pre-post-speedtest1-wasmfs.flags) \ $(speedtest1.cflags) \ $(sqlite3-wasmfs.cflags) \ diff --git a/manifest b/manifest index 03022549e2..d2b3c2ece4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s'snapshot'\starget\sto\screate\sdistinctly-named\ssnapshot\sbuilds\sof\sthe\swasm\sdeliverables\szip\sfile. -D 2022-11-20T04:13:16.594 +C Generic\scleanups\sand\sdoc\sadditions\sin\sthe\swasm\sbuild\sfiles. +D 2022-11-20T04:14:00.750 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,7 +488,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 3de890d61bccab21dce204848a9aaa4b009acee6e399604a1c9ecc5d1b418e7b +F ext/wasm/GNUmakefile 003235fe1156e208d66cefe43ff20248e190dd8e4a6d58c3bcf12039af514dce F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 @@ -554,7 +554,7 @@ F ext/wasm/tester1-worker.html 84d56db05bcea2b294a89ca13c21b76fa0521ca4ac240f005 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tester1.html 624ec41cd9f78a1f2b6d7df70aaa7a6394396b1f2455ecbd6de5775c1275b121 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 -F ext/wasm/wasmfs.make 8aa7565f9de8dd3c291ad8c3ceb1a2c67a3eb31a8e531070b25c6c6b1f0278bf +F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 205884a273128bb666b496b659b4fa9f031ebdbbc1aa704fdeb4b7e015740098 -R 68583681205ba1b84e3374ab2ab4b57b +P f7620aa09e4893971e00cdee5c6f1fe15c3bd21f985bec90fbd90fdfc457ac31 +R 7c89dac6ffbc92dde31ec50f0fbc1ade U stephan -Z 0d4fea2843b4049473725debb938394b +Z 0b0ddc4c7b7acf4fd988a55bdb70d0c8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d32aa7bfc4..9492a7d53a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7620aa09e4893971e00cdee5c6f1fe15c3bd21f985bec90fbd90fdfc457ac31 \ No newline at end of file +d10f385e36ee7fe3077d80d8d6e7ce55732d20ef73e2a63533d8d2932ec8bf62 \ No newline at end of file From 59a9654715f0dd8e20f9de8ddd97c4f52f391ce9 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 20 Nov 2022 04:14:29 +0000 Subject: [PATCH 029/282] Remove an obsolete reference to WASMFS from ext/wasm/index.html. FossilOrigin-Name: 51ff681864ec19844f8e7a46aef132e8a8601a1b64e1f5a243a53c6413f2a61a --- ext/wasm/index.html | 4 ---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 0a6cd0a620..37d66603f6 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -32,10 +32,6 @@ and testing is currently done against a dev-channel release of Chrome (v107 as of 2022-09-26).
    • -
  • Whether or not WASMFS/OPFS support is enabled on any given - page may depend on build-time options which are off by - default. -
    • The tests and demos... diff --git a/manifest b/manifest index d2b3c2ece4..8aaeeb13df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Generic\scleanups\sand\sdoc\sadditions\sin\sthe\swasm\sbuild\sfiles. -D 2022-11-20T04:14:00.750 +C Remove\san\sobsolete\sreference\sto\sWASMFS\sfrom\sext/wasm/index.html. +D 2022-11-20T04:14:29.454 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -534,7 +534,7 @@ F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf2 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c4337617c4d6d4d0796827cec28ac81d128c6f911dcf888a290a32ad50890408 -F ext/wasm/index.html 8289252ca8b25a4a76b8e5a86e84c42f5c7ca3302f2b5b1dbdf269fa96c28d84 +F ext/wasm/index.html 5393ced912ee9af18cc8cefbda96fac922839d192d7c3d4ec4f4b42dd7f1cf8b F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e F ext/wasm/module-symbols.html b8eebafef8e536624bbe5f7a3da40c07a9062b843dfd3161a0bb72cbb6763dc5 @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f7620aa09e4893971e00cdee5c6f1fe15c3bd21f985bec90fbd90fdfc457ac31 -R 7c89dac6ffbc92dde31ec50f0fbc1ade +P d10f385e36ee7fe3077d80d8d6e7ce55732d20ef73e2a63533d8d2932ec8bf62 +R 37857805b3b6071fa9b37ce0a0873f80 U stephan -Z 0b0ddc4c7b7acf4fd988a55bdb70d0c8 +Z f1048ad4ae84ec54ea82e7fb7b483cf6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9492a7d53a..6db9a839dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d10f385e36ee7fe3077d80d8d6e7ce55732d20ef73e2a63533d8d2932ec8bf62 \ No newline at end of file +51ff681864ec19844f8e7a46aef132e8a8601a1b64e1f5a243a53c6413f2a61a \ No newline at end of file From 6d97c1a844f297043c0ca49e117af599dd0915ff Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 20 Nov 2022 05:36:52 +0000 Subject: [PATCH 030/282] Further minor cleanups in the JS build related to vanilla vs ESM. FossilOrigin-Name: 100a596800eca61477d9880092465d594c22be3707f2a11aaf6eb9e234fc6f2d --- ext/wasm/GNUmakefile | 68 +++++++++++++------- ext/wasm/api/extern-post-js.js | 20 +++--- ext/wasm/dist.make | 4 ++ ext/wasm/tester1-esm.html | 25 ------- ext/wasm/tester1-worker.html | 2 +- ext/wasm/{tester1.html => tester1.c-pp.html} | 18 +++++- manifest | 21 +++--- manifest.uuid | 2 +- 8 files changed, 87 insertions(+), 73 deletions(-) delete mode 100644 ext/wasm/tester1-esm.html rename ext/wasm/{tester1.html => tester1.c-pp.html} (69%) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index f803696c6e..647044665c 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -24,7 +24,8 @@ # Required tools beyond those needed for the canonical builds: # # - Emscripten SDK: https://emscripten.org/docs/getting_started/downloads.html -# - GNU make, GNU sed, GNU awk, GNU grep +# - The bash shell +# - GNU make, GNU sed, GNU awk, GNU grep (all in the $PATH) # - wasm-strip for release builds: https://github.com/WebAssembly/wabt # - InfoZip for 'dist' zip file ######################################################################## @@ -426,7 +427,7 @@ endef ######################################################################## # emcc flags for .c/.o/.wasm/.js. emcc.flags := -ifeq (1,$(verbose)) +ifeq (1,$(emcc.verbose)) emcc.flags += -v # -v is _very_ loud but also informative about what it's doing endif @@ -490,10 +491,19 @@ emcc.jsflags += $(emcc.environment) ######################################################################## # $(sqlite3.js.init-func) is the name Emscripten assigns our exported # module init/load function. This symbol name is hard-coded in -# $(extern-post-js.js) as well as in numerous docs. If changed, it -# needs to be globally modified in *.js and all related documentation. -# Note that changing it will break client applications, so never -# change it unless you're creating a custom deliverable. +# $(extern-post-js.js) as well as in numerous docs. +# +# "sqlite3InitModule" is the symbol we document for client use, so +# that's the symbol name which must be exported, whether it comes from +# Emscripten or our own code in extern-post-js.js. +# +# That said... we can change $(sqlite3.js.init-func) as long as the +# name "sqlite3InitModule" is the one which gets exposed via the +# resulting JS files. That can be accomplished via +# extern-post-js.js. However... using a temporary symbol name here +# and then adding sqlite3InitModule() ourselves results in 2 global +# symbols: we cannot "delete" the Emscripten-defined +# $(sqlite3.js.init-func) because it's declared with "var". sqlite3.js.init-func := sqlite3InitModule emcc.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. @@ -562,8 +572,23 @@ $(sqlite3.js) $(sqlite3.mjs): $(MAKEFILE) $(sqlite3.wasm.obj) \ $(EXPORTED_FUNCTIONS.api) $(sqlite3.js): $(pre-post-sqlite3.deps.vanilla) $(sqlite3.mjs): $(pre-post-sqlite3.deps.esm) +######################################################################## # SQLITE3.xJS.RECIPE = the $(call)able recipe body for $(sqlite3.js) # and $(sqlite3.mjs). $1 = one of (vanilla, esm). +# +# Reminder for ESM builds: even if we use -sEXPORT_ES6=0, emcc _still_ +# adds: +# +# export default $(sqlite3.js.init-func); +# +# when building *.mjs, which is bad because we need to export an +# overwritten version of that function and cannot "export default" +# twice. Because of this, we have to sed $(sqlite3.mjs) to remove the +# _first_ instance (only) of /^export default/. +# +# Upstream RFE: +# https://github.com/emscripten-core/emscripten/issues/18237 +######################################################################## define SQLITE3.xJS.RECIPE @echo "Building $@ ..." $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ @@ -584,26 +609,19 @@ define SQLITE3.xJS.RECIPE endef emcc.flags.sqlite3.vanilla := emcc.flags.sqlite3.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META -# Reminder for ESM build: even if we use -sEXPORT_ES6=0, emcc _still_ -# adds: -# -# export default $(sqlite3.js.init-func); -# -# when building *.mjs, which is bad because we need to export an -# overwritten version of that function and cannot "export default" -# twice. Because of this, we have to sed $(sqlite3.mjs) to remove the -# _first_ instance (only) of /^export default/. $(sqlite3.js): $(call SQLITE3.xJS.RECIPE,vanilla) $(sqlite3.mjs): $(call SQLITE3.xJS.RECIPE,esm) -$(sqlite3.wasm): $(sqlite3.js) -$(sqlite3.mjs): $(sqlite3.js) +######################################################################## # We have to ensure that we do not build both $(sqlite3.js) and -# $(sqlite3.mjs) in parallel because both result in the build of +# $(sqlite3.mjs) in parallel because both result in the creation of # $(sqlite3.wasm). We have no(?) way to build just the .mjs file # without also building the .wasm file. i.e. we're building -# $(sqlite3.wasm) twice, but that's apparently unavoidable. +# $(sqlite3.wasm) twice, but that's apparently unavoidable (and +# harmless, just a waste of build time). +$(sqlite3.wasm): $(sqlite3.js) +$(sqlite3.mjs): $(sqlite3.js) CLEAN_FILES += $(sqlite3.js) $(sqlite3.mjs) $(sqlite3.wasm) all: $(sqlite3.mjs) wasm: $(sqlite3.mjs) @@ -717,12 +735,14 @@ CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) # To that end, we require two separate builds of tester1.js: # # tester1.js: cases 1 and 2 -# tester1-esm.js: cases 3 and 4 +# tester1.mjs: cases 3 and 4 # # To create those, we filter tester1.c-pp.js with $(bin.c-pp)... $(eval $(call C-PP.FILTER,tester1.c-pp.js,tester1.js)) -$(eval $(call C-PP.FILTER,tester1.c-pp.js,tester1-esm.js,$(c-pp.D.esm))) -tester1: tester1.js tester1-esm.js +$(eval $(call C-PP.FILTER,tester1.c-pp.js,tester1.mjs,$(c-pp.D.esm))) +$(eval $(call C-PP.FILTER,tester1.c-pp.html,tester1.html)) +$(eval $(call C-PP.FILTER,tester1.c-pp.html,tester1-esm.html,$(c-pp.D.esm))) +tester1: tester1.js tester1.mjs tester1.html tester1-esm.html all: tester1 ######################################################################## @@ -794,8 +814,8 @@ endif ######################################################################## # Push files to public wasm-testing.sqlite.org server -wasm-testing.include = $(dir.dout) *.js *.html \ - batch-runner.list $(dir.sql) $(dir.common) $(dir.fiddle) $(dir.jacc) +wasm-testing.include = *.js *.html batch-runner.list \ + $(dir.dout) $(dir.sql) $(dir.common) $(dir.fiddle) $(dir.jacc) wasm-testing.exclude = sql/speedtest1.sql wasm-testing.dir = /jail/sites/wasm-testing wasm-testing.dest ?= wasm-testing:$(wasm-testing.dir) diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index acf7068fef..cace6ed51c 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -13,19 +13,20 @@ const toExportForES6 = //#endif (function(){ /** - In order to hide the sqlite3InitModule()'s resulting Emscripten - module from downstream clients (and simplify our documentation by - being able to elide those details), we rewrite - sqlite3InitModule() to return the sqlite3 object. + In order to hide the sqlite3InitModule()'s resulting + Emscripten module from downstream clients (and simplify our + documentation by being able to elide those details), we hide that + function and expose a hand-written sqlite3InitModule() to return + the sqlite3 object (most of the time). Unfortunately, we cannot modify the module-loader/exporter-based impls which Emscripten installs at some point in the file above this. */ const originalInit = - /*Maintenance reminde: DO NOT use `self.` here. It's correct - for non-ES6 Module cases but wrong for ES6 modules because those - resolve this symbol differently! */ sqlite3InitModule; + /* Maintenance reminder: DO NOT use `self.` here. It's correct + for non-ES6 Module cases but wrong for ES6 modules because those + resolve this symbol differently. */ sqlite3InitModule; if(!originalInit){ throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); } @@ -104,10 +105,11 @@ const toExportForES6 = } /* Replace the various module exports performed by the Emscripten glue... */ - if (typeof exports === 'object' && typeof module === 'object') + if (typeof exports === 'object' && typeof module === 'object'){ module.exports = sqlite3InitModule; - else if (typeof exports === 'object') + }else if (typeof exports === 'object'){ exports["sqlite3InitModule"] = sqlite3InitModule; + } /* AMD modules get injected in a way we cannot override, so we can't handle those here. */ return self.sqlite3InitModule /* required for ESM */; diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index b64663c105..6cd4e48e6b 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -6,6 +6,10 @@ # 'make dist' rules for creating a distribution archive of the WASM/JS # pieces, noting that we only build a dist of the built files, not the # numerous pieces required to build them. +# +# Use 'make snapshot' to create "snapshot" releases. They use a +# distinctly different zip file and top directory name to distinguish +# them from release builds. ####################################################################### MAKEFILE.dist := $(lastword $(MAKEFILE_LIST)) diff --git a/ext/wasm/tester1-esm.html b/ext/wasm/tester1-esm.html deleted file mode 100644 index 498a1ab6a7..0000000000 --- a/ext/wasm/tester1-esm.html +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - sqlite3 tester #1: ES6 Module in UI thread - - - -

    sqlite3 tester #1: ES6 Module in UI thread

    -
    - - -
    -
    - - - diff --git a/ext/wasm/tester1-worker.html b/ext/wasm/tester1-worker.html index b750df9169..33ffa64186 100644 --- a/ext/wasm/tester1-worker.html +++ b/ext/wasm/tester1-worker.html @@ -46,7 +46,7 @@ logHtml('warning',"Attempting to run an ES6 Worker Module, "+ "which is not supported by all browsers! "+ "e.g. Firefox (as of 2022-11) cannot do this."); - workerArgs.push("tester1-esm.js",{type:"module"}); + workerArgs.push("tester1.mjs",{type:"module"}); document.querySelectorAll('title,#color-target').forEach((e)=>{ e.innerText = "sqlite3 tester #1: ES6 Worker Module"; }); diff --git a/ext/wasm/tester1.html b/ext/wasm/tester1.c-pp.html similarity index 69% rename from ext/wasm/tester1.html rename to ext/wasm/tester1.c-pp.html index f7a2fba4af..b1b68e486e 100644 --- a/ext/wasm/tester1.html +++ b/ext/wasm/tester1.c-pp.html @@ -6,7 +6,13 @@ - sqlite3 tester #1 (UI thread) + sqlite3 tester #1: +//#if target=es6-module +ES6 Module in UI thread +//#else +UI thread +//#endif + -

    sqlite3 WASM/JS tester #1 (UI thread)

    +

    See tester1-worker.html for the Worker-thread variant.
    @@ -22,7 +28,15 @@
    + +//#if target=es6-module + +//#else +//#endif diff --git a/manifest b/manifest index 8aaeeb13df..85775d3bfd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sobsolete\sreference\sto\sWASMFS\sfrom\sext/wasm/index.html. -D 2022-11-20T04:14:29.454 +C Further\sminor\scleanups\sin\sthe\sJS\sbuild\srelated\sto\svanilla\svs\sESM. +D 2022-11-20T05:36:52.173 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,13 +488,13 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 003235fe1156e208d66cefe43ff20248e190dd8e4a6d58c3bcf12039af514dce +F ext/wasm/GNUmakefile 712795c4893ea65f8d30fe414937a33b677a194dd58372b4074aee17039c845e F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 29276a845e57004e82efba61fa5866fd05f9137380a1dc26dc4c6d65264cd81c -F ext/wasm/api/extern-post-js.js 5a92c0afe8edbdfffc6831e60d7fd108594e37c4a364507035e7d3296589b153 +F ext/wasm/api/extern-post-js.js 59e52f579cd3a332d73dae94c91b9579daafb10dd6ada03803f1afa6bdad7689 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 @@ -527,7 +527,7 @@ F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98 F ext/wasm/demo-worker1-promiser.js b85a2bb1b918db4f09dfa24419241cb3edad7791389425c2505092e9b715017d F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa -F ext/wasm/dist.make ff970852dbf879c8e29a3b060b4451d54ea309cc5373feb746bce96a256cfce8 +F ext/wasm/dist.make c9f06b520390fc5ab354b4b124e69c1cc648f97daf52df9de36a852fbdd7a4ea F ext/wasm/fiddle.make 2812c44c9bafb5be9c8767963d1b9f374d77af7795fcaa06483c03e7059dea74 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d @@ -549,10 +549,9 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac -F ext/wasm/tester1-esm.html aef2e711655660ece4f726ff88332238da2811b9fe7e4987d621f35f9f41d6b6 -F ext/wasm/tester1-worker.html 84d56db05bcea2b294a89ca13c21b76fa0521ca4ac240f0055f1819934c713fc +F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 +F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 w ext/wasm/tester1.html F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d -F ext/wasm/tester1.html 624ec41cd9f78a1f2b6d7df70aaa7a6394396b1f2455ecbd6de5775c1275b121 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2057,8 +2056,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d10f385e36ee7fe3077d80d8d6e7ce55732d20ef73e2a63533d8d2932ec8bf62 -R 37857805b3b6071fa9b37ce0a0873f80 +P 51ff681864ec19844f8e7a46aef132e8a8601a1b64e1f5a243a53c6413f2a61a +R b4563c289dd01589021977b3acf0c574 U stephan -Z f1048ad4ae84ec54ea82e7fb7b483cf6 +Z ab7f6de1db50f486d71ae6e36cd71c04 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6db9a839dc..3140f0d9ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51ff681864ec19844f8e7a46aef132e8a8601a1b64e1f5a243a53c6413f2a61a \ No newline at end of file +100a596800eca61477d9880092465d594c22be3707f2a11aaf6eb9e234fc6f2d \ No newline at end of file From ae276719f002a92d1262fc45e67118922f4707b8 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 20 Nov 2022 05:47:17 +0000 Subject: [PATCH 031/282] js dist: account for a file rename in the previous checkin. FossilOrigin-Name: 469f9011a885e19b99210c5e3e582afa140b8b5f0aa7a720334848df5ab6ae98 --- ext/wasm/dist.make | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index 6cd4e48e6b..9b0267dac5 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -46,7 +46,7 @@ dist-dir.common := $(dist-dir.top)/common dist.top.extras := \ demo-123.html demo-123-worker.html demo-123.js \ tester1.html tester1-worker.html tester1-esm.html \ - tester1.js tester1-esm.js \ + tester1.js tester1.mjs \ demo-jsstorage.html demo-jsstorage.js \ demo-worker1.html demo-worker1.js \ demo-worker1-promiser.html demo-worker1-promiser.js diff --git a/manifest b/manifest index 85775d3bfd..1c39089785 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sminor\scleanups\sin\sthe\sJS\sbuild\srelated\sto\svanilla\svs\sESM. -D 2022-11-20T05:36:52.173 +C js\sdist:\saccount\sfor\sa\sfile\srename\sin\sthe\sprevious\scheckin. +D 2022-11-20T05:47:17.093 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -527,7 +527,7 @@ F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98 F ext/wasm/demo-worker1-promiser.js b85a2bb1b918db4f09dfa24419241cb3edad7791389425c2505092e9b715017d F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa -F ext/wasm/dist.make c9f06b520390fc5ab354b4b124e69c1cc648f97daf52df9de36a852fbdd7a4ea +F ext/wasm/dist.make 11b98da79385701a568a4728671821fe2524c1d1ecd05ff2e24cb3e33b2c6c4f F ext/wasm/fiddle.make 2812c44c9bafb5be9c8767963d1b9f374d77af7795fcaa06483c03e7059dea74 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d @@ -550,7 +550,7 @@ F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 -F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 w ext/wasm/tester1.html +F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d @@ -2056,8 +2056,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 51ff681864ec19844f8e7a46aef132e8a8601a1b64e1f5a243a53c6413f2a61a -R b4563c289dd01589021977b3acf0c574 +P 100a596800eca61477d9880092465d594c22be3707f2a11aaf6eb9e234fc6f2d +R fd97f4afc2e7676e23b759586618b38f U stephan -Z ab7f6de1db50f486d71ae6e36cd71c04 +Z 42f144497d0f844aac38afbea462a28b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3140f0d9ea..6d344e429d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -100a596800eca61477d9880092465d594c22be3707f2a11aaf6eb9e234fc6f2d \ No newline at end of file +469f9011a885e19b99210c5e3e582afa140b8b5f0aa7a720334848df5ab6ae98 \ No newline at end of file From 423003dca91a84409950bfd4e99ee68dc4f831ee Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 21 Nov 2022 00:11:09 +0000 Subject: [PATCH 032/282] Fix a base64 decode bug. Provide for convenient inclusion of extension(s) built into the CLI, to simplify testing and for its own sake. Improve comments. Cure collision between base64.c and base85.c when both are in the same translation unit. FossilOrigin-Name: 07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f --- ext/misc/base64.c | 55 ++++++++++++++++++++++++++-------------- ext/misc/base85.c | 50 ++++++++++++++++++++++++++++++------ ext/misc/basexx.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++ manifest | 17 +++++++------ manifest.uuid | 2 +- src/shell.c.in | 33 ++++++++++++++++++++++++ 6 files changed, 185 insertions(+), 36 deletions(-) create mode 100644 ext/misc/basexx.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 1ed1a4f174..d6f7834a85 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -12,7 +12,7 @@ ** ** This is a SQLite extension for converting in either direction ** between a (binary) blob and base64 text. Base64 can transit a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals, ** and can be used unmodified in XML-like documents. ** @@ -37,10 +37,10 @@ ** error will be thrown for other input argument types. ** ** This code relies on UTF-8 encoding only with respect to the -** meaning of the first 128 (7-bit) codes being the same as ASCII. +** meaning of the first 128 (7-bit) codes matching that of USASCII. ** It will fail miserably if somehow made to try to convert EBCDIC. -** Because it is table-driven, it could be enhanced to handle that. -** But the world and SQLite have moved on from that anachronism. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. ** ** To build the extension: ** Set shell variable SQDIR= @@ -52,12 +52,16 @@ #include +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ #include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #define PC 0x80 /* pad character */ #define WS 0x81 /* whitespace */ #define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' typedef unsigned char ubyte; @@ -89,10 +93,10 @@ static const char b64Numerals[64] #define IS_BX_PAD(bdp) ((bdp)==PC) #define BX_NUMERAL(dv) (b64Numerals[dv]) /* Width of base64 lines. Should be an integer multiple of 4. */ -#define DARK_MAX 72 +#define B64_DARK_MAX 72 /* Encode a byte buffer into base64 text. If pSep!=0, it's a C string -** to be appended to encoded groups to limit their length to DARK_MAX +** to be appended to encoded groups to limit their length to B64_DARK_MAX ** or to terminate the last group (to aid concatenation.) */ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ @@ -111,12 +115,12 @@ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ nc = ncio[nbi]; nbIn -= nbi; for( nbe=3; nbe>=0; --nbe ){ - char ce = (nbe0 && *pIn!='=' ){ + while( ncIn>0 && *pIn!=PAD_CHAR ){ static signed char nboi[] = { 0, 0, 1, 2, 3 }; char *pUse = skipNonB64(pIn); unsigned long qv = 0L; - int nti, nbo; + int nti, nbo, nac; ncIn -= (pUse - pIn); - if( ncIn<=0 ) break; pIn = pUse; nti = (ncIn>4)? 4 : ncIn; nbo = nboi[nti]; - while( nti>0 ){ - char c = *pIn++; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac<=nti)? *pIn++ : PAD_CHAR; ubyte bdp = BX_DV_PROTO(c); --ncIn; switch( bdp ){ @@ -157,15 +161,16 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ break; case PC: bdp = 0; + --nbo; /* fall thru */ - default: /* It's the digit value. */ + default: /* bdp is the digit value. */ qv = qv<<6 | bdp; - --nti; break; } } + nti = 2; while( nbo-- > 0 ){ - *pOut++ = (qv >> (8*nbo))&0xff; + *pOut++ = (qv >> (8*nti--))&0xff; } } return pOut; @@ -183,7 +188,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ case SQLITE_BLOB: nb = nv; nc = 4*(nv+2/3); /* quads needed */ - nc += (nc+(DARK_MAX-1))/DARK_MAX + 1; /* LFs and a 0-terminator */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ if( nvMax < nc ){ sqlite3_result_error(context, "blob expanded to base64 too big.", -1); } @@ -219,11 +224,15 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ /* ** Establish linkage to running SQLite library. */ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; return sqlite3_create_function @@ -231,3 +240,11 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, 0, base64, 0, 0); } + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 365db80f2b..dfa9bd0e31 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -14,7 +14,7 @@ ** It can be built as a standalone program or an SQLite3 extension. ** ** Much like base64 representations, base85 can be sent through a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals. ** It is not suited for unmodified use in XML-like documents. ** @@ -24,7 +24,7 @@ ** Further, this is an independent implementation of a base85 system. ** Hence, the author has rightfully put this into the public domain. ** -** Base85 numerals are taken from the set of 7-bit ASCII codes, +** Base85 numerals are taken from the set of 7-bit USASCII codes, ** excluding control characters and Space ! " ' ( ) { | } ~ Del ** in code order representing digit values 0 to 84 (base 10.) ** @@ -32,8 +32,11 @@ ** are represented as 5-digit base85 numbers with MS to LS digit ** order. Groups of 1-3 bytes are represented with 2-4 digits, ** still big-endian but 8-24 bit values. (Using big-endian yields -** the simplest transition to byte groups smaller than 4 bytes.) +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) ** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. ** ** Any character not in the base85 numeral set delimits groups. ** When base85 is streamed or stored in containers of indefinite @@ -82,7 +85,10 @@ #ifndef BASE85_STANDALONE +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ # include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #else @@ -107,7 +113,7 @@ static void sayHelp(){ } #endif -/* Classify c according to interval within ASCII set w.r.t. base85 +/* Classify c according to interval within USASCII set w.r.t. base85 * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. */ #define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) @@ -127,16 +133,25 @@ static unsigned char base85DigitValue( char c ){ } #endif +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + static char * skipNonB85( char *s ){ char c; while( (c = *s) && !IS_B85(c) ) ++s; return s; } +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.*/ static char base85Numeral( unsigned char b ){ return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); } +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ int nCol = 0; *pOut = 0; @@ -157,7 +172,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ pOut[--nco] = base85Numeral(dv); } pOut += ncio[nbi]; - if( pSep && ((nCol += ncio[nbi])>=80 || nbIn<=0) ){ + if( pSep && ((nCol += ncio[nbi])>=B85_DARK_MAX || nbIn<=0) ){ char *p = pSep; while( *p ) *pOut++ = *p++; nCol = 0; @@ -167,6 +182,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ return pOut; } +/* Decode base85 text into a byte buffer. */ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 ){ @@ -196,6 +212,7 @@ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ } #ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ static int allBase85( char *p, int len ){ char c; while( len-- > 0 && (c = *p++) != 0 ){ @@ -208,6 +225,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE # ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); switch( sqlite3_value_type(av[0]) ){ @@ -228,6 +246,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ } # endif +/* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ int nb, nc, nv = sqlite3_value_bytes(av[0]); int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), @@ -272,11 +291,18 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "base85 OOM", -1); } +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; # ifndef OMIT_BASE85_CHECKER @@ -294,12 +320,20 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, 0, base85, 0, 0); } +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + #else /* standalone program */ int main(int na, char *av[]){ int cin; int rc = 0; - unsigned char bBuf[64]; + unsigned char bBuf[4*(B85_DARK_MAX/5)]; char cBuf[5*(sizeof(bBuf)/4)+2]; size_t nio; # ifndef OMIT_BASE85_CHECKER diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c new file mode 100644 index 0000000000..700a99409c --- /dev/null +++ b/ext/misc/basexx.c @@ -0,0 +1,64 @@ +/* +** 2022-11-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This source allows multiple SQLite extensions to be either: combined +** into a single runtime-loadable library; or built into the SQLite shell +** using a preprocessing convention set by src/shell.c.in (and shell.c). +** +** Presently, it combines the base64.c and base85.c extensions. However, +** it can be used as a template for other combinations. +*/ + +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; +#endif + +static void init_api_ptr(const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); +} + +#undef SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT1 /* */ +#undef SQLITE_EXTENSION_INIT2 +#define SQLITE_EXTENSION_INIT2(v) (void)v + +/* These next 2 undef's are only needed because the entry point names + * collide when formulated per the rules stated for loadable extension + * entry point names that will be deduced from the file basenames. + */ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +#include "base64.c" + +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#include "base85.c" + +static int sqlite3_basexx_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + init_api_ptr(pApi); + int rc1 = BASE64_INIT(db); + int rc2 = BASE85_INIT(db); + int rc = SQLITE_OK; + + if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ + BASE64_EXPOSE(db, pzErr); + BASE64_EXPOSE(db, pzErr); + return SQLITE_OK; + }else{ + return SQLITE_ERROR; + } +} + +# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0) +# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/manifest b/manifest index faf2f3f49e..f02b4834b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sextensions\sfor\sbase85\sand\sbase64\sconversion\sUDFs -D 2022-11-19T02:39:16.296 +C Fix\sa\sbase64\sdecode\sbug.\sProvide\sfor\sconvenient\sinclusion\sof\sextension(s)\sbuilt\sinto\sthe\sCLI,\sto\ssimplify\stesting\sand\sfor\sits\sown\ssake.\sImprove\scomments.\sCure\scollision\sbetween\sbase64.c\sand\sbase85.c\swhen\sboth\sare\sin\sthe\ssame\stranslation\sunit. +D 2022-11-21T00:11:09.323 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,8 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c e4d3f13bb59aa903734b557e1400f3c5961f0a5abc40400e0e2aa3ad362c0fd7 -F ext/misc/base85.c 3e07aea038129ee646b129ac2ed736344ea03859ff447019e67250b80722c528 +F ext/misc/base64.c 05da915d991f24e59515d0566f9a206c338032410d42472a3d9348da8bc166f9 +F ext/misc/base85.c 2c680ca7733f9a86f5d292fec71d10777290e68e7ae59d90597ae75fc44a88b6 +F ext/misc/basexx.c d32037f1414d9da11e0be334395cd4d82438b756933c99fc4075b6346cd11fd2 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -640,7 +641,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 49b6aeaf950b9b7a74aca7b80e6d004ed32f0a16d2c9471e34587cc4755d5eaa F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea -R 4ecad003b3a6f60dd6b11d354f8edbe2 +P 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 +R 0260a27c5340f9b412a0f6e0981a0d6d U larrybr -Z d182470ca73df70df09238ac5a5e4382 +Z 18487c3a1bcdc27a89418b38ad3e3f97 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b47c8a0962..41284b2a7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 \ No newline at end of file +07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..478c055357 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1070,6 +1070,9 @@ INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h INCLUDE ../ext/recover/sqlite3recover.c #endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) +#endif #if defined(SQLITE_ENABLE_SESSION) /* @@ -5115,6 +5118,7 @@ static void open_db(ShellState *p, int openFlags){ } exit(1); } + #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif @@ -5137,6 +5141,34 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_sqlar_init(p->db, 0, 0); } #endif +#ifdef SQLITE_SHELL_EXTFUNCS + /* Create a preprocessing mechanism for extensions to make + * their own provisions for being built into the shell. + * This is a short-span macro. See further below for usage. + */ +#define SHELL_SUB_MACRO(base, variant) base ## _ ## variant +#define SHELL_SUBMACRO(base, variant) SHELL_SUB_MACRO(base, variant) + /* Let custom-included extensions get their ..._init() called. + * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause + * the extension's sqlite3_*_init( db, pzErrorMsg, pApi ) + * inititialization routine to be called. + */ + { + int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db); + /* Let custom-included extensions expose their functionality. + * The WHATEVER_EXPOSE( db, pzErrorMsg ) macro should cause + * the SQL functions, virtual tables, collating sequences or + * VFS's implemented by the extension to be registered. + */ + if( irc==SQLITE_OK + || irc==SQLITE_OK_LOAD_PERMANENTLY ){ + SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, EXPOSE)(p->db, 0); + } +#undef SHELL_SUB_MACRO +#undef SHELL_SUBMACRO + } +#endif + sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, @@ -5157,6 +5189,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); #endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", zDbFilename); From b5aaf0f28507a27a2ea8e18f642625d8727e9c48 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 21 Nov 2022 02:18:12 +0000 Subject: [PATCH 033/282] Get clean builds for MS tools. Add build examples for extension glommer (basexx.c) Make sure extensions build for dynamic load and built into shell. FossilOrigin-Name: da940d507e9e31a84daedede44a56f743198d7258a9e83892511fe8dd4c78d2b --- ext/misc/base64.c | 2 +- ext/misc/basexx.c | 22 +++++++++++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/ext/misc/base64.c b/ext/misc/base64.c index d6f7834a85..1ab0dc4ead 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -84,7 +84,7 @@ static const ubyte b64DigitValues[128] = { 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND }; -static const char b64Numerals[64] +static const char b64Numerals[64+1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; #define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80) diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c index 700a99409c..a113134c85 100644 --- a/ext/misc/basexx.c +++ b/ext/misc/basexx.c @@ -16,6 +16,23 @@ ** ** Presently, it combines the base64.c and base85.c extensions. However, ** it can be used as a template for other combinations. +** +** Example usages: +** +** - Build a runtime-loadable extension from SQLite checkout directory: +** *Nix, OSX: gcc -O2 -shared -I. -fPIC -o basexx.so ext/misc/basexx.c +** Win32: cl /Os -I. ext/misc/basexx.c -link -dll -out:basexx.dll +** +** - Incorporate as built-in in sqlite3 shell: +** *Nix, OSX with gcc on a like platform: +** export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** make sqlite3 "OPTS=$mop1 $mop2" +** Win32 with Microsoft toolset on Windows: +** set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** set mops="OPTS=%mop1% %mop2%" +** nmake -f Makefile.msc sqlite3.exe %mops% */ #ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ @@ -44,7 +61,10 @@ static void init_api_ptr(const sqlite3_api_routines *pApi){ #define sqlite3_base_init sqlite3_base85_init #include "base85.c" -static int sqlite3_basexx_init(sqlite3 *db, char **pzErr, +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_basexx_init(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ init_api_ptr(pApi); int rc1 = BASE64_INIT(db); diff --git a/manifest b/manifest index f02b4834b7..6d28b2a340 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbase64\sdecode\sbug.\sProvide\sfor\sconvenient\sinclusion\sof\sextension(s)\sbuilt\sinto\sthe\sCLI,\sto\ssimplify\stesting\sand\sfor\sits\sown\ssake.\sImprove\scomments.\sCure\scollision\sbetween\sbase64.c\sand\sbase85.c\swhen\sboth\sare\sin\sthe\ssame\stranslation\sunit. -D 2022-11-21T00:11:09.323 +C Get\sclean\sbuilds\sfor\sMS\stools.\sAdd\sbuild\sexamples\sfor\sextension\sglommer\s(basexx.c)\nMake\ssure\sextensions\sbuild\sfor\sdynamic\sload\sand\sbuilt\sinto\sshell. +D 2022-11-21T02:18:12.614 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,9 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c 05da915d991f24e59515d0566f9a206c338032410d42472a3d9348da8bc166f9 +F ext/misc/base64.c e611e8ce552b16251c1691191d95912f9e03400637311dbd458062d2d0b95e2a F ext/misc/base85.c 2c680ca7733f9a86f5d292fec71d10777290e68e7ae59d90597ae75fc44a88b6 -F ext/misc/basexx.c d32037f1414d9da11e0be334395cd4d82438b756933c99fc4075b6346cd11fd2 +F ext/misc/basexx.c 678dcc83894f78c26fd3662b322886777cc26bf2b40809236cd2abdad532a33c F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 -R 0260a27c5340f9b412a0f6e0981a0d6d +P 07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f +R 1b52ba3002b9af5fda34f27079570bff U larrybr -Z 18487c3a1bcdc27a89418b38ad3e3f97 +Z 7a2c313c3d99c0b1eaaafc6ea380893a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 41284b2a7c..81f4745948 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f \ No newline at end of file +da940d507e9e31a84daedede44a56f743198d7258a9e83892511fe8dd4c78d2b \ No newline at end of file From 27c4cd183d91d09e34e310d6349cda2b33c255ba Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 21 Nov 2022 03:50:52 +0000 Subject: [PATCH 034/282] Add test app for experimenting with multi-worker OPFS concurrency. Tweak OPFS VFS to significantly improve the otherwise "unfortunate" concurrency situation. FossilOrigin-Name: 96f76e7616f8157a342b9e1c42f7b1feab200d182268871a2b25f67d4ee2564c --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 1 + ext/wasm/api/extern-post-js.js | 3 + ext/wasm/api/sqlite3-api-opfs.js | 12 +- ext/wasm/api/sqlite3-api-prologue.js | 1 + ext/wasm/api/sqlite3-opfs-async-proxy.js | 187 +++++++++++--------- ext/wasm/index.html | 3 + ext/wasm/tests/opfs/concurrency/index.html | 34 ++++ ext/wasm/tests/opfs/concurrency/test.js | 97 ++++++++++ ext/wasm/tests/opfs/concurrency/worker.js | 95 ++++++++++ manifest | 25 +-- manifest.uuid | 2 +- 11 files changed, 364 insertions(+), 96 deletions(-) create mode 100644 ext/wasm/tests/opfs/concurrency/index.html create mode 100644 ext/wasm/tests/opfs/concurrency/test.js create mode 100644 ext/wasm/tests/opfs/concurrency/worker.js diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index b903bedee6..1f7908e3b8 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -7,6 +7,7 @@ _sqlite3_bind_null _sqlite3_bind_parameter_count _sqlite3_bind_parameter_index _sqlite3_bind_text +_sqlite3_busy_timeout _sqlite3_changes _sqlite3_changes64 _sqlite3_clear_bindings diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index cace6ed51c..b327837814 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -59,6 +59,9 @@ const toExportForES6 = li.pop(); initModuleState.sqlite3Dir = li.join('/') + '/'; } + if(initModuleState.sqlite3Dir){ + initModuleState.sqlite3Dir = initModuleState.sqlite3Dir.replace(/[/]{2,}/g,'/'); + } self.sqlite3InitModule = (...args)=>{ //console.warn("Using replaced sqlite3InitModule()",self.location); diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index a3f73cc7b2..1fd50dcc6f 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -92,7 +92,8 @@ const installOpfsVfs = function callee(options){ } const urlParams = new URL(self.location.href).searchParams; if(undefined===options.verbose){ - options.verbose = urlParams.has('opfs-verbose') ? 3 : 2; + options.verbose = urlParams.has('opfs-verbose') + ? (+urlParams.get('opfs-verbose') || 2) : 1; } if(undefined===options.sanityChecks){ options.sanityChecks = urlParams.has('opfs-sanity-check'); @@ -101,6 +102,8 @@ const installOpfsVfs = function callee(options){ options.proxyUri = callee.defaultProxyUri; } + //console.warn("OPFS options =",options,self.location); + if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } @@ -1154,7 +1157,10 @@ const installOpfsVfs = function callee(options){ [ /* Truncate journal mode is faster than delete or wal for this vfs, per speedtest1. */ - "pragma journal_mode=truncate;" + "pragma journal_mode=truncate;", + /* Set a default busy-timeout handler to help OPFS dbs + deal with multi-tab/multi-worker contention. */ + "pragma busy_timeout=2000;", /* This vfs benefits hugely from cache on moderate/large speedtest1 --size 50 and --size 100 workloads. We currently @@ -1162,7 +1168,7 @@ const installOpfsVfs = function callee(options){ sqlite3.wasm. If that policy changes, the cache can be set here. */ - //"pragma cache_size=-8388608;" + //"pragma cache_size=-16384;" ].join("") ); } diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index fed1c56669..8b2ce0936d 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -897,6 +897,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( the lines of sqlite3_prepare_v3(). The slightly problematic part is the final argument (text destructor). */ ], + ["sqlite3_busy_timeout","int", "sqlite3*", "int"], ["sqlite3_close_v2", "int", "sqlite3*"], ["sqlite3_changes", "int", "sqlite3*"], ["sqlite3_clear_bindings","int", "sqlite3_stmt*"], diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index e4657484ef..3701e8c30d 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -53,7 +53,7 @@ const state = Object.create(null); 2 = warnings and errors 3 = debug, warnings, and errors */ -state.verbose = 2; +state.verbose = 1; const loggers = { 0:console.error.bind(console), @@ -150,70 +150,6 @@ const getDirForFilename = async function f(absFilename, createDirs = false){ return [dh, filename]; }; -/** - An error class specifically for use with getSyncHandle(), the goal - of which is to eventually be able to distinguish unambiguously - between locking-related failures and other types, noting that we - cannot currently do so because createSyncAccessHandle() does not - define its exceptions in the required level of detail. -*/ -class GetSyncHandleError extends Error { - constructor(errorObject, ...msg){ - super(); - this.error = errorObject; - this.message = [ - ...msg, ': Original exception ['+errorObject.name+']:', - errorObject.message - ].join(' '); - this.name = 'GetSyncHandleError'; - } -}; - -/** - Returns the sync access handle associated with the given file - handle object (which must be a valid handle object, as created by - xOpen()), lazily opening it if needed. - - In order to help alleviate cross-tab contention for a dabase, - if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to 3 times. If acquisition - still fails at that point it will give up and propagate the - exception. -*/ -const getSyncHandle = async (fh)=>{ - if(!fh.syncHandle){ - const t = performance.now(); - log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 4, msBase = 300; - let i = 1, ms = msBase; - for(; true; ms = msBase * ++i){ - try { - //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); - //TODO? A config option which tells it to throw here - //randomly every now and then, for testing purposes. - fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); - break; - }catch(e){ - if(i === maxTries){ - throw new GetSyncHandleError( - e, "Error getting sync handle.",maxTries, - "attempts failed.",fh.filenameAbs - ); - } - warn("Error getting sync handle. Waiting",ms, - "ms and trying again.",fh.filenameAbs,e); - Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); - } - } - log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); - if(!fh.xLock){ - __autoLocks.add(fh.fid); - log("Auto-locked",fh.fid,fh.filenameAbs); - } - } - return fh.syncHandle; -}; - /** If the given file-holding object has a sync handle attached to it, that handle is remove and asynchronously closed. Though it may @@ -253,6 +189,101 @@ const closeSyncHandleNoThrow = async (fh)=>{ } }; +/* Release all auto-locks. */ +const closeAutoLocks = async ()=>{ + if(__autoLocks.size){ + /* Release all auto-locks. */ + for(const fid of __autoLocks){ + const fh = __openFiles[fid]; + await closeSyncHandleNoThrow(fh); + log("Auto-unlocked",fid,fh.filenameAbs); + } + } +}; + +/** + An error class specifically for use with getSyncHandle(), the goal + of which is to eventually be able to distinguish unambiguously + between locking-related failures and other types, noting that we + cannot currently do so because createSyncAccessHandle() does not + define its exceptions in the required level of detail. +*/ +class GetSyncHandleError extends Error { + constructor(errorObject, ...msg){ + super(); + this.error = errorObject; + this.message = [ + ...msg, ': Original exception ['+errorObject.name+']:', + errorObject.message + ].join(' '); + this.name = 'GetSyncHandleError'; + } +}; +GetSyncHandleError.convertRc = (e,rc)=>{ + if(1){ + /* This approach returns SQLITE_LOCKED to the C API + when getSyncHandle() fails but makes the very + wild assumption that such a failure _is_ a locking + error. In practice that appears to be the most + common error, by far, but we cannot unambiguously + distinguish that from other errors. + + This approach demonstrably reduces concurrency-related + errors but is highly questionable. + */ + return (e instanceof GetSyncHandleError) + ? state.sq3Codes.SQLITE_LOCKED + : rc; + }else{ + return ec; + } +} +/** + Returns the sync access handle associated with the given file + handle object (which must be a valid handle object, as created by + xOpen()), lazily opening it if needed. + + In order to help alleviate cross-tab contention for a dabase, + if an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to 3 times. If acquisition + still fails at that point it will give up and propagate the + exception. +*/ +const getSyncHandle = async (fh)=>{ + if(!fh.syncHandle){ + const t = performance.now(); + log("Acquiring sync handle for",fh.filenameAbs); + const maxTries = 4, msBase = 300; + let i = 1, ms = msBase; + for(; true; ms = msBase * ++i){ + try { + //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); + //TODO? A config option which tells it to throw here + //randomly every now and then, for testing purposes. + fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); + break; + }catch(e){ + if(i === maxTries){ + throw new GetSyncHandleError( + e, "Error getting sync handle.",maxTries, + "attempts failed.",fh.filenameAbs + ); + } + warn("Error getting sync handle. Waiting",ms, + "ms and trying again.",fh.filenameAbs,e); + //await closeAutoLocks(); + Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); + } + } + log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + if(!fh.xLock){ + __autoLocks.add(fh.fid); + log("Auto-locked",fh.fid,fh.filenameAbs); + } + } + return fh.syncHandle; +}; + /** Stores the given value at state.sabOPView[state.opIds.rc] and then Atomics.notify()'s it. @@ -451,7 +482,7 @@ const vfsAsyncImpls = { rc = 0; }catch(e){ state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR; + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); } wTimeEnd(); storeAndNotify('xFileSize', rc); @@ -471,7 +502,7 @@ const vfsAsyncImpls = { __autoLocks.delete(fid); }catch(e){ state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_LOCK; + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); fh.xLock = oldLockType; } wTimeEnd(); @@ -545,7 +576,7 @@ const vfsAsyncImpls = { if(undefined===nRead) wTimeEnd(); error("xRead() failed",e,fh); state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_READ; + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } storeAndNotify('xRead',rc); mTimeEnd(); @@ -579,7 +610,7 @@ const vfsAsyncImpls = { }catch(e){ error("xTruncate():",e,fh); state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } wTimeEnd(); storeAndNotify('xTruncate',rc); @@ -619,7 +650,7 @@ const vfsAsyncImpls = { }catch(e){ error("xWrite():",e,fh); state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_WRITE; + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } wTimeEnd(); storeAndNotify('xWrite',rc); @@ -746,22 +777,16 @@ const waitLoop = async function f(){ /** waitTime is how long (ms) to wait for each Atomics.wait(). We need to wake up periodically to give the thread a chance - to do other things. + to do other things. If this is too high (e.g. 500ms) then + even two workers/tabs can easily run into locking errors. */ - const waitTime = 500; + const waitTime = 150; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( state.sabOPView, state.opIds.whichOp, 0, waitTime )){ - if(__autoLocks.size){ - /* Release all auto-locks. */ - for(const fid of __autoLocks){ - const fh = __openFiles[fid]; - await closeSyncHandleNoThrow(fh); - log("Auto-unlocked",fid,fh.filenameAbs); - } - } + await closeAutoLocks(); continue; } const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); @@ -791,7 +816,7 @@ navigator.storage.getDirectory().then(function(d){ const opt = data.args; state.littleEndian = opt.littleEndian; state.asyncS11nExceptions = opt.asyncS11nExceptions; - state.verbose = opt.verbose ?? 2; + state.verbose = opt.verbose ?? 1; state.fileBufferSize = opt.fileBufferSize; state.sabS11nOffset = opt.sabS11nOffset; state.sabS11nSize = opt.sabS11nSize; diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 37d66603f6..9fa5bbdf49 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -104,6 +104,9 @@ synchronous sqlite3_vfs interface and the async OPFS impl. +
  • OPFS concurrency + tests using multiple workers. +
    • diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html new file mode 100644 index 0000000000..79a46692cc --- /dev/null +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -0,0 +1,34 @@ + + + + + + + + sqlite3 OPFS Worker concurrency tester + + + +

    +

    + OPFS concurrency tester using multiple independent Workers. + This app is incomplete. +

    +
    + + +
    +
    + + + + diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js new file mode 100644 index 0000000000..d045f3271f --- /dev/null +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -0,0 +1,97 @@ +(async function(self){ + + const logClass = (function(){ + const mapToString = (v)=>{ + switch(typeof v){ + case 'number': case 'string': case 'boolean': + case 'undefined': case 'bigint': + return ''+v; + default: break; + } + if(null===v) return 'null'; + if(v instanceof Error){ + v = { + message: v.message, + stack: v.stack, + errorClass: v.name + }; + } + return JSON.stringify(v,undefined,2); + }; + const normalizeArgs = (args)=>args.map(mapToString); + const logTarget = document.querySelector('#test-output'); + const logClass = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass){ + for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){ + ln.classList.add(c); + } + } + ln.append(document.createTextNode(normalizeArgs(args).join(' '))); + logTarget.append(ln); + }; + const cbReverse = document.querySelector('#cb-log-reverse'); + const cbReverseKey = 'tester1:cb-log-reverse'; + const cbReverseIt = ()=>{ + logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); + localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); + }; + cbReverse.addEventListener('change', cbReverseIt, true); + if(localStorage.getItem(cbReverseKey)){ + cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); + } + cbReverseIt(); + return logClass; + })(); + const stdout = (...args)=>logClass('',...args); + const stderr = (...args)=>logClass('error',...args); + + const wait = async (ms)=>{ + return new Promise((resolve)=>setTimeout(resolve,ms)); + }; + + const urlArgsJs = new URL(document.currentScript.src).searchParams; + const urlArgsHtml = new URL(self.location.href).searchParams; + const options = Object.create(null); + options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); + options.workerCount = ( + urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 + ) || 3; + const workers = []; + workers.post = (type,...args)=>{ + for(const w of workers) w.postMessage({type, payload:args}); + }; + workers.loadedCount = 0; + workers.onmessage = function(msg){ + msg = msg.data; + const wName = msg.worker; + const prefix = 'Worker ['+wName+']:'; + switch(msg.type){ + case 'stdout': stdout(prefix,...msg.payload); break; + case 'stderr': stderr(prefix,...msg.payload); break; + case 'error': stderr(prefix,"ERROR:",...msg.payload); break; + case 'loaded': + stdout(prefix,"loaded"); + if(++workers.loadedCount === workers.length){ + stdout("All workers loaded. Telling them to run..."); + workers.post('run'); + } + break; + default: logClass('error',"Unhandled message type:",msg); break; + } + }; + + stdout("Launching",options.workerCount,"workers..."); + workers.uri = ( + 'worker.js?' + + 'sqlite3.dir='+options.sqlite3Dir + + '&opfs-verbose=2' + ); + for(let i = 0; i < options.workerCount; ++i){ + stdout("Launching worker..."); + workers.push(new Worker(workers.uri+(i ? '' : '&unlink-db'))); + } + // Have to delay onmessage assignment until after the loop + // to avoid that early workers get an undue head start. + workers.forEach((w)=>w.onmessage = workers.onmessage); +})(self); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js new file mode 100644 index 0000000000..7ba15bf8c1 --- /dev/null +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -0,0 +1,95 @@ +importScripts( + (new URL(self.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' +); +self.sqlite3InitModule().then(async function(sqlite3){ + const wName = Math.round(Math.random()*10000); + const wPost = (type,...payload)=>{ + postMessage({type, worker: wName, payload}); + }; + const stdout = (...args)=>wPost('stdout',...args); + const stderr = (...args)=>wPost('stderr',...args); + const postErr = (...args)=>wPost('error',...args); + if(!sqlite3.opfs){ + stderr("OPFS support not detected. Aborting."); + return; + } + + const wait = async (ms)=>{ + return new Promise((resolve)=>setTimeout(resolve,ms)); + }; + + const dbName = 'concurrency-tester.db'; + if((new URL(self.location.href).searchParams).has('unlink-db')){ + await sqlite3.opfs.unlink(dbName); + stdout("Unlinked",dbName); + } + wPost('loaded'); + + const run = async function(){ + const db = new sqlite3.opfs.OpfsDb(dbName); + //sqlite3.capi.sqlite3_busy_timeout(db.pointer, 2000); + db.transaction((db)=>{ + db.exec([ + "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", + "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" + ]); + }); + + const maxIterations = 10; + const interval = Object.assign(Object.create(null),{ + delay: 300, + handle: undefined, + count: 0 + }); + stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); + const doWork = async ()=>{ + const tm = new Date().getTime(); + ++interval.count; + const prefix = "v(#"+interval.count+")"; + stdout("Setting",prefix,"=",tm); + try{ + db.exec({ + sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", + bind: [wName, new Date().getTime()] + }); + //stdout("Set",prefix); + }catch(e){ + interval.error = e; + } + }; + const finish = ()=>{ + if(interval.error) stderr("Ending work due to error:",e.message); + else stdout("Ending work after",interval.count,"interval(s)"); + db.close(); + }; + if(1){/*use setInterval()*/ + interval.handle = setInterval(async ()=>{ + await doWork(); + if(interval.error || maxIterations === interval.count){ + clearInterval(interval.handle); + finish(); + } + }, interval.delay); + }else{ + /*This approach provides no concurrency whatsoever: each worker + is run to completion before any others can work.*/ + let i; + for(i = 0; i < maxIterations; ++i){ + await doWork(); + if(interval.error) break; + await wait(interval.ms); + } + finish(); + } + }/*run()*/; + + self.onmessage = function({data}){ + switch(data.type){ + case 'run': run().catch((e)=>postErr(e.message)); + break; + default: + stderr("Unhandled message type '"+data.type+"'."); + break; + } + }; +}); diff --git a/manifest b/manifest index 1c39089785..ccf4fd8d76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C js\sdist:\saccount\sfor\sa\sfile\srename\sin\sthe\sprevious\scheckin. -D 2022-11-20T05:47:17.093 +C Add\stest\sapp\sfor\sexperimenting\swith\smulti-worker\sOPFS\sconcurrency.\sTweak\sOPFS\sVFS\sto\ssignificantly\simprove\sthe\sotherwise\s"unfortunate"\sconcurrency\ssituation. +D 2022-11-21T03:50:52.240 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,10 +491,10 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/GNUmakefile 712795c4893ea65f8d30fe414937a33b677a194dd58372b4074aee17039c845e F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9120c2f8f51fa85f46dcf4dcb6b12f4a807d428f6089b99cdb08d8ddfcfd88b2 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 29276a845e57004e82efba61fa5866fd05f9137380a1dc26dc4c6d65264cd81c -F ext/wasm/api/extern-post-js.js 59e52f579cd3a332d73dae94c91b9579daafb10dd6ada03803f1afa6bdad7689 +F ext/wasm/api/extern-post-js.js 31400dd1c0ae3458a0e6510229e59318e45eac402a75dd703c2950b9b5758b46 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js b4ece97f94aacd408b37fbe5f6d6bb2cbfbed484ce700b17d1d446a55e6b7e81 -F ext/wasm/api/sqlite3-api-prologue.js fd526fa017fa2578673ca18158354515c719e719a5d93f2f6d0e43f39170430e +F ext/wasm/api/sqlite3-api-opfs.js 4c75ed11df5efff6bcd8dad4ad904d8b11efac2e1dd4cc2c84d1ee8ace4129ef +F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 24d1c1982a012d998907105a4ff1ff6881bf462395e90c06326817701e69f093 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 97cf1909670575eced940d36f1b5ea35c51a431d1035dc2f7ea6982faee97c1b F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -534,7 +534,7 @@ F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf2 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c4337617c4d6d4d0796827cec28ac81d128c6f911dcf888a290a32ad50890408 -F ext/wasm/index.html 5393ced912ee9af18cc8cefbda96fac922839d192d7c3d4ec4f4b42dd7f1cf8b +F ext/wasm/index.html 5be176de5be8ae96889798f803fef4f6a2ef31cee305a0430ca4629f6ae04c27 F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e F ext/wasm/module-symbols.html b8eebafef8e536624bbe5f7a3da40c07a9062b843dfd3161a0bb72cbb6763dc5 @@ -552,6 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d +F ext/wasm/tests/opfs/concurrency/index.html c7cf329e5b206dd8226d94ab9fec02f5f350d8ed69a57c96d84e876afd3d3d1b +F ext/wasm/tests/opfs/concurrency/test.js 44cfcc04503593256abe2dd663349718f80ee7ab25e19eb066de220101bd604a +F ext/wasm/tests/opfs/concurrency/worker.js f8f3e4f9b21726bef354a74ec9c90f6736df5b16b4f655bfd16a3b9c6ee063ff F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2056,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 100a596800eca61477d9880092465d594c22be3707f2a11aaf6eb9e234fc6f2d -R fd97f4afc2e7676e23b759586618b38f +P 469f9011a885e19b99210c5e3e582afa140b8b5f0aa7a720334848df5ab6ae98 +R c87fca3e6d0a9c36a2598013e36db2a5 U stephan -Z 42f144497d0f844aac38afbea462a28b +Z b0030359261e278f67d2690556943dbd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6d344e429d..2e8d22d1e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -469f9011a885e19b99210c5e3e582afa140b8b5f0aa7a720334848df5ab6ae98 \ No newline at end of file +96f76e7616f8157a342b9e1c42f7b1feab200d182268871a2b25f67d4ee2564c \ No newline at end of file From 36d5554c9abaa3080e85e3b7b517605c6106587d Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 21 Nov 2022 04:12:38 +0000 Subject: [PATCH 035/282] Resolve missing SQLITE_LOCKED result code which triggered a new (since last checkin) exception in the OPFS VFS. Improve output of the OPFS contention tester app. FossilOrigin-Name: 2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b --- ext/wasm/api/sqlite3-api-opfs.js | 1 + ext/wasm/api/sqlite3-opfs-async-proxy.js | 4 ++-- ext/wasm/tests/opfs/concurrency/test.js | 24 ++++++++++++++--------- ext/wasm/tests/opfs/concurrency/worker.js | 12 ++++++++---- manifest | 18 ++++++++--------- manifest.uuid | 2 +- 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 1fd50dcc6f..deb4c923ab 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -343,6 +343,7 @@ const installOpfsVfs = function callee(options){ 'SQLITE_LOCK_PENDING', 'SQLITE_LOCK_RESERVED', 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCKED', 'SQLITE_MISUSE', 'SQLITE_NOTFOUND', 'SQLITE_OPEN_CREATE', diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 3701e8c30d..58cf8ca3c7 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -477,8 +477,8 @@ const vfsAsyncImpls = { wTimeStart('xFileSize'); try{ affirmLocked('xFileSize',fh); - rc = await (await getSyncHandle(fh)).getSize(); - state.s11n.serialize(Number(rc)); + const sz = await (await getSyncHandle(fh)).getSize(); + state.s11n.serialize(Number(sz)); rc = 0; }catch(e){ state.s11n.storeException(2,e); diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index d045f3271f..b80dad24c4 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -1,6 +1,6 @@ (async function(self){ - const logClass = (function(){ + const logCss = (function(){ const mapToString = (v)=>{ switch(typeof v){ case 'number': case 'string': case 'boolean': @@ -20,7 +20,7 @@ }; const normalizeArgs = (args)=>args.map(mapToString); const logTarget = document.querySelector('#test-output'); - const logClass = function(cssClass,...args){ + const logCss = function(cssClass,...args){ const ln = document.createElement('div'); if(cssClass){ for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){ @@ -41,10 +41,10 @@ cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); } cbReverseIt(); - return logClass; + return logCss; })(); - const stdout = (...args)=>logClass('',...args); - const stderr = (...args)=>logClass('error',...args); + const stdout = (...args)=>logCss('',...args); + const stderr = (...args)=>logCss('error',...args); const wait = async (ms)=>{ return new Promise((resolve)=>setTimeout(resolve,ms)); @@ -67,9 +67,6 @@ const wName = msg.worker; const prefix = 'Worker ['+wName+']:'; switch(msg.type){ - case 'stdout': stdout(prefix,...msg.payload); break; - case 'stderr': stderr(prefix,...msg.payload); break; - case 'error': stderr(prefix,"ERROR:",...msg.payload); break; case 'loaded': stdout(prefix,"loaded"); if(++workers.loadedCount === workers.length){ @@ -77,7 +74,16 @@ workers.post('run'); } break; - default: logClass('error',"Unhandled message type:",msg); break; + case 'stdout': stdout(prefix,...msg.payload); break; + case 'stderr': stderr(prefix,...msg.payload); break; + case 'error': stderr(prefix,"ERROR:",...msg.payload); break; + case 'finished': + logCss('tests-pass',prefix,...msg.payload); + break; + case 'failed': + logCss('tests-fail',prefix,"FAILED:",...msg.payload); + break; + default: logCss('error',"Unhandled message type:",msg); break; } }; diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 7ba15bf8c1..9aaa2f4c78 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -26,7 +26,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ wPost('loaded'); const run = async function(){ - const db = new sqlite3.opfs.OpfsDb(dbName); + const db = new sqlite3.opfs.OpfsDb(dbName,'c'); //sqlite3.capi.sqlite3_busy_timeout(db.pointer, 2000); db.transaction((db)=>{ db.exec([ @@ -37,7 +37,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ const maxIterations = 10; const interval = Object.assign(Object.create(null),{ - delay: 300, + delay: 500, handle: undefined, count: 0 }); @@ -58,9 +58,13 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const finish = ()=>{ - if(interval.error) stderr("Ending work due to error:",e.message); - else stdout("Ending work after",interval.count,"interval(s)"); db.close(); + if(interval.error){ + wPost('failed',"Ending work after interval #"+interval.count, + "due to error:",interval.error); + }else{ + wPost('finished',"Ending work after",interval.count,"intervals."); + } }; if(1){/*use setInterval()*/ interval.handle = setInterval(async ()=>{ diff --git a/manifest b/manifest index ccf4fd8d76..75a05f7f6c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sapp\sfor\sexperimenting\swith\smulti-worker\sOPFS\sconcurrency.\sTweak\sOPFS\sVFS\sto\ssignificantly\simprove\sthe\sotherwise\s"unfortunate"\sconcurrency\ssituation. -D 2022-11-21T03:50:52.240 +C Resolve\smissing\sSQLITE_LOCKED\sresult\scode\swhich\striggered\sa\snew\s(since\slast\scheckin)\sexception\sin\sthe\sOPFS\sVFS.\sImprove\soutput\sof\sthe\sOPFS\scontention\stester\sapp. +D 2022-11-21T04:12:38.735 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js 4c75ed11df5efff6bcd8dad4ad904d8b11efac2e1dd4cc2c84d1ee8ace4129ef +F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 97cf1909670575eced940d36f1b5ea35c51a431d1035dc2f7ea6982faee97c1b +F ext/wasm/api/sqlite3-opfs-async-proxy.js 021af8b3d1754e308c09eebee5f8d235fb245bea1f9b1c1414141cc2ebd5649c F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -553,8 +553,8 @@ F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b51073 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html c7cf329e5b206dd8226d94ab9fec02f5f350d8ed69a57c96d84e876afd3d3d1b -F ext/wasm/tests/opfs/concurrency/test.js 44cfcc04503593256abe2dd663349718f80ee7ab25e19eb066de220101bd604a -F ext/wasm/tests/opfs/concurrency/worker.js f8f3e4f9b21726bef354a74ec9c90f6736df5b16b4f655bfd16a3b9c6ee063ff +F ext/wasm/tests/opfs/concurrency/test.js 6f7d49d97c27906f8f5a39d6da176122350f375cd811e51182128bc2db74e464 +F ext/wasm/tests/opfs/concurrency/worker.js 571bcc525520fedfc71e25486265247633a4bf8aee554923aa7219399ed749fd F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 469f9011a885e19b99210c5e3e582afa140b8b5f0aa7a720334848df5ab6ae98 -R c87fca3e6d0a9c36a2598013e36db2a5 +P 96f76e7616f8157a342b9e1c42f7b1feab200d182268871a2b25f67d4ee2564c +R 37b708b698da9468667dd15ac1e5f125 U stephan -Z b0030359261e278f67d2690556943dbd +Z 415bce67c5fc2cb5709db14ccdfe0252 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2e8d22d1e3..7de2c9eef7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96f76e7616f8157a342b9e1c42f7b1feab200d182268871a2b25f67d4ee2564c \ No newline at end of file +2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b \ No newline at end of file From b38ac0986e86d56115396c36355d4e751cc4f7f5 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 21 Nov 2022 05:18:24 +0000 Subject: [PATCH 036/282] More tweaking of OPFS concurrency measures and the related test app. FossilOrigin-Name: a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870 --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 21 +++++----- ext/wasm/tests/opfs/concurrency/index.html | 16 +++++++- ext/wasm/tests/opfs/concurrency/test.js | 9 ++++- ext/wasm/tests/opfs/concurrency/worker.js | 46 ++++++++++++---------- manifest | 18 ++++----- manifest.uuid | 2 +- 6 files changed, 68 insertions(+), 44 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 58cf8ca3c7..c208932e17 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -220,22 +220,19 @@ class GetSyncHandleError extends Error { } }; GetSyncHandleError.convertRc = (e,rc)=>{ - if(1){ - /* This approach returns SQLITE_LOCKED to the C API - when getSyncHandle() fails but makes the very - wild assumption that such a failure _is_ a locking - error. In practice that appears to be the most - common error, by far, but we cannot unambiguously + if(0){ + /* This approach makes the very wild assumption that such a + failure _is_ a locking error. In practice that appears to be + the most common error, by far, but we cannot unambiguously distinguish that from other errors. - This approach demonstrably reduces concurrency-related - errors but is highly questionable. + This approach is highly questionable. */ return (e instanceof GetSyncHandleError) - ? state.sq3Codes.SQLITE_LOCKED + ? state.sq3Codes.SQLITE_IOERR_LOCK : rc; }else{ - return ec; + return rc; } } /** @@ -253,7 +250,7 @@ const getSyncHandle = async (fh)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 4, msBase = 300; + const maxTries = 6, msBase = 300; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -271,7 +268,7 @@ const getSyncHandle = async (fh)=>{ } warn("Error getting sync handle. Waiting",ms, "ms and trying again.",fh.filenameAbs,e); - //await closeAutoLocks(); + await closeAutoLocks(); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index 79a46692cc..a082dfe997 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -18,7 +18,21 @@

    OPFS concurrency tester using multiple independent Workers. - This app is incomplete. + Disclaimer: concurrency in OPFS is currently a pain point + and timing/concurrency mitigation in this environment is + highly unpredictable! +

    +

    + URL flags: pass a number of workers using + the workers=N URL flag and the worker work interval + as interval=N (milliseconds). Enable OPFS VFS + verbosity with verbose=1-3 (output goes to the + dev console). +

    +

    Achtung: if it does not start to do anything within a couple of + seconds, check the dev console: Chrome often fails with "cannot allocate + WasmMemory" at startup. Closing and re-opening the tab usually resolves + it.

    diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index b80dad24c4..8b75ea4c78 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -57,6 +57,12 @@ options.workerCount = ( urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 ) || 3; + options.opfsVerbose = ( + urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 + ) || 1; + options.interval = ( + urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 + ) || 750; const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); @@ -91,7 +97,8 @@ workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir - + '&opfs-verbose=2' + + '&interval='+options.interval + + '&opfs-verbose='+options.opfsVerbose ); for(let i = 0; i < options.workerCount; ++i){ stdout("Launching worker..."); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 9aaa2f4c78..85a5cf19b7 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -8,7 +8,6 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); - const postErr = (...args)=>wPost('error',...args); if(!sqlite3.opfs){ stderr("OPFS support not detected. Aborting."); return; @@ -19,15 +18,33 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const dbName = 'concurrency-tester.db'; - if((new URL(self.location.href).searchParams).has('unlink-db')){ + const urlArgs = new URL(self.location.href).searchParams; + if(urlArgs.has('unlink-db')){ await sqlite3.opfs.unlink(dbName); stdout("Unlinked",dbName); } wPost('loaded'); - + let db; + const interval = Object.assign(Object.create(null),{ + delay: urlArgs.has('interval') ? (+urlArgs.get('interval') || 750) : 750, + handle: undefined, + count: 0 + }); + const finish = ()=>{ + if(db){ + if(!db.pointer) return; + db.close(); + } + if(interval.error){ + wPost('failed',"Ending work after interval #"+interval.count, + "due to error:",interval.error); + }else{ + wPost('finished',"Ending work after",interval.count,"intervals."); + } + }; const run = async function(){ - const db = new sqlite3.opfs.OpfsDb(dbName,'c'); - //sqlite3.capi.sqlite3_busy_timeout(db.pointer, 2000); + db = new sqlite3.opfs.OpfsDb(dbName,'c'); + sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ db.exec([ "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", @@ -36,11 +53,6 @@ self.sqlite3InitModule().then(async function(sqlite3){ }); const maxIterations = 10; - const interval = Object.assign(Object.create(null),{ - delay: 500, - handle: undefined, - count: 0 - }); stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); @@ -57,15 +69,6 @@ self.sqlite3InitModule().then(async function(sqlite3){ interval.error = e; } }; - const finish = ()=>{ - db.close(); - if(interval.error){ - wPost('failed',"Ending work after interval #"+interval.count, - "due to error:",interval.error); - }else{ - wPost('finished',"Ending work after",interval.count,"intervals."); - } - }; if(1){/*use setInterval()*/ interval.handle = setInterval(async ()=>{ await doWork(); @@ -89,7 +92,10 @@ self.sqlite3InitModule().then(async function(sqlite3){ self.onmessage = function({data}){ switch(data.type){ - case 'run': run().catch((e)=>postErr(e.message)); + case 'run': run().catch((e)=>{ + if(!interval.error) interval.error = e; + finish(); + }); break; default: stderr("Unhandled message type '"+data.type+"'."); diff --git a/manifest b/manifest index 75a05f7f6c..0d16ed6e6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Resolve\smissing\sSQLITE_LOCKED\sresult\scode\swhich\striggered\sa\snew\s(since\slast\scheckin)\sexception\sin\sthe\sOPFS\sVFS.\sImprove\soutput\sof\sthe\sOPFS\scontention\stester\sapp. -D 2022-11-21T04:12:38.735 +C More\stweaking\sof\sOPFS\sconcurrency\smeasures\sand\sthe\srelated\stest\sapp. +D 2022-11-21T05:18:24.071 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -506,7 +506,7 @@ F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 021af8b3d1754e308c09eebee5f8d235fb245bea1f9b1c1414141cc2ebd5649c +F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -552,9 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d -F ext/wasm/tests/opfs/concurrency/index.html c7cf329e5b206dd8226d94ab9fec02f5f350d8ed69a57c96d84e876afd3d3d1b -F ext/wasm/tests/opfs/concurrency/test.js 6f7d49d97c27906f8f5a39d6da176122350f375cd811e51182128bc2db74e464 -F ext/wasm/tests/opfs/concurrency/worker.js 571bcc525520fedfc71e25486265247633a4bf8aee554923aa7219399ed749fd +F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 +F ext/wasm/tests/opfs/concurrency/test.js cea04456ffae7bbfd726ef0e859d2afc2a51ead66940fc4a2499884ac44d16dd +F ext/wasm/tests/opfs/concurrency/worker.js a92fa4da1431d3c6db3e1dd0b98794e09d1021e23a8c490335410cce283b11c1 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 96f76e7616f8157a342b9e1c42f7b1feab200d182268871a2b25f67d4ee2564c -R 37b708b698da9468667dd15ac1e5f125 +P 2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b +R cdfdba12cf28ae07442f6b59e52570f8 U stephan -Z 415bce67c5fc2cb5709db14ccdfe0252 +Z bce379ce157581c096a0eabd8577e81e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7de2c9eef7..8ac00ec32b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b \ No newline at end of file +a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870 \ No newline at end of file From a303392a2cadec50082344a266c16e5dabd34d37 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 21 Nov 2022 06:07:22 +0000 Subject: [PATCH 037/282] Add new JS tests dir to those pushed to the test server. FossilOrigin-Name: 07182dca9f2a4ffea1af0684c93e55e105465b2ee9820c70764e3e7bc1c28efc --- ext/wasm/GNUmakefile | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 647044665c..38c1cdaca8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -815,6 +815,7 @@ endif ######################################################################## # Push files to public wasm-testing.sqlite.org server wasm-testing.include = *.js *.html batch-runner.list \ + ./tests \ $(dir.dout) $(dir.sql) $(dir.common) $(dir.fiddle) $(dir.jacc) wasm-testing.exclude = sql/speedtest1.sql wasm-testing.dir = /jail/sites/wasm-testing diff --git a/manifest b/manifest index 0d16ed6e6d..e7f833fab3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\stweaking\sof\sOPFS\sconcurrency\smeasures\sand\sthe\srelated\stest\sapp. -D 2022-11-21T05:18:24.071 +C Add\snew\sJS\stests\sdir\sto\sthose\spushed\sto\sthe\stest\sserver. +D 2022-11-21T06:07:22.911 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,7 +488,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 712795c4893ea65f8d30fe414937a33b677a194dd58372b4074aee17039c845e +F ext/wasm/GNUmakefile bc1696a1189f4c571b3d878b8f3a67c1f4b52c222f52d356027a5a0c707337c7 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b -R cdfdba12cf28ae07442f6b59e52570f8 +P a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870 +R de977a34ec9f33a01661a43c62f057f6 U stephan -Z bce379ce157581c096a0eabd8577e81e +Z 6c2e6c110bc94fae29d67ff59b3bd9c5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8ac00ec32b..b925e621f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870 \ No newline at end of file +07182dca9f2a4ffea1af0684c93e55e105465b2ee9820c70764e3e7bc1c28efc \ No newline at end of file From d01dee5e6aef8aebc2932f743c90f41db70e2a7c Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Nov 2022 13:35:00 +0000 Subject: [PATCH 038/282] Small performance improvement in sqlite3BtreeTransferRow(). FossilOrigin-Name: dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e7f833fab3..4cd29a45bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sJS\stests\sdir\sto\sthose\spushed\sto\sthe\stest\sserver. -D 2022-11-21T06:07:22.911 +C Small\sperformance\simprovement\sin\ssqlite3BtreeTransferRow(). +D 2022-11-21T13:35:00.374 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -580,7 +580,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 308474d81388fa8b4fb92da7b2c73cae864c1ac468a7a5ffab61670af4b488f0 +F src/btree.c 89e0e5af46d1921a0a1d17bc5ec95cd0ec89b050b32373d825c76b5d3f3a2ec9 F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870 -R de977a34ec9f33a01661a43c62f057f6 -U stephan -Z 6c2e6c110bc94fae29d67ff59b3bd9c5 +P 07182dca9f2a4ffea1af0684c93e55e105465b2ee9820c70764e3e7bc1c28efc +R 06d7ab66cd3f5a521e55d8df4889b401 +U drh +Z a6fa3008dc4759fddba2bf18530ff99d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b925e621f1..e7e8aac79f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07182dca9f2a4ffea1af0684c93e55e105465b2ee9820c70764e3e7bc1c28efc \ No newline at end of file +dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 114136bc50..83ebc3d3f5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9314,7 +9314,6 @@ end_insert: ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ - int rc = SQLITE_OK; BtShared *pBt = pDest->pBt; u8 *aOut = pBt->pTmpSpace; /* Pointer to next output buffer */ const u8 *aIn; /* Pointer to next input buffer */ @@ -9337,7 +9336,9 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ if( nIn==nRem && nInpPage->maxLocal ){ memcpy(aOut, aIn, nIn); pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + return SQLITE_OK; }else{ + int rc = SQLITE_OK; Pager *pSrcPager = pSrc->pBt->pPager; u8 *pPgnoOut = 0; Pgno ovflIn = 0; @@ -9405,9 +9406,8 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ releasePage(pPageOut); sqlite3PagerUnref(pPageIn); + return rc; } - - return rc; } /* From e7d53844184be4e6fb3806c5e7950b2f02174953 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Nov 2022 14:13:10 +0000 Subject: [PATCH 039/282] Performance optimization to sqlite3BtreeInsert(). FossilOrigin-Name: b8976ebfe03fbc1e09a38d598a62493a7f19ff7a2a3acd1ec54d0dee190471e9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 37 ++++++++++++++++++------------------- src/btreeInt.h | 4 ++-- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 4cd29a45bd..a769291001 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sin\ssqlite3BtreeTransferRow(). -D 2022-11-21T13:35:00.374 +C Performance\soptimization\sto\ssqlite3BtreeInsert(). +D 2022-11-21T14:13:10.344 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -580,9 +580,9 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 89e0e5af46d1921a0a1d17bc5ec95cd0ec89b050b32373d825c76b5d3f3a2ec9 +F src/btree.c 6fc3f0c6db7ec2296778cca7c696bb10be9de42f531f106c43d0f410cec0ee2a F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de -F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e +F src/btreeInt.h 88ad499c92b489afedbfefc3f067c4d15023ec021afe622db240dc9d2277cfa5 F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07182dca9f2a4ffea1af0684c93e55e105465b2ee9820c70764e3e7bc1c28efc -R 06d7ab66cd3f5a521e55d8df4889b401 +P dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 +R 2eb76fb48a48a84e8cbe3f1574fa7eb8 U drh -Z a6fa3008dc4759fddba2bf18530ff99d +Z 44675fc68f20e0cf82e7b5bf3a25ed71 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e7e8aac79f..62bf73f605 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 \ No newline at end of file +b8976ebfe03fbc1e09a38d598a62493a7f19ff7a2a3acd1ec54d0dee190471e9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 83ebc3d3f5..f56ff8d4c3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6606,7 +6606,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); if( rc ) goto freepage_out; } @@ -7682,7 +7682,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ ** be marked as dirty. Returning an error code will cause a ** rollback, undoing any changes made to the parent page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); if( szCell>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); @@ -7820,7 +7820,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ *pRC = setChildPtrmaps(pTo); } } @@ -8308,7 +8308,7 @@ static int balance_nonroot( cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); if( rc!=SQLITE_OK ){ goto balance_cleanup; @@ -8401,7 +8401,7 @@ static int balance_nonroot( ** updated. This happens below, after the sibling pages have been ** populated, not here. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ MemPage *pOld; MemPage *pNew = pOld = apNew[0]; int cntOldNext = pNew->nCell + pNew->nOverflow; @@ -8594,7 +8594,7 @@ static int balance_nonroot( ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM && !leafCorrection ){ + }else if( ISAUTOVACUUM(pBt) && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken ** care of. */ @@ -8615,7 +8615,7 @@ static int balance_nonroot( } #if 0 - if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){ + if( ISAUTOVACUUM(pBt) && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may @@ -8677,7 +8677,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ if( rc==SQLITE_OK ){ rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); copyNodeContent(pRoot, pChild, &rc); - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); } } @@ -9009,7 +9009,6 @@ int sqlite3BtreeInsert( int idx; MemPage *pPage; Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; unsigned char *oldCell; unsigned char *newCell = 0; @@ -9028,7 +9027,7 @@ int sqlite3BtreeInsert( ** not to clear the cursor here. */ if( pCur->curFlags & BTCF_Multiple ){ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + rc = saveAllCursors(p->pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; if( loc && pCur->iPage<0 ){ /* This can only happen if the schema is corrupt such that there is more @@ -9052,8 +9051,8 @@ int sqlite3BtreeInsert( assert( cursorOwnsBtShared(pCur) ); assert( (pCur->curFlags & BTCF_WriteFlag)!=0 - && pBt->inTransaction==TRANS_WRITE - && (pBt->btsFlags & BTS_READ_ONLY)==0 ); + && p->pBt->inTransaction==TRANS_WRITE + && (p->pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); /* Assert that the caller has been consistent. If this cursor was opened @@ -9170,19 +9169,19 @@ int sqlite3BtreeInsert( pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit || CORRUPT_DB ); - newCell = pBt->pTmpSpace; + newCell = p->pBt->pTmpSpace; assert( newCell!=0 ); assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); if( flags & BTREE_PREFORMAT ){ rc = SQLITE_OK; - szNew = pBt->nPreformatSize; + szNew = p->pBt->nPreformatSize; if( szNew<4 ) szNew = 4; - if( ISAUTOVACUUM && szNew>pPage->maxLocal ){ + if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); if( info.nPayload!=info.nLocal ){ Pgno ovfl = get4byte(&newCell[szNew-4]); - ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + ptrmapPut(p->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); if( NEVER(rc) ) goto end_insert; } } @@ -9191,7 +9190,7 @@ int sqlite3BtreeInsert( if( rc ) goto end_insert; } assert( szNew==pPage->xCellSize(pPage, newCell) ); - assert( szNew <= MX_CELL_SIZE(pBt) ); + assert( szNew <= MX_CELL_SIZE(p->pBt) ); idx = pCur->ix; if( loc==0 ){ CellInfo info; @@ -9211,7 +9210,7 @@ int sqlite3BtreeInsert( testcase( pCur->curFlags & BTCF_ValidOvfl ); invalidateOverflowCache(pCur); if( info.nSize==szNew && info.nLocal==info.nPayload - && (!ISAUTOVACUUM || szNewminLocal) + && (!ISAUTOVACUUM(p->pBt) || szNewminLocal) ){ /* Overwrite the old cell with the new if they are the same size. ** We could also try to do this if the old cell is smaller, then add @@ -9390,7 +9389,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ MemPage *pNew = 0; rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); put4byte(pPgnoOut, pgnoNew); - if( ISAUTOVACUUM && pPageOut ){ + if( ISAUTOVACUUM(pBt) && pPageOut ){ ptrmapPut(pBt, pgnoNew, PTRMAP_OVERFLOW2, pPageOut->pgno, &rc); } releasePage(pPageOut); diff --git a/src/btreeInt.h b/src/btreeInt.h index 1f45553dc9..af295dd507 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -674,9 +674,9 @@ struct BtCursor { ** So, this macro is defined instead. */ #ifndef SQLITE_OMIT_AUTOVACUUM -#define ISAUTOVACUUM (pBt->autoVacuum) +#define ISAUTOVACUUM(pBt) (pBt->autoVacuum) #else -#define ISAUTOVACUUM 0 +#define ISAUTOVACUUM(pBt) 0 #endif From b53d8fa9f262f77fd1ae2835773cf26a8de4e78b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Nov 2022 15:55:57 +0000 Subject: [PATCH 040/282] Performance optimization and size reduction in insertCell() by omitting the "pRC" argument and instead returning the result code as an integer. FossilOrigin-Name: bee94d1bb0daade023cc1e274339daafc249e1978c0765fc45042b5f9060e478 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 28 +++++++++++++--------------- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index a769291001..d57ebb0958 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sto\ssqlite3BtreeInsert(). -D 2022-11-21T14:13:10.344 +C Performance\soptimization\sand\ssize\sreduction\sin\sinsertCell()\sby\somitting\nthe\s"pRC"\sargument\sand\sinstead\sreturning\sthe\sresult\scode\sas\san\sinteger. +D 2022-11-21T15:55:57.369 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -580,7 +580,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 6fc3f0c6db7ec2296778cca7c696bb10be9de42f531f106c43d0f410cec0ee2a +F src/btree.c 522df0f1173495e06c5589f0b17a61f53a212017b82be53171133fcfc0e1e90a F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de F src/btreeInt.h 88ad499c92b489afedbfefc3f067c4d15023ec021afe622db240dc9d2277cfa5 F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 -R 2eb76fb48a48a84e8cbe3f1574fa7eb8 +P b8976ebfe03fbc1e09a38d598a62493a7f19ff7a2a3acd1ec54d0dee190471e9 +R 6d5890069b8a565895abb49912f71e80 U drh -Z 44675fc68f20e0cf82e7b5bf3a25ed71 +Z df511e58d09c8af289615762e74d854c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 62bf73f605..5bdfc3f30e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8976ebfe03fbc1e09a38d598a62493a7f19ff7a2a3acd1ec54d0dee190471e9 \ No newline at end of file +bee94d1bb0daade023cc1e274339daafc249e1978c0765fc45042b5f9060e478 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f56ff8d4c3..cabcf675e3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7046,24 +7046,20 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. -** -** *pRC must be SQLITE_OK when this routine is called. */ -static void insertCell( +static int insertCell( MemPage *pPage, /* Page into which we are copying */ int i, /* New cell becomes the i-th cell of the page */ u8 *pCell, /* Content of the new cell */ int sz, /* Bytes of content in pCell */ u8 *pTemp, /* Temp storage space for pCell, if needed */ - Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ - int *pRC /* Read and write return code from here */ + Pgno iChild /* If non-zero, replace first 4 bytes with this value */ ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ u8 *data; /* The content of the whole page */ u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ - assert( *pRC==SQLITE_OK ); assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); @@ -7098,14 +7094,13 @@ static void insertCell( }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ - *pRC = rc; - return; + return rc; } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; assert( &data[pPage->cellOffset]==pPage->aCellIdx ); rc = allocateSpace(pPage, sz, &idx); - if( rc ){ *pRC = rc; return; } + if( rc ){ return rc; } /* The allocateSpace() routine guarantees the following properties ** if it returns successfully */ assert( idx >= 0 ); @@ -7132,13 +7127,16 @@ static void insertCell( assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ + int rc = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pPage, pCell, pRC); + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc); + if( rc ) return rc; } #endif } + return SQLITE_OK; } /* @@ -7710,8 +7708,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ /* Insert the new divider cell into pParent. */ if( rc==SQLITE_OK ){ - insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); + rc = insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno); } /* Set the right-child pointer of pParent to point to the new page. */ @@ -8498,7 +8496,7 @@ static int balance_nonroot( rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } - insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + rc = insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno); if( rc!=SQLITE_OK ) goto balance_cleanup; assert( sqlite3PagerIswriteable(pParent->pDbPage) ); } @@ -9240,7 +9238,7 @@ int sqlite3BtreeInsert( }else{ assert( pPage->leaf ); } - insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + rc = insertCell(pPage, idx, newCell, szNew, 0, 0); assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); @@ -9561,7 +9559,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); if( rc==SQLITE_OK ){ - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + rc = insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n); } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; From 9f2be5db04ddc73466f068a6cda3b196d09eaf86 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 21 Nov 2022 16:00:26 +0000 Subject: [PATCH 041/282] OPFS contention test: give each worker a distinct recognizable name instead of a random one. FossilOrigin-Name: 5f564bf7de7ce3ad7bedb5f06b3086ceaec55da768a60d74059fa4fba4328567 --- ext/wasm/tests/opfs/concurrency/test.js | 7 ++++--- ext/wasm/tests/opfs/concurrency/worker.js | 4 ++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 8b75ea4c78..27bc47b19d 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -70,8 +70,7 @@ workers.loadedCount = 0; workers.onmessage = function(msg){ msg = msg.data; - const wName = msg.worker; - const prefix = 'Worker ['+wName+']:'; + const prefix = 'Worker #'+msg.worker+':'; switch(msg.type){ case 'loaded': stdout(prefix,"loaded"); @@ -102,7 +101,9 @@ ); for(let i = 0; i < options.workerCount; ++i){ stdout("Launching worker..."); - workers.push(new Worker(workers.uri+(i ? '' : '&unlink-db'))); + workers.push(new Worker( + workers.uri+'&workerId='+(i+1)+(i ? '' : '&unlink-db') + )); } // Have to delay onmessage assignment until after the loop // to avoid that early workers get an undue head start. diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 85a5cf19b7..c315508e0b 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -2,7 +2,8 @@ importScripts( (new URL(self.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' ); self.sqlite3InitModule().then(async function(sqlite3){ - const wName = Math.round(Math.random()*10000); + const urlArgs = new URL(self.location.href).searchParams; + const wName = urlArgs.get('workerId') || Math.round(Math.random()*10000); const wPost = (type,...payload)=>{ postMessage({type, worker: wName, payload}); }; @@ -18,7 +19,6 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const dbName = 'concurrency-tester.db'; - const urlArgs = new URL(self.location.href).searchParams; if(urlArgs.has('unlink-db')){ await sqlite3.opfs.unlink(dbName); stdout("Unlinked",dbName); diff --git a/manifest b/manifest index d57ebb0958..ab782561db 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sand\ssize\sreduction\sin\sinsertCell()\sby\somitting\nthe\s"pRC"\sargument\sand\sinstead\sreturning\sthe\sresult\scode\sas\san\sinteger. -D 2022-11-21T15:55:57.369 +C OPFS\scontention\stest:\sgive\seach\sworker\sa\sdistinct\srecognizable\sname\sinstead\sof\sa\srandom\sone. +D 2022-11-21T16:00:26.047 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -553,8 +553,8 @@ F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b51073 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js cea04456ffae7bbfd726ef0e859d2afc2a51ead66940fc4a2499884ac44d16dd -F ext/wasm/tests/opfs/concurrency/worker.js a92fa4da1431d3c6db3e1dd0b98794e09d1021e23a8c490335410cce283b11c1 +F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff +F ext/wasm/tests/opfs/concurrency/worker.js df065bb386ff994951f7fbdd76e12f16e58fbef0e929b2caf74553359da40afc F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8976ebfe03fbc1e09a38d598a62493a7f19ff7a2a3acd1ec54d0dee190471e9 -R 6d5890069b8a565895abb49912f71e80 -U drh -Z df511e58d09c8af289615762e74d854c +P bee94d1bb0daade023cc1e274339daafc249e1978c0765fc45042b5f9060e478 +R 24e9b381879a473eda77c6d52873b2a6 +U stephan +Z 5200ed97b9b65f90e5d50e34624e223d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5bdfc3f30e..2dbda6c4cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bee94d1bb0daade023cc1e274339daafc249e1978c0765fc45042b5f9060e478 \ No newline at end of file +5f564bf7de7ce3ad7bedb5f06b3086ceaec55da768a60d74059fa4fba4328567 \ No newline at end of file From 03af6d7157c0053fad2a55286004e0f16ea6e95f Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Nov 2022 16:40:12 +0000 Subject: [PATCH 042/282] Change the name of the Parse.pIdxExpr field to pIdxEpr so that the name is distinct from all other identifiers in the code and thus less confusing. FossilOrigin-Name: a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 12 ++++++------ src/sqliteInt.h | 2 +- src/where.c | 22 +++++++++++----------- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index ab782561db..d3ee673914 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\scontention\stest:\sgive\seach\sworker\sa\sdistinct\srecognizable\sname\sinstead\sof\sa\srandom\sone. -D 2022-11-21T16:00:26.047 +C Change\sthe\sname\sof\sthe\sParse.pIdxExpr\sfield\sto\spIdxEpr\sso\sthat\sthe\sname\sis\ndistinct\sfrom\sall\sother\sidentifiers\sin\sthe\scode\sand\sthus\sless\sconfusing. +D 2022-11-21T16:40:12.023 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 847f87d9df3ede2b2b0a8db088af0b9c1923b21009f8ea1b9b7b28cb0a383170 +F src/expr.c bc6527e3dff813c8102418e6e201870626a7fa5f69329ea7b082d602e7ed1cd9 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -646,7 +646,7 @@ F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 1a2cec0f7682da69975eb203fef2058949051ff500445b997d8047cbd8813315 +F src/sqliteInt.h 42bcbb6cebef64c99068e7bc141a2541d40d65c7c26a9ca5ada4d83ca353555d F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -728,7 +728,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 1ef5aae7fac877057b9f360f06b26d4275888460d8fb6e92bbb9e70e07afe946 +F src/where.c ea0f518df9e00aa44013a1d384090b4b3a499ee11d4daa0a7d99c4eb9f7ab4ba F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bee94d1bb0daade023cc1e274339daafc249e1978c0765fc45042b5f9060e478 -R 24e9b381879a473eda77c6d52873b2a6 -U stephan -Z 5200ed97b9b65f90e5d50e34624e223d +P 5f564bf7de7ce3ad7bedb5f06b3086ceaec55da768a60d74059fa4fba4328567 +R fadcb96891525abc2e619c860d5616b7 +U drh +Z 75407774f59d7d22e403811dc4acdbf3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2dbda6c4cc..987c32b82f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f564bf7de7ce3ad7bedb5f06b3086ceaec55da768a60d74059fa4fba4328567 \ No newline at end of file +a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 7a4e59f28d..fb718eb23e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4037,7 +4037,7 @@ static int exprCodeInlineFunction( } /* -** Check to see if pExpr is one of the indexed expressions on pParse->pIdxExpr. +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. ** If it is, then resolve the expression by reading from the index and ** return the register into which the value has been read. If pExpr is ** not an indexed expression, then return negative. @@ -4049,7 +4049,7 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( ){ IndexedExpr *p; Vdbe *v; - for(p=pParse->pIdxExpr; p; p=p->pIENext){ + for(p=pParse->pIdxEpr; p; p=p->pIENext){ int iDataCur = p->iDataCur; if( iDataCur<0 ) continue; if( pParse->iSelfTab ){ @@ -4069,10 +4069,10 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); sqlite3VdbeGoto(v, 0); - p = pParse->pIdxExpr; - pParse->pIdxExpr = 0; + p = pParse->pIdxEpr; + pParse->pIdxEpr = 0; sqlite3ExprCode(pParse, pExpr, target); - pParse->pIdxExpr = p; + pParse->pIdxEpr = p; sqlite3VdbeJumpHere(v, addr+2); }else{ sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); @@ -4111,7 +4111,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ expr_code_doover: if( pExpr==0 ){ op = TK_NULL; - }else if( pParse->pIdxExpr!=0 + }else if( pParse->pIdxEpr!=0 && !ExprHasProperty(pExpr, EP_Leaf) && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 469ce9bba1..6ff49ab5d0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3656,7 +3656,7 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ - IndexedExpr *pIdxExpr;/* List of expressions used by active indexes */ + IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ diff --git a/src/where.c b/src/where.c index 7ba72d2766..ae3d513cae 100644 --- a/src/where.c +++ b/src/where.c @@ -4591,8 +4591,8 @@ static i8 wherePathSatisfiesOrderBy( if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; }else{ - Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr; - if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){ + Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr; + if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){ continue; } } @@ -5461,13 +5461,13 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( /* ** This is an sqlite3ParserAddCleanup() callback that is invoked to -** free the Parse->pIdxExpr list when the Parse object is destroyed. +** free the Parse->pIdxEpr list when the Parse object is destroyed. */ static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ Parse *pParse = (Parse*)pObject; - while( pParse->pIdxExpr!=0 ){ - IndexedExpr *p = pParse->pIdxExpr; - pParse->pIdxExpr = p->pIENext; + while( pParse->pIdxEpr!=0 ){ + IndexedExpr *p = pParse->pIdxEpr; + pParse->pIdxEpr = p->pIENext; sqlite3ExprDelete(db, p->pExpr); sqlite3DbFreeNN(db, p); } @@ -5479,13 +5479,13 @@ static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ ** number for the index and iDataCur is the cursor number for the corresponding ** table. ** -** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for +** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for ** each of the expressions in the index so that the expression code generator ** will know to replace occurrences of the indexed expression with ** references to the corresponding column of the index. */ static SQLITE_NOINLINE void whereAddIndexedExpr( - Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */ + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */ Index *pIdx, /* The index-on-expression that contains the expressions */ int iIdxCur, /* Cursor number for pIdx */ SrcItem *pTabItem /* The FROM clause entry for the table */ @@ -5514,7 +5514,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( if( sqlite3ExprIsConstant(pExpr) ) continue; p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; - p->pIENext = pParse->pIdxExpr; + p->pIENext = pParse->pIdxEpr; p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; @@ -5523,7 +5523,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS p->zIdxName = pIdx->zName; #endif - pParse->pIdxExpr = p; + pParse->pIdxEpr = p; if( p->pIENext==0 ){ sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); } @@ -6474,7 +6474,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ last = pWInfo->iEndWhere; } if( pIdx->bHasExpr ){ - IndexedExpr *p = pParse->pIdxExpr; + IndexedExpr *p = pParse->pIdxEpr; while( p ){ if( p->iIdxCur==pLevel->iIdxCur ){ p->iDataCur = -1; From da217c958eb14824c3bd308a279d9cad19fea35b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Nov 2022 17:40:23 +0000 Subject: [PATCH 043/282] Split out the debugging code that dumps an AggInfo object into a separate subroutine called printAggInfo() so that it can be invoked interactively during debugging. No changes to production code. FossilOrigin-Name: dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 41 +++++++++++++++++++++++++---------------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index d3ee673914..d6216cacf6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sParse.pIdxExpr\sfield\sto\spIdxEpr\sso\sthat\sthe\sname\sis\ndistinct\sfrom\sall\sother\sidentifiers\sin\sthe\scode\sand\sthus\sless\sconfusing. -D 2022-11-21T16:40:12.023 +C Split\sout\sthe\sdebugging\scode\sthat\sdumps\san\sAggInfo\sobject\sinto\sa\sseparate\nsubroutine\scalled\sprintAggInfo()\sso\sthat\sit\scan\sbe\sinvoked\sinteractively\nduring\sdebugging.\s\sNo\schanges\sto\sproduction\scode. +D 2022-11-21T17:40:23.307 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 +F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f564bf7de7ce3ad7bedb5f06b3086ceaec55da768a60d74059fa4fba4328567 -R fadcb96891525abc2e619c860d5616b7 +P a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe +R 9fdeab1b213168256e8eff07917f0804 U drh -Z 75407774f59d7d22e403811dc4acdbf3 +Z 716f42c6984876d9d59330e78a2ea955 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 987c32b82f..f17714d1e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe \ No newline at end of file +dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b0f79f840b..f996ac0852 100644 --- a/src/select.c +++ b/src/select.c @@ -6629,6 +6629,30 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ return 0; } +#if TREETRACE_ENABLED +/* +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, pCol->iMem, + pCol->iSorterColumn); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, pAggInfo->aFunc[ii].iMem); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7428,28 +7452,13 @@ int sqlite3Select( if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x400 ){ - int ii; SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); if( minMaxFlag ){ sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); } - for(ii=0; iinColumn; ii++){ - struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; - sqlite3DebugPrintf( - "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" - " iSorterColumn=%d\n", - ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, pCol->iMem, - pCol->iSorterColumn); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } + printAggInfo(pAggInfo); } #endif From c59b7a8053f0bad8d5df2d834b446a0594ade49c Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 12:47:32 +0000 Subject: [PATCH 044/282] Convert an ALWAYS() in sqlite3DbSpanDup() into an assert(), for a performance increase and size reduction. FossilOrigin-Name: 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d6216cacf6..f5beaaa03b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Split\sout\sthe\sdebugging\scode\sthat\sdumps\san\sAggInfo\sobject\sinto\sa\sseparate\nsubroutine\scalled\sprintAggInfo()\sso\sthat\sit\scan\sbe\sinvoked\sinteractively\nduring\sdebugging.\s\sNo\schanges\sto\sproduction\scode. -D 2022-11-21T17:40:23.307 +C Convert\san\sALWAYS()\sin\ssqlite3DbSpanDup()\sinto\san\sassert(),\sfor\sa\sperformance\nincrease\sand\ssize\sreduction. +D 2022-11-22T12:47:32.742 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -605,7 +605,7 @@ F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9 F src/main.c fa53bb2ae09549dab5629271c3cfd681f89059f5192afaaaf5c0d396bb3957fe -F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90 +F src/malloc.c 3d4ec162214024ee071d85711b93bec25cd3371280aee3702b63bcf312ca8238 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2962d0187534f3496282b48efbf38dabcd226bf8972be925c8433c55a8853fe -R 9fdeab1b213168256e8eff07917f0804 +P dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 +R 6f72c126be40c707db8479e3d23315a3 U drh -Z 716f42c6984876d9d59330e78a2ea955 +Z 9806e837fcd3094e05cfafc89f1e72fe # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f17714d1e3..02e6c3a494 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 \ No newline at end of file +21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index af83743fc0..68d9f5f55c 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -793,9 +793,14 @@ char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ */ char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ int n; +#ifdef SQLITE_DEBUG + /* Because of the way the parser works, the span is guaranteed to contain + ** at least one non-space character */ + for(n=0; sqlite3Isspace(zStart[n]); n++){ assert( &zStart[n]0) && sqlite3Isspace(zStart[n-1]) ) n--; + while( sqlite3Isspace(zStart[n-1]) ) n--; return sqlite3DbStrNDup(db, zStart, n); } From ff5e4eccedbfc2294a8072888de596a8825adee0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 13:33:58 +0000 Subject: [PATCH 045/282] This is the first in what is anticipated to be a long sequence of incremental changes aimed at improving aggregate query processing, and in particular helping aggregate queries take better advantage of indexes on expression. The end goal is to resolve ticket [99378177930f87bd], though it remains to be seen whether or not I can get there with this approach. FossilOrigin-Name: cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/select.c | 10 ++++------ src/sqliteInt.h | 2 +- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f5beaaa03b..bf95667970 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\san\sALWAYS()\sin\ssqlite3DbSpanDup()\sinto\san\sassert(),\sfor\sa\sperformance\nincrease\sand\ssize\sreduction. -D 2022-11-22T12:47:32.742 +C This\sis\sthe\sfirst\sin\swhat\sis\santicipated\sto\sbe\sa\slong\ssequence\sof\sincremental\nchanges\saimed\sat\simproving\saggregate\squery\sprocessing,\sand\sin\sparticular\nhelping\saggregate\squeries\stake\sbetter\sadvantage\sof\sindexes\son\sexpression.\nThe\send\sgoal\sis\sto\sresolve\sticket\s[99378177930f87bd],\sthough\sit\sremains\sto\nbe\sseen\swhether\sor\snot\sI\scan\sget\sthere\swith\sthis\sapproach. +D 2022-11-22T13:33:58.528 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d +F src/select.c 0f78363cb997bba3df6f974c5022b1cef9c6d7bc9ae73bcfd75b1e4b5f7d95fc F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 42bcbb6cebef64c99068e7bc141a2541d40d65c7c26a9ca5ada4d83ca353555d +F src/sqliteInt.h 62ff4e899d1a10e7fe60617f9d58ad49f20de4ad30ee02b68dc403ece984b617 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 -R 6f72c126be40c707db8479e3d23315a3 +P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 +R 8a6d83ef799149939ad05ed6225df7c1 +T *branch * agg-with-indexed-expr +T *sym-agg-with-indexed-expr * +T -sym-trunk * U drh -Z 9806e837fcd3094e05cfafc89f1e72fe +Z c8ad55df8af287c23ae8c2fae9e521d0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 02e6c3a494..3f8ba1bdb5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 \ No newline at end of file +cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 \ No newline at end of file diff --git a/src/select.c b/src/select.c index f996ac0852..5306eb399b 100644 --- a/src/select.c +++ b/src/select.c @@ -6223,18 +6223,17 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pParse->nErr ) return; #ifdef SQLITE_DEBUG /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..AggInfo.mxReg */ - assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); + ** AggInfo.mnReg..(AggInfo.mnReg+nReg-1) */ for(i=0; inColumn; i++){ assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); + && pAggInfo->aCol[i].iMemmnReg+nReg ); } for(i=0; inFunc; i++){ assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); + && pAggInfo->aFunc[i].iMemmnReg+nReg ); } #endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mnReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -7448,7 +7447,6 @@ int sqlite3Select( #endif sNC.ncFlags &= ~NC_InAggFunc; } - pAggInfo->mxReg = pParse->nMem; if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x400 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6ff49ab5d0..d4ffaa2251 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2719,7 +2719,7 @@ struct AggInfo { int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + int mnReg; /* First in a range of regsiters for aCol and aFunc */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ From 42b78237cdd11b79e4c37d0373a338c1409f8283 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 14:10:22 +0000 Subject: [PATCH 046/282] Factor out the allocation of registers for aggregates into a separate subroutine. FossilOrigin-Name: 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/expr.c | 4 ++-- src/select.c | 21 ++++++++++++++++++++- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index bf95667970..7f4b62ade6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sis\sthe\sfirst\sin\swhat\sis\santicipated\sto\sbe\sa\slong\ssequence\sof\sincremental\nchanges\saimed\sat\simproving\saggregate\squery\sprocessing,\sand\sin\sparticular\nhelping\saggregate\squeries\stake\sbetter\sadvantage\sof\sindexes\son\sexpression.\nThe\send\sgoal\sis\sto\sresolve\sticket\s[99378177930f87bd],\sthough\sit\sremains\sto\nbe\sseen\swhether\sor\snot\sI\scan\sget\sthere\swith\sthis\sapproach. -D 2022-11-22T13:33:58.528 +C Factor\sout\sthe\sallocation\sof\sregisters\sfor\saggregates\sinto\sa\sseparate\nsubroutine. +D 2022-11-22T14:10:22.456 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c bc6527e3dff813c8102418e6e201870626a7fa5f69329ea7b082d602e7ed1cd9 +F src/expr.c 5de79bb844b02b6ebc6778161e549d1e3f2a8c0f203234d0212bd97161ed81bc F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 0f78363cb997bba3df6f974c5022b1cef9c6d7bc9ae73bcfd75b1e4b5f7d95fc +F src/select.c 32d4f232ee178a4a57595061f26b10d6c51c3530cc3e721311c586201f41337a F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,11 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 -R 8a6d83ef799149939ad05ed6225df7c1 -T *branch * agg-with-indexed-expr -T *sym-agg-with-indexed-expr * -T -sym-trunk * +P cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 +R 8ec59d9575fc141fe0aff25c195ea672 U drh -Z c8ad55df8af287c23ae8c2fae9e521d0 +Z dd7dd4863c6318a4016da919a253840a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f8ba1bdb5..d932c72af1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 \ No newline at end of file +4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fb718eb23e..f6b065208b 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4441,6 +4441,7 @@ expr_code_doover: assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ + assert( pInfo->aFunc[pExpr->iAgg].iMem>0 ); return pInfo->aFunc[pExpr->iAgg].iMem; } break; @@ -4730,6 +4731,7 @@ expr_code_doover: if( pAggInfo ){ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); if( !pAggInfo->directMode ){ + assert( pAggInfo->aCol[pExpr->iAgg].iMem>0 ); inReg = pAggInfo->aCol[pExpr->iAgg].iMem; break; } @@ -6294,7 +6296,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pCol->pTab = pExpr->y.pTab; pCol->iTable = pExpr->iTable; pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; pCol->iSorterColumn = -1; pCol->pCExpr = pExpr; if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ @@ -6357,7 +6358,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; - pItem->iMem = ++pParse->nMem; assert( ExprUseUToken(pExpr) ); pItem->pFunc = sqlite3FindFunction(pParse->db, pExpr->u.zToken, diff --git a/src/select.c b/src/select.c index 5306eb399b..0872a95454 100644 --- a/src/select.c +++ b/src/select.c @@ -6204,6 +6204,21 @@ void sqlite3SelectPrep( sqlite3SelectAddTypeInfo(pParse, p); } +/* +** Assign register numbers to all pAggInfo->aCol[] and pAggInfo->aFunc[] +** entries. +*/ +static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ + int i, m; + assert( pAggInfo!=0 ); + assert( pAggInfo->mnReg==0 ); + m = pParse->nMem; + pAggInfo->mnReg = m+1; + for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; + for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; + pParse->nMem = m; +} + /* ** Reset the aggregate accumulator. ** @@ -6219,6 +6234,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); + assert( pAggInfo->mnReg>0 ); /* assignAggregateRegisters() has been run */ if( nReg==0 ) return; if( pParse->nErr ) return; #ifdef SQLITE_DEBUG @@ -7413,7 +7429,6 @@ int sqlite3Select( sNC.pSrcList = pTabList; sNC.uNC.pAggInfo = pAggInfo; VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) - pAggInfo->mnReg = pParse->nMem+1; pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); @@ -7534,6 +7549,7 @@ int sqlite3Select( sqlite3ExprListDelete(db, pDistinct); goto select_end; } + assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ @@ -7772,6 +7788,8 @@ int sqlite3Select( if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } + assignAggregateRegisters(pParse, pAggInfo); + assert( pAggInfo->aFunc[0].iMem>=0 ); sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); @@ -7808,6 +7826,7 @@ int sqlite3Select( pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; } + assignAggregateRegisters(pParse, pAggInfo); /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row From 89e5dfac01904ef55be9f2e5c007b25d57bc0263 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 14:31:13 +0000 Subject: [PATCH 047/282] Omit the unnecessary AggInfo.mnReg field. FossilOrigin-Name: d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 26 ++++++++++++++++---------- src/sqliteInt.h | 1 - 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 7f4b62ade6..e95765ddfb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sout\sthe\sallocation\sof\sregisters\sfor\saggregates\sinto\sa\sseparate\nsubroutine. -D 2022-11-22T14:10:22.456 +C Omit\sthe\sunnecessary\sAggInfo.mnReg\sfield. +D 2022-11-22T14:31:13.165 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 32d4f232ee178a4a57595061f26b10d6c51c3530cc3e721311c586201f41337a +F src/select.c 27822fc0fe69d316448294c44c638bd1aad9b980ad62f0a7bb501555245eeb26 F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 62ff4e899d1a10e7fe60617f9d58ad49f20de4ad30ee02b68dc403ece984b617 +F src/sqliteInt.h a16a36f7b2a77b66f8ed0a460aad9f0a5f2897026a51d656cb3556461d1f6be3 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cba837eae93f6b842d4e78ef00661a4f09deb99c53f12b3e8f46763749602597 -R 8ec59d9575fc141fe0aff25c195ea672 +P 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d +R 5936ef480bfa81b2caa13a6a172023d0 U drh -Z dd7dd4863c6318a4016da919a253840a +Z 8bcded315dd8c6f4a10b877ff07bc1b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d932c72af1..8897d99aa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d \ No newline at end of file +d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0872a95454..dd02265958 100644 --- a/src/select.c +++ b/src/select.c @@ -6211,9 +6211,7 @@ void sqlite3SelectPrep( static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ int i, m; assert( pAggInfo!=0 ); - assert( pAggInfo->mnReg==0 ); m = pParse->nMem; - pAggInfo->mnReg = m+1; for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; pParse->nMem = m; @@ -6230,26 +6228,34 @@ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; + int iFirstReg; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); - assert( pAggInfo->mnReg>0 ); /* assignAggregateRegisters() has been run */ if( nReg==0 ) return; if( pParse->nErr ) return; + if( pAggInfo->nColumn==0 ){ + iFirstReg = pAggInfo->aFunc[0].iMem; + }else{ + iFirstReg = pAggInfo->aCol[0].iMem; + } #ifdef SQLITE_DEBUG - /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..(AggInfo.mnReg+nReg-1) */ + /* Verify that all AggInfo register numbers have been assigned and that + ** they are all sequential. */ + assert( iFirstReg>0 ); for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMemmnReg+nReg ); + assert( pAggInfo->aCol[i].iMem>=iFirstReg ); + assert( i==0 || pAggInfo->aCol[i].iMem==pAggInfo->aCol[i-1].iMem+1 ); } for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMemmnReg+nReg ); + assert( pAggInfo->aFunc[i].iMem>=iFirstReg ); + assert( i>0 || pAggInfo->nColumn==0 + || pAggInfo->aFunc[i].iMem==pAggInfo->aCol[pAggInfo->nColumn-1].iMem+1 ); + assert( i==0 || pAggInfo->aFunc[i].iMem==pAggInfo->aFunc[i-1].iMem+1 ); } #endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mnReg+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, iFirstReg, iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d4ffaa2251..68dbb3ab9f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2719,7 +2719,6 @@ struct AggInfo { int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg; /* First in a range of regsiters for aCol and aFunc */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ From 2a072506ca4ca14839f88ca1c027436e9d68c7c4 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 22 Nov 2022 15:41:32 +0000 Subject: [PATCH 048/282] Fix last function pointer in sqlite3Apis init. Reported at https://sqlite.org/forum/info/eba0faa96d FossilOrigin-Name: 6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/loadext.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f5beaaa03b..e1376b3753 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\san\sALWAYS()\sin\ssqlite3DbSpanDup()\sinto\san\sassert(),\sfor\sa\sperformance\nincrease\sand\ssize\sreduction. -D 2022-11-22T12:47:32.742 +C Fix\slast\sfunction\spointer\sin\ssqlite3Apis\sinit.\sReported\sat\shttps://sqlite.org/forum/info/eba0faa96d +D 2022-11-22T15:41:32.493 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -603,7 +603,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 90a32bc7faa755cd5292ade21d2b3c6edba8fd1d70754a364caccabfde2c3bb2 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9 +F src/loadext.c 25663175950c5c4404b9377840b7b4c6fe5c53b415caf43634c62f442c02a9a7 F src/main.c fa53bb2ae09549dab5629271c3cfd681f89059f5192afaaaf5c0d396bb3957fe F src/malloc.c 3d4ec162214024ee071d85711b93bec25cd3371280aee3702b63bcf312ca8238 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc33cfbedef4b444adeadea17f8183b7c4ce5b87432d0c712f986b34c7374ff1 -R 6f72c126be40c707db8479e3d23315a3 -U drh -Z 9806e837fcd3094e05cfafc89f1e72fe +P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 +R b2f32015706aa1808841626a0ff1391c +U larrybr +Z d9d2dcaeb03b2f8d48cf12bc3dcb589d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 02e6c3a494..1264b5622a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 \ No newline at end of file +6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 421ccca804..c14338f8b4 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -510,7 +510,7 @@ static const sqlite3_api_routines sqlite3Apis = { #endif sqlite3_db_name, /* Version 3.40.0 and later */ - sqlite3_value_type + sqlite3_value_encoding }; /* True if x is the directory separator character From 3c8e438583c617eea850ff52bec56f5db5a8d099 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 15:43:16 +0000 Subject: [PATCH 049/282] Since the memory registers used by the columns and functions of an AggInfo object are sequential, it is not neecessary to remember each register separately. We can simply remember the first one and do the math when others are needed. FossilOrigin-Name: dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 9 +++------ src/select.c | 49 ++++++++++++++----------------------------------- src/sqliteInt.h | 11 ++++++++--- 5 files changed, 34 insertions(+), 53 deletions(-) diff --git a/manifest b/manifest index e95765ddfb..8e8fab603a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\sunnecessary\sAggInfo.mnReg\sfield. -D 2022-11-22T14:31:13.165 +C Since\sthe\smemory\sregisters\sused\sby\sthe\scolumns\sand\sfunctions\sof\san\nAggInfo\sobject\sare\ssequential,\sit\sis\snot\sneecessary\sto\sremember\seach\sregister\nseparately.\s\sWe\scan\ssimply\sremember\sthe\sfirst\sone\sand\sdo\sthe\smath\swhen\nothers\sare\sneeded. +D 2022-11-22T15:43:16.425 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 5de79bb844b02b6ebc6778161e549d1e3f2a8c0f203234d0212bd97161ed81bc +F src/expr.c 63cce2c219748d8f8e00a30e4dc43d65b686b62489f154eebe892933bfb4a249 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 27822fc0fe69d316448294c44c638bd1aad9b980ad62f0a7bb501555245eeb26 +F src/select.c cf17743a695b3f4ff2dbd06d2ca8152997d37e3b3fd951efe0a67966740bd2d7 F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h a16a36f7b2a77b66f8ed0a460aad9f0a5f2897026a51d656cb3556461d1f6be3 +F src/sqliteInt.h fd605195929a3ce7f20da20b6a4da32045f9d4b343b51d94a0c7fdbad6207f24 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4475799d5b41e93eabc32fac502ac2de240642b3b64739216e32e9af92ee191d -R 5936ef480bfa81b2caa13a6a172023d0 +P d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b +R a8a65a86f26eb6db314813261a6e0287 U drh -Z 8bcded315dd8c6f4a10b877ff07bc1b3 +Z 3374f8495add90924d12780ade06b13a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8897d99aa9..b43b5e65eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d79c58ef08b917bacc0f24d210d8eb23f659f955c219b4757af42eee8f17099b \ No newline at end of file +dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f6b065208b..42099536ed 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4128,8 +4128,7 @@ expr_code_doover: assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - return pCol->iMem; + return AggInfoColumnReg(pAggInfo, pExpr->iAgg); }else if( pAggInfo->useSortingIdx ){ Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, @@ -4441,8 +4440,7 @@ expr_code_doover: assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ - assert( pInfo->aFunc[pExpr->iAgg].iMem>0 ); - return pInfo->aFunc[pExpr->iAgg].iMem; + return AggInfoFuncReg(pInfo, pExpr->iAgg); } break; } @@ -4731,8 +4729,7 @@ expr_code_doover: if( pAggInfo ){ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); if( !pAggInfo->directMode ){ - assert( pAggInfo->aCol[pExpr->iAgg].iMem>0 ); - inReg = pAggInfo->aCol[pExpr->iAgg].iMem; + inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg); break; } if( pExpr->pAggInfo->useSortingIdx ){ diff --git a/src/select.c b/src/select.c index dd02265958..e40de38a59 100644 --- a/src/select.c +++ b/src/select.c @@ -6209,12 +6209,10 @@ void sqlite3SelectPrep( ** entries. */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ - int i, m; assert( pAggInfo!=0 ); - m = pParse->nMem; - for(i=0; inColumn; i++) pAggInfo->aCol[i].iMem = ++m; - for(i=0; inFunc; i++) pAggInfo->aFunc[i].iMem = ++m; - pParse->nMem = m; + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->iFirstReg = pParse->nMem + 1; + pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc; } /* @@ -6228,34 +6226,15 @@ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; - int iFirstReg; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; + assert( pAggInfo->iFirstReg>0 ); if( pParse->nErr ) return; - if( pAggInfo->nColumn==0 ){ - iFirstReg = pAggInfo->aFunc[0].iMem; - }else{ - iFirstReg = pAggInfo->aCol[0].iMem; - } -#ifdef SQLITE_DEBUG - /* Verify that all AggInfo register numbers have been assigned and that - ** they are all sequential. */ - assert( iFirstReg>0 ); - for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=iFirstReg ); - assert( i==0 || pAggInfo->aCol[i].iMem==pAggInfo->aCol[i-1].iMem+1 ); - } - for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=iFirstReg ); - assert( i>0 || pAggInfo->nColumn==0 - || pAggInfo->aFunc[i].iMem==pAggInfo->aCol[pAggInfo->nColumn-1].iMem+1 ); - assert( i==0 || pAggInfo->aFunc[i].iMem==pAggInfo->aFunc[i-1].iMem+1 ); - } -#endif - sqlite3VdbeAddOp3(v, OP_Null, 0, iFirstReg, iFirstReg+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, + pAggInfo->iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -6287,7 +6266,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; - sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), + pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } @@ -6375,7 +6355,7 @@ static void updateAccumulator( if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); @@ -6390,7 +6370,7 @@ static void updateAccumulator( addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); } pAggInfo->directMode = 0; @@ -6662,13 +6642,13 @@ static void printAggInfo(AggInfo *pAggInfo){ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" " iSorterColumn=%d\n", ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, pCol->iMem, + pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), pCol->iSorterColumn); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); + sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + ii, AggInfoFuncReg(pAggInfo,ii)); sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); } } @@ -7795,8 +7775,7 @@ int sqlite3Select( sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } assignAggregateRegisters(pParse, pAggInfo); - assert( pAggInfo->aFunc[0].iMem>=0 ); - sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0)); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 68dbb3ab9f..45075cc361 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2716,15 +2716,15 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ + u16 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - int nSortingColumn; /* Number of columns in the sorting index */ + int iFirstReg; /* First register in range for aCol[] and aFunc[] */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iMem; /* Memory location that acts as accumulator */ i16 iColumn; /* Column number within the source table */ i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; @@ -2735,7 +2735,6 @@ struct AggInfo { struct AggInfo_func { /* For each aggregate function */ Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ - int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ } *aFunc; @@ -2743,6 +2742,12 @@ struct AggInfo { u32 selId; /* Select to which this AggInfo belongs */ }; +/* +** Macros to compute aCol[] and aFunc[] register numbers: +*/ +#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) ((A)->iFirstReg+(A)->nColumn+(I)) + /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater From 63b74e02a70b499d3783b1116e49840cb7a97e87 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 22 Nov 2022 16:12:53 +0000 Subject: [PATCH 050/282] Add Makefile.in targets for sqlite3r.c and sqlite3r.h, versions of the amalgamation that include the recover extension. FossilOrigin-Name: 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b --- Makefile.in | 9 +++++++++ manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/shell.c.in | 4 +++- tool/mksqlite3c.tcl | 18 ++++++++++++++---- tool/mksqlite3h.tcl | 10 ++++++++++ 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/Makefile.in b/Makefile.in index 09e75bb882..0a09e999f8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -778,6 +778,15 @@ sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . +sqlite3r.h: sqlite3.h + $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h + +sqlite3r.c: sqlite3.c sqlite3r.h + cp $(TOP)/ext/recover/sqlite3recover.c tsrc/ + cp $(TOP)/ext/recover/sqlite3recover.h tsrc/ + cp $(TOP)/ext/recover/dbdata.c tsrc/ + $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS) + sqlite3ext.h: .target_source cp tsrc/sqlite3ext.h . diff --git a/manifest b/manifest index 4cd29a45bd..b6d1cb0513 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Small\sperformance\simprovement\sin\ssqlite3BtreeTransferRow(). -D 2022-11-21T13:35:00.374 +C Add\sMakefile.in\stargets\sfor\ssqlite3r.c\sand\ssqlite3r.h,\sversions\sof\sthe\samalgamation\sthat\sinclude\sthe\srecover\sextension. +D 2022-11-22T16:12:53.393 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 78e4c4916f2c3993a8a454018745ff02094a8029d449d0c22db98f1cf8260420 +F Makefile.in 9a1228fd6406cc607958ccf1386a9a8f74f8255a7e98bc9c0a2792b56bd3a309 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc e7a564ceec71f0d9666031d5638cf0d4f88b050b44e8df5d32125137cd259ac0 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -642,7 +642,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 947c5714ae9b15a41bc69421ed33f642ed438d09536d0eeecd8964cc05d740ce F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -1995,8 +1995,8 @@ F tool/mkshellc.tcl 02d0de8349ef830c0fb20d29680320bde2466e2ec422e5bd94c4317a7a7e F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f -F tool/mksqlite3c.tcl 4fc26a9bfa0c4505b203d7ca0cf086e75ebcd4d63eb719c82f37e3fecdf23d37 -F tool/mksqlite3h.tcl 1f5e4a1dbbbc43c83cc6e74fe32c6c620502240b66c7c0f33a51378e78fc4edf +F tool/mksqlite3c.tcl eb47021591b1ad4a6862e2cb5625f1ac67ec1e0c6db5ba3953c069c635284cf5 +F tool/mksqlite3h.tcl d391cff7cad0a372ee1406faee9ccc7dad9cb80a0c95cae0f73d10dd26e06762 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c6115285845 @@ -2059,8 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07182dca9f2a4ffea1af0684c93e55e105465b2ee9820c70764e3e7bc1c28efc -R 06d7ab66cd3f5a521e55d8df4889b401 -U drh -Z a6fa3008dc4759fddba2bf18530ff99d +P dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 +R 1ce5997170cf4e38f2813ab1d1191f64 +T *branch * make-sqlite3r.c +T *sym-make-sqlite3r.c * +T -sym-trunk * +U dan +Z c67d4c65fcd4c164d590b57cc9ec79e4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e7e8aac79f..5f09b562a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dab959ea3edf99788bfd76352cd46a3e56876b0e7d7008c6927aa14534853c50 \ No newline at end of file +59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..29e61c7b16 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1066,9 +1066,11 @@ INCLUDE ../ext/expert/sqlite3expert.c #define SQLITE_SHELL_HAVE_RECOVER 0 #endif #if SQLITE_SHELL_HAVE_RECOVER -INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h +# ifndef SQLITE_HAVE_SQLITE3R +INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.c +# endif #endif #if defined(SQLITE_ENABLE_SESSION) diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 0cdf514e44..bc1aadd194 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -40,11 +40,14 @@ set help {Usage: tclsh mksqlite3c.tcl set addstatic 1 set linemacros 0 set useapicall 0 +set enable_recover 0 set srcdir tsrc for {set i 0} {$i<[llength $argv]} {incr i} { set x [lindex $argv $i] - if {[regexp {^-?-nostatic$} $x]} { + if {[regexp {^-?-enable-recover$} $x]} { + set enable_recover 1 + } elseif {[regexp {^-?-nostatic$} $x]} { set addstatic 0 } elseif {[regexp {^-?-linemacros(?:=([01]))?$} $x ma ulm]} { if {$ulm == ""} {set ulm 1} @@ -78,7 +81,9 @@ close $in # Open the output file and write a header comment at the beginning # of the file. # -set out [open sqlite3.c w] +set fname sqlite3.c +if {$enable_recover} { set fname sqlite3r.c } +set out [open $fname w] # Force the output to use unix line endings, even on Windows. fconfigure $out -translation lf set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] @@ -162,6 +167,7 @@ foreach hdr { vxworks.h wal.h whereInt.h + sqlite3recover.h } { set available_hdr($hdr) 1 } @@ -325,7 +331,7 @@ proc copy_file {filename} { # used subroutines first in order to help the compiler find # inlining opportunities. # -foreach file { +set flist { sqliteInt.h os_common.h ctime.c @@ -441,7 +447,11 @@ foreach file { sqlite3session.c fts5.c stmt.c -} { +} +if {$enable_recover} { + lappend flist sqlite3recover.c dbdata.c +} +foreach file $flist { copy_file $srcdir/$file } diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index 9078a15753..bd579c28b0 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -40,9 +40,16 @@ set TOP [lindex $argv 0] # set useapicall 0 +# Include sqlite3recover.h? +# +set enable_recover 0 + if {[lsearch -regexp [lrange $argv 1 end] {^-+useapicall}] != -1} { set useapicall 1 } +if {[lsearch -regexp [lrange $argv 1 end] {^-+enable-recover}] != -1} { + set enable_recover 1 +} # Get the SQLite version number (ex: 3.6.18) from the $TOP/VERSION file. # @@ -84,6 +91,9 @@ set filelist [subst { $TOP/ext/session/sqlite3session.h $TOP/ext/fts5/fts5.h }] +if {$enable_recover} { + lappend filelist "$TOP/ext/recover/sqlite3recover.h" +} # These are the functions that accept a variable number of arguments. They # always need to use the "cdecl" calling convention even when another calling From c9b7d943ad594bcf563b44405d8cea6256d37210 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 18:24:07 +0000 Subject: [PATCH 051/282] Fix the error message in the CLI for "no such VFS". [forum:/info/1217fdd6b3|Forum post 1217fdd6b3]. FossilOrigin-Name: 861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e1376b3753..4b77a98c23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\slast\sfunction\spointer\sin\ssqlite3Apis\sinit.\sReported\sat\shttps://sqlite.org/forum/info/eba0faa96d -D 2022-11-22T15:41:32.493 +C Fix\sthe\serror\smessage\sin\sthe\sCLI\sfor\s"no\ssuch\sVFS".\n[forum:/info/1217fdd6b3|Forum\spost\s1217fdd6b3]. +D 2022-11-22T18:24:07.441 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21e80a29737c367babcc0cf2533eed61b5d0fcf3cc3c33ab88761899e394eaf3 -R b2f32015706aa1808841626a0ff1391c -U larrybr -Z d9d2dcaeb03b2f8d48cf12bc3dcb589d +P 6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a +R c32ff36f085986b819f258da8a8464b1 +U drh +Z f12a671d42ea42f2f1054a78088f03c4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1264b5622a..26acee9357 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a \ No newline at end of file +861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..7f7ec779d4 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11767,7 +11767,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( pVfs ){ sqlite3_vfs_register(pVfs, 1); }else{ - utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]); + utf8_printf(stderr, "no such VFS: \"%s\"\n", zVfs); exit(1); } } From 5d7aef16bffb21331e7724cf475c18ac8d2d6d0f Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 19:49:16 +0000 Subject: [PATCH 052/282] Rename the SELECTTRACE macro to TREETRACE, so that is corresponds to the new CLI command. Renumber all of the bits in the bitmask used to enable various kinds of tracing, and add a trace bitmap decoder in sqliteInt.h. Changes to debugging logic only. No (intentional) changes to production code. FossilOrigin-Name: 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd --- manifest | 16 ++++---- manifest.uuid | 2 +- src/select.c | 104 ++++++++++++++++++++++++------------------------ src/sqliteInt.h | 4 +- src/window.c | 2 +- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 4b77a98c23..19b1249875 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\serror\smessage\sin\sthe\sCLI\sfor\s"no\ssuch\sVFS".\n[forum:/info/1217fdd6b3|Forum\spost\s1217fdd6b3]. -D 2022-11-22T18:24:07.441 +C Rename\sthe\sSELECTTRACE\smacro\sto\sTREETRACE,\sso\sthat\sis\scorresponds\sto\sthe\snew\nCLI\scommand.\s\sRenumber\sall\sof\sthe\sbits\sin\sthe\sbitmask\sused\sto\senable\nvarious\skinds\sof\stracing,\sand\sadd\sa\strace\sbitmap\sdecoder\sin\ssqliteInt.h.\nChanges\sto\sdebugging\slogic\sonly.\s\sNo\s(intentional)\schanges\sto\sproduction\scode. +D 2022-11-22T19:49:16.908 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 3fc60a8f0b54db15e86584c5fd68dbf63c20fe86886a39267ce7dfc17b68853d +F src/select.c 4c48373abb4e67129c36bc15d1f5a99a0dfd9534afeb539a2169a09ae91ccec9 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 42bcbb6cebef64c99068e7bc141a2541d40d65c7c26a9ca5ada4d83ca353555d +F src/sqliteInt.h 3bdf90cb7578dfccfd3e8c0f9c8b0eecc5280f5813891d6b3016cd6f1bdb1f7e F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -732,7 +732,7 @@ F src/where.c ea0f518df9e00aa44013a1d384090b4b3a499ee11d4daa0a7d99c4eb9f7ab4ba F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae -F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86 +F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test b5c19d504dec222c0dc66642673d23dce915d35737b68e74d9f237b80493eb53 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d5dc9eed54b7d883aa7dbe2eb5ff17d2a703eabdb7d548a27cccc1fe5addb4a -R c32ff36f085986b819f258da8a8464b1 +P 861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b +R 513d506718a1d19eb8b1625991fbf39f U drh -Z f12a671d42ea42f2f1054a78088f03c4 +Z 151ecb78555215509ebdea4705787b10 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 26acee9357..89e90a49a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b \ No newline at end of file +8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd \ No newline at end of file diff --git a/src/select.c b/src/select.c index f996ac0852..4d5bde9a31 100644 --- a/src/select.c +++ b/src/select.c @@ -2091,7 +2091,7 @@ void sqlite3GenerateColumnNames( if( pParse->colNamesSet ) return; /* Column names are determined by the left-most term of a compound select */ while( pSelect->pPrior ) pSelect = pSelect->pPrior; - SELECTTRACE(1,pParse,pSelect,("generating column names\n")); + TREETRACE(0x80,pParse,pSelect,("generating column names\n")); pTabList = pSelect->pSrc; pEList = pSelect->pEList; assert( v!=0 ); @@ -2877,7 +2877,7 @@ static int multiSelect( pPrior->iLimit = p->iLimit; pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); rc = sqlite3Select(pParse, pPrior, &dest); pPrior->pLimit = 0; if( rc ){ @@ -2895,7 +2895,7 @@ static int multiSelect( } } ExplainQueryPlan((pParse, 1, "UNION ALL")); - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -2948,7 +2948,7 @@ static int multiSelect( */ assert( !pPrior->pOrderBy ); sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); rc = sqlite3Select(pParse, pPrior, &uniondest); if( rc ){ goto multi_select_end; @@ -2968,7 +2968,7 @@ static int multiSelect( uniondest.eDest = op; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); rc = sqlite3Select(pParse, p, &uniondest); testcase( rc!=SQLITE_OK ); assert( p->pOrderBy==0 ); @@ -3029,7 +3029,7 @@ static int multiSelect( /* Code the SELECTs to our left into temporary table "tab1". */ sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT left...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); rc = sqlite3Select(pParse, pPrior, &intersectdest); if( rc ){ goto multi_select_end; @@ -3046,7 +3046,7 @@ static int multiSelect( intersectdest.iSDParm = tab2; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT right...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -4395,7 +4395,7 @@ static int flattenSubquery( } /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + TREETRACE(0x4,pParse,p,("flatten %u.%p from term %d\n", pSub->selId, pSub, iFrom)); /* Authorize the subquery */ @@ -4474,7 +4474,7 @@ static int flattenSubquery( if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; - SELECTTRACE(2,pParse,p,("compound-subquery flattener" + TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } assert( pSubitem->pSelect==0 ); @@ -4654,8 +4654,8 @@ static int flattenSubquery( sqlite3SelectDelete(db, pSub1); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After flattening:\n")); + if( sqlite3TreeTrace & 0x4 ){ + TREETRACE(0x4,pParse,p,("After flattening:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6068,8 +6068,8 @@ static int selectExpander(Walker *pWalker, Select *p){ } } #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After result-set wildcard expansion:\n")); + if( sqlite3TreeTrace & 0x8 ){ + TREETRACE(0x8,pParse,p,("After result-set wildcard expansion:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6465,7 +6465,7 @@ static void havingToWhere(Parse *pParse, Select *p){ sqlite3WalkExpr(&sWalker, p->pHaving); #if TREETRACE_ENABLED if( sWalker.eCode && (sqlite3TreeTrace & 0x100)!=0 ){ - SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); + TREETRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6597,8 +6597,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ p->selFlags &= ~SF_Aggregate; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n")); + if( sqlite3TreeTrace & 0x200 ){ + TREETRACE(0x200,pParse,p,("After count-of-view optimization:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6698,8 +6698,8 @@ int sqlite3Select( assert( db->mallocFailed==0 ); if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; #if TREETRACE_ENABLED - SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); - if( sqlite3TreeTrace & 0x10100 ){ + TREETRACE(0x1,pParse,p, ("begin processing:\n", pParse->addrExplain)); + if( sqlite3TreeTrace & 0x10000 ){ if( (sqlite3TreeTrace & 0x10001)==0x10000 ){ sqlite3TreeViewLine(0, "In sqlite3Select() at %s:%d", __FILE__, __LINE__); @@ -6719,8 +6719,8 @@ int sqlite3Select( /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ #if TREETRACE_ENABLED - SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n")); - if( sqlite3TreeTrace & 0x100 ){ + TREETRACE(0x800,pParse,p, ("dropping superfluous ORDER BY:\n")); + if( sqlite3TreeTrace & 0x800 ){ sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); } #endif @@ -6740,8 +6740,8 @@ int sqlite3Select( assert( db->mallocFailed==0 ); assert( p->pEList!=0 ); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x104 ){ - SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); + if( sqlite3TreeTrace & 0x10 ){ + TREETRACE(0x10,pParse,p, ("after name resolution:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6782,8 +6782,8 @@ int sqlite3Select( goto select_end; } #if TREETRACE_ENABLED - if( p->pWin && (sqlite3TreeTrace & 0x108)!=0 ){ - SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); + if( p->pWin && (sqlite3TreeTrace & 0x40)!=0 ){ + TREETRACE(0x40,pParse,p, ("after window rewrite:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6814,7 +6814,7 @@ int sqlite3Select( && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) && OptimizationEnabled(db, SQLITE_SimplifyJoin) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x1000,pParse,p, ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); assert( pItem->iCursor>=0 ); @@ -6870,7 +6870,7 @@ int sqlite3Select( && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ && OptimizationEnabled(db, SQLITE_OmitOrderBy) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x800,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); sqlite3ParserAddCleanup(pParse, (void(*)(sqlite3*,void*))sqlite3ExprListDelete, @@ -6925,8 +6925,8 @@ int sqlite3Select( if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); #if TREETRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end compound-select processing\n")); - if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + TREETRACE(0x400,pParse,p,("end compound-select processing\n")); + if( (sqlite3TreeTrace & 0x400)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif @@ -6946,13 +6946,13 @@ int sqlite3Select( && propagateConstants(pParse, p) ){ #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); + if( sqlite3TreeTrace & 0x2000 ){ + TREETRACE(0x2000,pParse,p,("After constant propagation:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif }else{ - SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } #ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION @@ -7025,15 +7025,15 @@ int sqlite3Select( && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem) ){ #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p, + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); sqlite3TreeViewSelect(0, p, 0); } #endif assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); }else{ - SELECTTRACE(0x100,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); } zSavedAuthContext = pParse->zAuthContext; @@ -7148,8 +7148,8 @@ int sqlite3Select( sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + if( sqlite3TreeTrace & 0x8000 ){ + TREETRACE(0x8000,pParse,p,("After all FROM-clause analysis:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -7185,8 +7185,8 @@ int sqlite3Select( sDistinct.isTnct = 2; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + if( sqlite3TreeTrace & 0x20000 ){ + TREETRACE(0x20000,pParse,p,("Transform DISTINCT into GROUP BY:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -7272,7 +7272,7 @@ int sqlite3Select( /* Begin the database scan. */ - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, p->pEList, p, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; @@ -7289,7 +7289,7 @@ int sqlite3Select( sSort.pOrderBy = 0; } } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -7328,7 +7328,7 @@ int sqlite3Select( /* End the database scan loop. */ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); } }else{ @@ -7451,8 +7451,8 @@ int sqlite3Select( pAggInfo->mxReg = pParse->nMem; if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); if( minMaxFlag ){ sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); @@ -7527,7 +7527,7 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 @@ -7537,7 +7537,7 @@ int sqlite3Select( goto select_end; } eDist = sqlite3WhereIsDistinct(pWInfo); - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be @@ -7586,7 +7586,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); @@ -7664,7 +7664,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); VdbeCoverage(v); }else{ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx); } @@ -7826,13 +7826,13 @@ int sqlite3Select( assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, pDistinct, p, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ goto select_end; } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); eDist = sqlite3WhereIsDistinct(pWInfo); updateAccumulator(pParse, regAcc, pAggInfo, eDist); if( eDist!=WHERE_DISTINCT_NOOP ){ @@ -7846,7 +7846,7 @@ int sqlite3Select( if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); } - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, pAggInfo); } @@ -7907,8 +7907,8 @@ select_end: #endif #if TREETRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end processing\n")); - if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + TREETRACE(0x1,pParse,p,("end processing\n")); + if( (sqlite3TreeTrace & 0x40000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6ff49ab5d0..9585bd4b2e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1029,12 +1029,12 @@ extern u32 sqlite3TreeTrace; && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE) \ || defined(SQLITE_ENABLE_TREETRACE)) # define TREETRACE_ENABLED 1 -# define SELECTTRACE(K,P,S,X) \ +# define TREETRACE(K,P,S,X) \ if(sqlite3TreeTrace&(K)) \ sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ sqlite3DebugPrintf X #else -# define SELECTTRACE(K,P,S,X) +# define TREETRACE(K,P,S,X) # define TREETRACE_ENABLED 0 #endif diff --git a/src/window.c b/src/window.c index f13ea8b027..1ed3e49214 100644 --- a/src/window.c +++ b/src/window.c @@ -1069,7 +1069,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ pSub = sqlite3SelectNew( pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 ); - SELECTTRACE(1,pParse,pSub, + TREETRACE(0x40,pParse,pSub, ("New window-function subquery in FROM clause of (%u/%p)\n", p->selId, p)); p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); From c7c5b8a508a2f941b00d43f5f1aa413e7cc1b171 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 19:56:54 +0000 Subject: [PATCH 053/282] Include the treetrace bitmap comment accidentally omitted from the previous check-in. FossilOrigin-Name: db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 19b1249875..d995c765ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\sthe\sSELECTTRACE\smacro\sto\sTREETRACE,\sso\sthat\sis\scorresponds\sto\sthe\snew\nCLI\scommand.\s\sRenumber\sall\sof\sthe\sbits\sin\sthe\sbitmask\sused\sto\senable\nvarious\skinds\sof\stracing,\sand\sadd\sa\strace\sbitmap\sdecoder\sin\ssqliteInt.h.\nChanges\sto\sdebugging\slogic\sonly.\s\sNo\s(intentional)\schanges\sto\sproduction\scode. -D 2022-11-22T19:49:16.908 +C Include\sthe\streetrace\sbitmap\scomment\saccidentally\somitted\sfrom\sthe\sprevious\ncheck-in. +D 2022-11-22T19:56:54.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb2 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 3bdf90cb7578dfccfd3e8c0f9c8b0eecc5280f5813891d6b3016cd6f1bdb1f7e +F src/sqliteInt.h 6a24230f2928b3d1d9b0fdbedb98c862b828a4b1a9170306108e74cd6277f476 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 861af465fda8485dfff143dc45c659b884d826aaec2ebaa941566404d1fe427b -R 513d506718a1d19eb8b1625991fbf39f +P 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd +R cda0c33c9442df99ec4e280c19853a61 U drh -Z 151ecb78555215509ebdea4705787b10 +Z 4e00dfd5e0c8561123895f1dda8a08f5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 89e90a49a0..e6bb7d9672 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd \ No newline at end of file +db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9585bd4b2e..86f33dec9e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1038,6 +1038,29 @@ extern u32 sqlite3TreeTrace; # define TREETRACE_ENABLED 0 #endif +/* TREETRACE flag meanings: +** +** 0x00000001 Beginning and end of SELECT processing +** 0x00000002 WHERE clause processing +** 0x00000004 Query flattener +** 0x00000008 Result-set wildcard expansion +** 0x00000010 Query name resolution +** 0x00000020 Aggregate analysis +** 0x00000040 Window functions +** 0x00000080 Generated column names +** 0x00000100 Move HAVING terms into WHERE +** 0x00000200 Count-of-view optimization +** 0x00000400 Compound SELECT processing +** 0x00000800 Drop superfluous ORDER BY +** 0x00001000 LEFT JOIN simplifies to JOIN +** 0x00002000 Constant propagation +** 0x00004000 Push-down optimization +** 0x00008000 After all FROM-clause analysis +** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing +** 0x00020000 Transform DISTINCT into GROUP BY +** 0x00040000 SELECT tree dump after all code has been generated +*/ + /* ** Macros for "wheretrace" */ From c62052217504fe651505328c32ea024c554e8a08 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 22 Nov 2022 20:04:00 +0000 Subject: [PATCH 054/282] Avoid naming collision between the sha1 and shathree extensions. FossilOrigin-Name: 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 --- ext/misc/shathree.c | 8 ++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index f80dd855da..765c691811 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -531,7 +531,7 @@ static void sha3Func( /* Compute a string using sqlite3_vsnprintf() with a maximum length ** of 50 bytes and add it to the hash. */ -static void hash_step_vformat( +static void sha3_step_vformat( SHA3Context *p, /* Add content to this context */ const char *zFormat, ... @@ -627,7 +627,7 @@ static void sha3QueryFunc( z = sqlite3_sql(pStmt); if( z ){ n = (int)strlen(z); - hash_step_vformat(&cx,"S%d:",n); + sha3_step_vformat(&cx,"S%d:",n); SHA3Update(&cx,(unsigned char*)z,n); } @@ -671,14 +671,14 @@ static void sha3QueryFunc( case SQLITE_TEXT: { int n2 = sqlite3_column_bytes(pStmt, i); const unsigned char *z2 = sqlite3_column_text(pStmt, i); - hash_step_vformat(&cx,"T%d:",n2); + sha3_step_vformat(&cx,"T%d:",n2); SHA3Update(&cx, z2, n2); break; } case SQLITE_BLOB: { int n2 = sqlite3_column_bytes(pStmt, i); const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - hash_step_vformat(&cx,"B%d:",n2); + sha3_step_vformat(&cx,"B%d:",n2); SHA3Update(&cx, z2, n2); break; } diff --git a/manifest b/manifest index d995c765ef..ee115293de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Include\sthe\streetrace\sbitmap\scomment\saccidentally\somitted\sfrom\sthe\sprevious\ncheck-in. -D 2022-11-22T19:56:54.514 +C Avoid\snaming\scollision\sbetween\sthe\ssha1\sand\sshathree\sextensions. +D 2022-11-22T20:04:00.704 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -322,7 +322,7 @@ F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d385 F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c 8d79354f2c3d46b95ee21272a07cf0bcabb58d1f2b06d9e7b8a31dca1dacb3e5 F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d -F ext/misc/shathree.c 7b17615869a495659f1569ada1d8d3d21b4a24614f2746d93cc87ef7c0b6b36d +F ext/misc/shathree.c 9b7af7b7d55b27c5bbd16548b67fe6aa47a819e71bc6a80a456144f86f4e834d F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 94df9bbfa514a563c1484f684a2df3d128a2f7209a84ca3ca100c68a0163e29f F ext/misc/sqlar.c 0ace5d3c10fe736dc584bf1159a36b8e2e60fab309d310cd8a0eecd9036621b6 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd -R cda0c33c9442df99ec4e280c19853a61 -U drh -Z 4e00dfd5e0c8561123895f1dda8a08f5 +P db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 +R 028f4214066f285c3501b4fa202e9f4b +U mistachkin +Z 9f8febbd78f0b476308550c8b424fd90 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e6bb7d9672..7b2a4dc5e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 \ No newline at end of file +9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 \ No newline at end of file From b23f1572ab9c11e524023678a5347152aae48b04 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Nov 2022 20:37:41 +0000 Subject: [PATCH 055/282] Add the stub function: optimizeAggregateUsingIndexedExpr(). The hope is that we can fill this in with a routine that does useful optimizations. FossilOrigin-Name: d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8586f2587b..e809fe26ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sand\senhancements\sinto\sthe\sagg-with-indexed-expr\sbranch. -D 2022-11-22T19:51:16.237 +C Add\sthe\sstub\sfunction:\soptimizeAggregateUsingIndexedExpr().\s\sThe\shope\sis\sthat\nwe\scan\sfill\sthis\sin\swith\sa\sroutine\sthat\sdoes\suseful\soptimizations. +D 2022-11-22T20:37:41.721 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 35c2b1b4d82190f202887620f99070c0d414c3e030e466032797792cffdbd9f9 +F src/select.c dfc504356e15bb8585d8535a927a72512d1d7a01f9074824964f1781e286231b F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc5bd34963b761c819c565653156d0befbf65cc2cc5dc4113b0ce952450f0352 8036445a36d9d982c1305935e7e37367bdf9e466b923eb6286b52524802e3ccd -R e6ec32714f5519d3f42f7a87b92d7eba +P 070634781a5eb41f96b001e48b064e3cd8c82314f576335eb1fcd43792179291 +R 5bcfbb263a340b6c57659d80015cbb33 U drh -Z 31a3ccb601c45d898a857131ff2f396b +Z 28092ed1d737fad311ecc59737148e5d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1556ed2eb3..0a7b42af87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -070634781a5eb41f96b001e48b064e3cd8c82314f576335eb1fcd43792179291 \ No newline at end of file +d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 946649ad0b..ecd1791060 100644 --- a/src/select.c +++ b/src/select.c @@ -6654,6 +6654,34 @@ static void printAggInfo(AggInfo *pAggInfo){ } #endif /* TREETRACE_ENABLED */ +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static int optimizeAggregateUsingIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT being coded */ + AggInfo *pAggInfo /* The aggregate info */ +){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x100000 ){ + IndexedExpr *pIEpr; + TREETRACE(0x1000000, pParse, pSelect, + ("Attempting to optimize AggInfo for Indexed Exprs\n")); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("cur=%d\n", pIEpr->iDataCur); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#endif + return 0; +} + + /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7535,6 +7563,9 @@ int sqlite3Select( sqlite3ExprListDelete(db, pDistinct); goto select_end; } + if( pParse->pIdxEpr ){ + optimizeAggregateUsingIndexedExpr(pParse, p, pAggInfo); + } assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); From c6138e970e4afca027635e97d0516a58e5c026a4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 14:13:39 +0000 Subject: [PATCH 056/282] Further foundation prep work prior to starting to flesh-out the optimizeAggregateUseOfIndexedExpr() routine. FossilOrigin-Name: 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e --- manifest | 12 ++-- manifest.uuid | 2 +- src/select.c | 171 +++++++++++++++++++++++++++++--------------------- 3 files changed, 108 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 768c50fa57..1ffe8b4bb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sand\sfixes\sfrom\strunk. -D 2022-11-22T20:58:18.661 +C Further\sfoundation\sprep\swork\sprior\sto\sstarting\sto\sflesh-out\sthe\noptimizeAggregateUseOfIndexedExpr()\sroutine. +D 2022-11-23T14:13:39.215 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6f5403b8f1849d2a8ba152dd790ec189bba18242157904ddca77707e209265d4 +F src/select.c 1d3f8a5b510f8e2859c7f800babb5a63802789f93c9c29498312b37655dac1b8 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d85bb724fdd6fbad2b88ed7f60e4174e3f65182356f404d04620c5cf6b17f77e 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 -R 55f08c65335ec615b6caa516a7697c9d +P f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 +R e49e6155bccbafba580b14baaea3db86 U drh -Z dbc39758acc6c225892e0cac1e391b2a +Z b6500d1e8d8b2720badce4287d0e1594 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9c7d27e3fe..f88bcea3d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 \ No newline at end of file +23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e \ No newline at end of file diff --git a/src/select.c b/src/select.c index f71fa3197e..8f2517ad4f 100644 --- a/src/select.c +++ b/src/select.c @@ -6204,9 +6204,106 @@ void sqlite3SelectPrep( sqlite3SelectAddTypeInfo(pParse, p); } +#if TREETRACE_ENABLED /* -** Assign register numbers to all pAggInfo->aCol[] and pAggInfo->aFunc[] -** entries. +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d %s\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), + pCol->iSorterColumn, + ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + ii, AggInfoFuncReg(pAggInfo,ii)); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + +/* +** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[] +** entries for columns that are arguments to aggregate functions but which +** are not otherwise used. +** +** The aCol[] entries in AggInfo prior to nAccumulator are columns that +** are referenced outside of aggregate functions. These might be columns +** that are part of the GROUP by clause, for example. Other database engines +** would throw an error if there is a column reference that is not in the +** GROUP BY clause and that is not part of an aggregate function argument. +** But SQLite allows this. +** +** The aCol[] entries beginning with the aCol[nAccumulator] and following +** are column references that are used exclusively as arguments to +** aggregate functions. This routine is responsible for computing +** (or recomputing) those aCol[] entries. +*/ +static void analyzeAggFuncArgs( + Parse *pParse, + AggInfo *pAggInfo, + NameContext *pNC +){ + int i; + assert( pAggInfo!=0 ); + pNC->ncFlags |= NC_InAggFunc; + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( ExprUseXList(pExpr) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter); + } +#endif + } + pNC->ncFlags &= ~NC_InAggFunc; +} + +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static void optimizeAggregateUseOfIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT statement being processed */ + AggInfo *pAggInfo, /* The aggregate info */ + NameContext *pNC /* Name context used to resolve agg-func args */ +){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + IndexedExpr *pIEpr; + TREETRACE(0x80000, pParse, pSelect, + ("Attempting to optimize AggInfo for Indexed Exprs\n")); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("data-cursor=%d index={%d,%d}\n", + pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#else + (void)pSelect; /* Suppress unused-parameter warnings */ +#endif + pAggInfo->nColumn = pAggInfo->nAccumulator; + analyzeAggFuncArgs(pParse, pAggInfo, pNC); +} + +/* +** Allocate a block of registers so that there is one register for each +** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first +** register in this block is stored in pAggInfo->iFirstReg. */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ assert( pAggInfo!=0 ); @@ -6630,60 +6727,6 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ return 0; } -#if TREETRACE_ENABLED -/* -** Display all information about an AggInfo object -*/ -static void printAggInfo(AggInfo *pAggInfo){ - int ii; - for(ii=0; iinColumn; ii++){ - struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; - sqlite3DebugPrintf( - "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" - " iSorterColumn=%d %s\n", - ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), - pCol->iSorterColumn, - ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=\n", - ii, AggInfoFuncReg(pAggInfo,ii)); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } -} -#endif /* TREETRACE_ENABLED */ - -/* -** An index on expressions is being used in the inner loop of an -** aggregate query with a GROUP BY clause. This routine attempts -** to adjust the AggInfo object to take advantage of index and to -** perhaps use the index as a covering index. -** -*/ -static int optimizeAggregateUsingIndexedExpr( - Parse *pParse, /* Parsing context */ - Select *pSelect, /* The SELECT being coded */ - AggInfo *pAggInfo /* The aggregate info */ -){ -#if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x80000 ){ - IndexedExpr *pIEpr; - TREETRACE(0x80000, pParse, pSelect, - ("Attempting to optimize AggInfo for Indexed Exprs\n")); - for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ - printf("data-cursor=%d index={%d,%d}\n", - pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); - sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); - } - printAggInfo(pAggInfo); - } -#endif - return 0; -} - - /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7465,19 +7508,7 @@ int sqlite3Select( }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - for(i=0; inFunc; i++){ - Expr *pExpr = pAggInfo->aFunc[i].pFExpr; - assert( ExprUseXList(pExpr) ); - sNC.ncFlags |= NC_InAggFunc; - sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); -#ifndef SQLITE_OMIT_WINDOWFUNC - assert( !IsWindowFunc(pExpr) ); - if( ExprHasProperty(pExpr, EP_WinFunc) ){ - sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); - } -#endif - sNC.ncFlags &= ~NC_InAggFunc; - } + analyzeAggFuncArgs(pParse, pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ @@ -7566,7 +7597,7 @@ int sqlite3Select( goto select_end; } if( pParse->pIdxEpr ){ - optimizeAggregateUsingIndexedExpr(pParse, p, pAggInfo); + optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC); } assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); From 3eaaec945c4cc3b35e9bd3c0f9a133c7f94cae1f Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 15:33:49 +0000 Subject: [PATCH 057/282] OPFS concurrency tester: ensure that the work interval timer does not overlap with the work time. FossilOrigin-Name: 8f2076553bc486ea6a17934695ecc02217461af2082d891697b62aab89bd1b43 --- ext/wasm/tests/opfs/concurrency/worker.js | 5 +++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index c315508e0b..19b0a068e7 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -70,11 +70,12 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; if(1){/*use setInterval()*/ - interval.handle = setInterval(async ()=>{ + setTimeout(async function timer(){ await doWork(); if(interval.error || maxIterations === interval.count){ - clearInterval(interval.handle); finish(); + }else{ + setTimeout(timer, interval.delay); } }, interval.delay); }else{ diff --git a/manifest b/manifest index ee115293de..0133034207 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\snaming\scollision\sbetween\sthe\ssha1\sand\sshathree\sextensions. -D 2022-11-22T20:04:00.704 +C OPFS\sconcurrency\stester:\sensure\sthat\sthe\swork\sinterval\stimer\sdoes\snot\soverlap\swith\sthe\swork\stime. +D 2022-11-23T15:33:49.725 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js df065bb386ff994951f7fbdd76e12f16e58fbef0e929b2caf74553359da40afc +F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db07471c531766a8eec1d5b41c9fd0283b5e64ee13166dc3391f70a1e1946121 -R 028f4214066f285c3501b4fa202e9f4b -U mistachkin -Z 9f8febbd78f0b476308550c8b424fd90 +P 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 +R f934382a93ebec7212ddc22bd89cd134 +U stephan +Z 5fd98b4beaa4b9278c801856cb1d0dd1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7b2a4dc5e4..04ede9801e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 \ No newline at end of file +8f2076553bc486ea6a17934695ecc02217461af2082d891697b62aab89bd1b43 \ No newline at end of file From 91a81316b9b184126b3546b332b36749209d34bf Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 15:52:52 +0000 Subject: [PATCH 058/282] Remove a bit of over-cleverness which breaks loading of sqlite3.js in some main-thread cases. FossilOrigin-Name: 220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 --- ext/wasm/api/extern-post-js.js | 3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index b327837814..cace6ed51c 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -59,9 +59,6 @@ const toExportForES6 = li.pop(); initModuleState.sqlite3Dir = li.join('/') + '/'; } - if(initModuleState.sqlite3Dir){ - initModuleState.sqlite3Dir = initModuleState.sqlite3Dir.replace(/[/]{2,}/g,'/'); - } self.sqlite3InitModule = (...args)=>{ //console.warn("Using replaced sqlite3InitModule()",self.location); diff --git a/manifest b/manifest index 0133034207..eab1abce2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sconcurrency\stester:\sensure\sthat\sthe\swork\sinterval\stimer\sdoes\snot\soverlap\swith\sthe\swork\stime. -D 2022-11-23T15:33:49.725 +C Remove\sa\sbit\sof\sover-cleverness\swhich\sbreaks\sloading\sof\ssqlite3.js\sin\ssome\smain-thread\scases. +D 2022-11-23T15:52:52.817 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 29276a845e57004e82efba61fa5866fd05f9137380a1dc26dc4c6d65264cd81c -F ext/wasm/api/extern-post-js.js 31400dd1c0ae3458a0e6510229e59318e45eac402a75dd703c2950b9b5758b46 +F ext/wasm/api/extern-post-js.js 59e52f579cd3a332d73dae94c91b9579daafb10dd6ada03803f1afa6bdad7689 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 -R f934382a93ebec7212ddc22bd89cd134 +P 8f2076553bc486ea6a17934695ecc02217461af2082d891697b62aab89bd1b43 +R 48a8d083139c4365079fe8210cacb510 U stephan -Z 5fd98b4beaa4b9278c801856cb1d0dd1 +Z c7336980b293a293ed2df30baba9b924 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 04ede9801e..9666b96008 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f2076553bc486ea6a17934695ecc02217461af2082d891697b62aab89bd1b43 \ No newline at end of file +220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 \ No newline at end of file From ad1285c5c0be7eb92cc44a3357be71507c3c07f2 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 16:39:07 +0000 Subject: [PATCH 059/282] Initial infrastructure for adding a mode to the OPFS VFS which causes implicit locks to be released ASAP, which increases concurrency at the cost of performance. FossilOrigin-Name: c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 --- ext/wasm/api/sqlite3-api-oo1.js | 5 +- ext/wasm/api/sqlite3-api-opfs.js | 1 + ext/wasm/api/sqlite3-opfs-async-proxy.js | 123 ++++++++++++++++------ ext/wasm/tests/opfs/concurrency/worker.js | 6 +- manifest | 21 ++-- manifest.uuid | 2 +- 6 files changed, 112 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 02ce9c0ced..bc32ec1067 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -200,9 +200,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){ const arg = {}; - if(1===arguments.length && 'object'===typeof arguments[0]){ - const x = arguments[0]; - Object.keys(x).forEach((k)=>arg[k] = x[k]); + if(1===arguments.length && arguments[0] && 'object'===typeof arguments[0]){ + Object.assign(arg, arguments[0]); if(undefined===arg.flags) arg.flags = 'c'; if(undefined===arg.vfs) arg.vfs = null; if(undefined===arg.filename) arg.filename = ':memory:'; diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index deb4c923ab..53f68a1d58 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -348,6 +348,7 @@ const installOpfsVfs = function callee(options){ 'SQLITE_NOTFOUND', 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', 'SQLITE_OPEN_READONLY' ].forEach((k)=>{ if(undefined === (state.sq3Codes[k] = capi[k])){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index c208932e17..cbe0549b3e 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -105,7 +105,7 @@ metrics.dump = ()=>{ */ const __openFiles = Object.create(null); /** - __autoLocks is a Set of sqlite3_file pointers (integers) which were + __implicitLocks is a Set of sqlite3_file pointers (integers) which were "auto-locked". i.e. those for which we obtained a sync access handle without an explicit xLock() call. Such locks will be released during db connection idle time, whereas a sync access @@ -117,7 +117,7 @@ const __openFiles = Object.create(null); penalty: speedtest1 benchmarks take up to 4x as long. By delaying the lock release until idle time, the hit is negligible. */ -const __autoLocks = new Set(); +const __implicitLocks = new Set(); /** Expects an OPFS file path. It gets resolved, such that ".." @@ -166,7 +166,7 @@ const closeSyncHandle = async (fh)=>{ const h = fh.syncHandle; delete fh.syncHandle; delete fh.xLock; - __autoLocks.delete(fh.fid); + __implicitLocks.delete(fh.fid); return h.close(); } }; @@ -190,10 +190,10 @@ const closeSyncHandleNoThrow = async (fh)=>{ }; /* Release all auto-locks. */ -const closeAutoLocks = async ()=>{ - if(__autoLocks.size){ +const releaseImplicitLocks = async ()=>{ + if(__implicitLocks.size){ /* Release all auto-locks. */ - for(const fid of __autoLocks){ + for(const fid of __implicitLocks){ const fh = __openFiles[fid]; await closeSyncHandleNoThrow(fh); log("Auto-unlocked",fid,fh.filenameAbs); @@ -201,6 +201,32 @@ const closeAutoLocks = async ()=>{ } }; +/** + If true, any routine which implicitly acquires a sync access handle + (i.e. an OPFS lock) will release that locks at the end of the call + which acquires it. If false, such "autolocks" are not released + until the VFS is idle for some brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). +*/ +state.defaultReleaseImplicitLocks = false; + +/** + An experiment in improving concurrency by freeing up implicit locks + sooner. This is known to impact performance dramatically but it has + also shown to improve concurrency considerably. + + If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, + this routine returns closeSyncHandleNoThrow(), else it is a no-op. +*/ +const releaseImplicitLock = async (fh)=>{ + if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ + return closeSyncHandleNoThrow(fh); + } +}; + /** An error class specifically for use with getSyncHandle(), the goal of which is to eventually be able to distinguish unambiguously @@ -246,7 +272,7 @@ GetSyncHandleError.convertRc = (e,rc)=>{ still fails at that point it will give up and propagate the exception. */ -const getSyncHandle = async (fh)=>{ +const getSyncHandle = async (fh,opName)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); @@ -262,20 +288,21 @@ const getSyncHandle = async (fh)=>{ }catch(e){ if(i === maxTries){ throw new GetSyncHandleError( - e, "Error getting sync handle.",maxTries, + e, "Error getting sync handle for",opName+"().",maxTries, "attempts failed.",fh.filenameAbs ); } - warn("Error getting sync handle. Waiting",ms, + warn("Error getting sync handle for",opName+"(). Waiting",ms, "ms and trying again.",fh.filenameAbs,e); - await closeAutoLocks(); + //await releaseImplicitLocks(); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } - log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + log("Got",opName+"() sync handle for",fh.filenameAbs, + 'in',performance.now() - t,'ms'); if(!fh.xLock){ - __autoLocks.add(fh.fid); - log("Auto-locked",fh.fid,fh.filenameAbs); + __implicitLocks.add(fh.fid); + log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); } } return fh.syncHandle; @@ -409,7 +436,7 @@ const vfsAsyncImpls = { xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; mTimeStart(opName); - __autoLocks.delete(fid); + __implicitLocks.delete(fid); const fh = __openFiles[fid]; let rc = 0; wTimeStart(opName); @@ -474,13 +501,14 @@ const vfsAsyncImpls = { wTimeStart('xFileSize'); try{ affirmLocked('xFileSize',fh); - const sz = await (await getSyncHandle(fh)).getSize(); + const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); state.s11n.serialize(Number(sz)); rc = 0; }catch(e){ state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xFileSize', rc); mTimeEnd(); @@ -495,8 +523,8 @@ const vfsAsyncImpls = { if( !fh.syncHandle ){ wTimeStart('xLock'); try { - await getSyncHandle(fh); - __autoLocks.delete(fid); + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); }catch(e){ state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); @@ -511,7 +539,6 @@ const vfsAsyncImpls = { flags/*SQLITE_OPEN_...*/){ const opName = 'xOpen'; mTimeStart(opName); - const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); wTimeStart('xOpen'); try{ @@ -526,14 +553,8 @@ const vfsAsyncImpls = { return; } const hFile = await hDir.getFileHandle(filenamePart, {create}); - /** - wa-sqlite, at this point, grabs a SyncAccessHandle and - assigns it to the syncHandle prop of the file state - object, but only for certain cases and it's unclear why it - places that limitation on it. - */ wTimeEnd(); - __openFiles[fid] = Object.assign(Object.create(null),{ + const fh = Object.assign(Object.create(null),{ fid: fid, filenameAbs: filename, filenamePart: filenamePart, @@ -542,8 +563,47 @@ const vfsAsyncImpls = { sabView: state.sabFileBufView, readOnly: create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), - deleteOnClose: deleteOnClose + deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); + fh.releaseImplicitLocks = + state.defaultReleaseImplicitLocks + /* TODO: check URI flags for "opfs-auto-unlock". First we need to + reshape the API a bit to be able to pass those on to here + from the other half of the proxy. */; + /*if(fh.releaseImplicitLocks){ + console.warn("releaseImplicitLocks is ON for",fh); + }*/ + if(0 /* this block is modelled after something wa-sqlite + does but it leads to horrible contention on journal files. */ + && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ + /* sqlite does not lock these files, so go ahead and grab an OPFS + lock. + + Regarding "immutable": that flag is not _really_ applicable + here. It's intended for use on read-only media. If, + however, a file is opened with that flag but changes later + (which can happen if we _don't_ grab a sync handle here) + then sqlite may misbehave. + + Regarding "nolock": ironically, the nolock flag forces us + to lock the file up front. "nolock" tells sqlite to _not_ + use its locking API, but OPFS requires a lock to perform + most of the operations performed in this file. If we don't + grab that lock up front, another handle could end up grabbing + it and mutating the database out from under our nolocked'd + handle. In the interest of preventing corruption, at the cost + of decreased concurrency, we have to lock it for the duration + of this file handle. + + https://www.sqlite.org/uri.html + */ + fh.xLock = "atOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; + await getSyncHandle(fh,'xOpen'); + } + __openFiles[fid] = fh; storeAndNotify(opName, 0); }catch(e){ wTimeEnd(); @@ -560,7 +620,7 @@ const vfsAsyncImpls = { try{ affirmLocked('xRead',fh); wTimeStart('xRead'); - nRead = (await getSyncHandle(fh)).read( + nRead = (await getSyncHandle(fh,'xRead')).read( fh.sabView.subarray(0, n), {at: Number(offset64)} ); @@ -575,6 +635,7 @@ const vfsAsyncImpls = { state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } + await releaseImplicitLock(fh); storeAndNotify('xRead',rc); mTimeEnd(); }, @@ -603,12 +664,13 @@ const vfsAsyncImpls = { try{ affirmLocked('xTruncate',fh); affirmNotRO('xTruncate', fh); - await (await getSyncHandle(fh)).truncate(size); + await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xTruncate',rc); mTimeEnd(); @@ -640,7 +702,7 @@ const vfsAsyncImpls = { affirmLocked('xWrite',fh); affirmNotRO('xWrite', fh); rc = ( - n === (await getSyncHandle(fh)) + n === (await getSyncHandle(fh,'xWrite')) .write(fh.sabView.subarray(0, n), {at: Number(offset64)}) ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; @@ -649,6 +711,7 @@ const vfsAsyncImpls = { state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xWrite',rc); mTimeEnd(); @@ -783,7 +846,7 @@ const waitLoop = async function f(){ if('timed-out'===Atomics.wait( state.sabOPView, state.opIds.whichOp, 0, waitTime )){ - await closeAutoLocks(); + await releaseImplicitLocks(); continue; } const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 19b0a068e7..9b4898f4c3 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -43,7 +43,11 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.opfs.OpfsDb(dbName,'c'); + db = new sqlite3.oo1.DB({ + filename: 'file:'+dbName, + flags: 'c', + vfs: 'opfs' + }); sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ db.exec([ diff --git a/manifest b/manifest index 3ecb281109..d6e43cfa43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMakefile.in\sto\sinclude\snew\starget\s"sqlite3r.c".\sFor\sgenerating\s"sqlite3r.c"\sand\s"sqlite3r.h",\sversions\sof\sthe\samalgamation\sthat\sinclude\sthe\srecover\sextension.\sTo\sbuild\sthe\sshell\stool\sagainst\sthese\sfiles,\sadd\s-DSQLITE_HAVE_SQLITE3R. -D 2022-11-23T16:08:49.765 +C Initial\sinfrastructure\sfor\sadding\sa\smode\sto\sthe\sOPFS\sVFS\swhich\scauses\simplicit\slocks\sto\sbe\sreleased\sASAP,\swhich\sincreases\sconcurrency\sat\sthe\scost\sof\sperformance. +D 2022-11-23T16:39:07.866 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,12 +501,12 @@ F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b387181 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 -F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b +F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 +F ext/wasm/api/sqlite3-api-opfs.js 69a897eae705816a002f44e8a37693e2ceb7b3e32f9889df2302aeba7df24c70 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a +F ext/wasm/api/sqlite3-opfs-async-proxy.js 3142ed3a636d9a1a3f4793c6af6373996593c615a3a5fa29de59ba3e0ea45bee F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de +F ext/wasm/tests/opfs/concurrency/worker.js cc43ea47bb59582523e3f27c2198247c4adca2fae9a891f84dd3bccfccef2833 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,9 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -R 6b5395a59674684ae7409c1a67752bd1 -T +closed 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -U dan -Z 08a51ae8157c87edcee2b45d299f4155 +P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 +R 5ffa2a11b06c4a44d92d27455a2fc3e8 +U stephan +Z 70f5adf76d87ddf8a488ff2bcd290afb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c19a5133e..54a2776a7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 \ No newline at end of file +c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 \ No newline at end of file From 590f013a87609ea33abc718aab71c8785a7213f9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 17:56:00 +0000 Subject: [PATCH 060/282] This attempt at modifying AggInfo to make use of indexed expressions does not work. It gets an incorrect answer for the test case shown in the ticket. FossilOrigin-Name: 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 --- manifest | 16 ++--- manifest.uuid | 2 +- src/expr.c | 157 ++++++++++++++++++++++++++++++------------------ src/select.c | 21 +++++-- src/sqliteInt.h | 1 - 5 files changed, 122 insertions(+), 75 deletions(-) diff --git a/manifest b/manifest index 1ffe8b4bb7..294477b047 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sfoundation\sprep\swork\sprior\sto\sstarting\sto\sflesh-out\sthe\noptimizeAggregateUseOfIndexedExpr()\sroutine. -D 2022-11-23T14:13:39.215 +C This\sattempt\sat\smodifying\sAggInfo\sto\smake\suse\sof\sindexed\sexpressions\sdoes\snot\nwork.\s\sIt\sgets\san\sincorrect\sanswer\sfor\sthe\stest\scase\sshown\sin\sthe\sticket. +D 2022-11-23T17:56:00.136 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 63cce2c219748d8f8e00a30e4dc43d65b686b62489f154eebe892933bfb4a249 +F src/expr.c 527fc56468f75c380a34d38426cc1deece693a3cc6139fb45544923ff461d569 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 1d3f8a5b510f8e2859c7f800babb5a63802789f93c9c29498312b37655dac1b8 +F src/select.c 8ac7f73d40d5615cdb73660a85e05ee26943a618ec5c7262160994f263fa7178 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 894f7d98b1104fecb2ca2f457a4c598944250d9462eb433f12bd42b1080d525f +F src/sqliteInt.h f4917d663170308601d794cdff7b8cf9704c4bafe03d7e926a156be835e29f29 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f8932e04d4d18eb9d71edda15aa08af2eb139ff14d77ca147ea6e9b173e0f5e0 -R e49e6155bccbafba580b14baaea3db86 +P 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e +R f3df5c5148120a1bf76a131c870fdc09 U drh -Z b6500d1e8d8b2720badce4287d0e1594 +Z 97820e8b433aee285f0d7a5d438ada65 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f88bcea3d3..c7c9044f2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e \ No newline at end of file +84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 42099536ed..db81983e4a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4135,7 +4135,7 @@ expr_code_doover: pCol->iSorterColumn, target); if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( ALWAYS(pTab!=0) ){ + }else if( pTab!=0 ){ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -6241,6 +6241,71 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ return i; } +/* +** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. +** Return the index in aCol[] of the entry that describes that column. +** +** If no prior entry is found, create a new one and return -1. The +** new column will have an idex of pAggInfo->nColumn-1. +*/ +static void findOrCreateAggInfoColumn( + Parse *pParse, /* Parsing context */ + AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */ + Expr *pExpr /* Expr describing the column to find or insert */ +){ + struct AggInfo_col *pCol; + int k; + + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ + goto fix_up_expr; + } + } + k = addAggInfoColumn(pParse->db, pAggInfo); + if( k<0 ){ + /* OOM on resize */ + assert( pParse->db->mallocFailed ); + return; + } + pCol = &pAggInfo->aCol[k]; + assert( ExprUseYTab(pExpr) ); + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } +fix_up_expr: + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->pAggInfo = pAggInfo; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } + pExpr->iAgg = (i16)k; +} + /* ** This is the xExprCallback for a tree walker. It is used to ** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates @@ -6255,6 +6320,37 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( pNC->ncFlags & NC_UAggInfo ); switch( pExpr->op ){ + default: { + IndexedExpr *pIEpr; + Expr tmp; + if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; + if( pParse->pIdxEpr==0 ) break; + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + int iDataCur = pIEpr->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; + } + if( pIEpr==0 ) break; + if( !ExprUseYTab(pExpr) ) break; + + /* If we reach this point, it means that expression pExpr can be + ** translated into a reference to an index column as described by + ** pIEpr. + */ + memset(&tmp, 0, sizeof(tmp)); + tmp.op = TK_AGG_COLUMN; + tmp.iTable = pIEpr->iIdxCur; + tmp.iColumn = pIEpr->iIdxCol; + findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; + pExpr->pAggInfo = pAggInfo; + pExpr->iAgg = tmp.iAgg; + return WRC_Prune; + } case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { @@ -6266,66 +6362,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( ALWAYS(pSrcList!=0) ){ SrcItem *pItem = pSrcList->a; for(i=0; inSrc; i++, pItem++){ - struct AggInfo_col *pCol; assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable - && pCol->iColumn==pExpr->iColumn - && pExpr->op!=TK_IF_NULL_ROW - ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - assert( ExprUseYTab(pExpr) ); - pCol->pTab = pExpr->y.pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iSorterColumn = -1; - pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; jpExpr; - if( pE->op==TK_COLUMN - && pE->iTable==pExpr->iTable - && pE->iColumn==pExpr->iColumn - ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetVVAProperty(pExpr, EP_NoReduce); - pExpr->pAggInfo = pAggInfo; - if( pExpr->op==TK_COLUMN ){ - pExpr->op = TK_AGG_COLUMN; - } - pExpr->iAgg = (i16)k; + findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr); break; } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ diff --git a/src/select.c b/src/select.c index 8f2517ad4f..49bef86bab 100644 --- a/src/select.c +++ b/src/select.c @@ -6281,11 +6281,22 @@ static void optimizeAggregateUseOfIndexedExpr( AggInfo *pAggInfo, /* The aggregate info */ NameContext *pNC /* Name context used to resolve agg-func args */ ){ + pAggInfo->nColumn = pAggInfo->nAccumulator; + if( pAggInfo->nSortingColumn>0 ){ + if( pAggInfo->nColumn==0 ){ + pAggInfo->nSortingColumn = 0; + }else{ + pAggInfo->nSortingColumn = + pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; + } + } + analyzeAggFuncArgs(pParse, pAggInfo, pNC); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x80000 ){ + if( sqlite3TreeTrace & 0x20 ){ IndexedExpr *pIEpr; - TREETRACE(0x80000, pParse, pSelect, - ("Attempting to optimize AggInfo for Indexed Exprs\n")); + TREETRACE(0x20, pParse, pSelect, + ("AggInfo (possibly) adjusted for Indexed Exprs\n")); + sqlite3TreeViewSelect(0, pSelect, 0); for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ printf("data-cursor=%d index={%d,%d}\n", pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); @@ -6296,8 +6307,6 @@ static void optimizeAggregateUseOfIndexedExpr( #else (void)pSelect; /* Suppress unused-parameter warnings */ #endif - pAggInfo->nColumn = pAggInfo->nAccumulator; - analyzeAggFuncArgs(pParse, pAggInfo, pNC); } /* @@ -7959,7 +7968,7 @@ select_end: if( pAggInfo && !db->mallocFailed ){ for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; - assert( pExpr!=0 ); + if( pExpr==0 ) continue; assert( pExpr->pAggInfo==pAggInfo ); assert( pExpr->iAgg==i ); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 74c997c6dd..3ecdb35fee 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1059,7 +1059,6 @@ extern u32 sqlite3TreeTrace; ** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated -** 0x00080000 Optimize Aggregates using indexed expressions */ /* From 2e30d95fb63e73cbae3cf07b0ac57d8d7e0321ba Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 23 Nov 2022 18:51:04 +0000 Subject: [PATCH 061/282] Aggregates with GROUP BY now make use of expressions on indexes. This code works and gets the correct answer for the test case in the ticket. Lots more testing and documentation is needed, however. FossilOrigin-Name: 8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 6 ++++-- src/select.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 294477b047..5a5586e90e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sattempt\sat\smodifying\sAggInfo\sto\smake\suse\sof\sindexed\sexpressions\sdoes\snot\nwork.\s\sIt\sgets\san\sincorrect\sanswer\sfor\sthe\stest\scase\sshown\sin\sthe\sticket. -D 2022-11-23T17:56:00.136 +C Aggregates\swith\sGROUP\sBY\snow\smake\suse\sof\sexpressions\son\sindexes.\s\sThis\scode\nworks\sand\sgets\sthe\scorrect\sanswer\sfor\sthe\stest\scase\sin\sthe\sticket.\s\sLots\smore\ntesting\sand\sdocumentation\sis\sneeded,\showever. +D 2022-11-23T18:51:04.363 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 527fc56468f75c380a34d38426cc1deece693a3cc6139fb45544923ff461d569 +F src/expr.c 141af8139174010ab37591df6234a647ecc38b41f72ac6e2c128ebcf167e0bc0 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 8ac7f73d40d5615cdb73660a85e05ee26943a618ec5c7262160994f263fa7178 +F src/select.c f253214cbd2458f744b95fc6bf8dee103cecf4b051f620fd5f62b43dbab92ec1 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23145fe999ff74d787a3999baedd4ffe755c5f1f1275082ed0338ba637ecc56e -R f3df5c5148120a1bf76a131c870fdc09 +P 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 +R 2344df85f68d9987b55d50d38bac47c1 U drh -Z 97820e8b433aee285f0d7a5d438ada65 +Z 4c924f435822db601e489fbe4dcd4a9e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c7c9044f2a..56c84edce5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 \ No newline at end of file +8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index db81983e4a..c34a28dc47 100644 --- a/src/expr.c +++ b/src/expr.c @@ -53,7 +53,7 @@ char sqlite3ExprAffinity(const Expr *pExpr){ } op = pExpr->op; if( op==TK_REGISTER ) op = pExpr->op2; - if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ + if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){ assert( ExprUseYTab(pExpr) ); assert( pExpr->y.pTab!=0 ); return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); @@ -173,7 +173,9 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ while( p ){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; - if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + || op==TK_COLUMN || op==TK_TRIGGER + ){ int j; assert( ExprUseYTab(p) ); assert( p->y.pTab!=0 ); diff --git a/src/select.c b/src/select.c index 49bef86bab..dcb878dde9 100644 --- a/src/select.c +++ b/src/select.c @@ -6222,7 +6222,7 @@ static void printAggInfo(AggInfo *pAggInfo){ sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=\n", + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", ii, AggInfoFuncReg(pAggInfo,ii)); sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); } @@ -6309,6 +6309,40 @@ static void optimizeAggregateUseOfIndexedExpr( #endif } +/* +** Walker callback for aggregateConvertIndexedExprRefToColumn(). +*/ +static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ + AggInfo *pAggInfo; + struct AggInfo_col *pCol; + if( pExpr->pAggInfo==0 ) return WRC_Continue; + if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + pAggInfo = pExpr->pAggInfo; + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + pExpr->op = TK_AGG_COLUMN; + pExpr->iTable = pCol->iTable; + pExpr->iColumn = pCol->iColumn; + return WRC_Prune; +} + +/* +** Convert every pAggInfo->aFunc[].pExpr such that any node within +** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN +** opcode. +*/ +static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ + int i; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = aggregateIdxEprRefToColCallback; + for(i=0; inFunc; i++){ + sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr); + } +} + + /* ** Allocate a block of registers so that there is one register for each ** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first @@ -7669,6 +7703,18 @@ int sqlite3Select( pAggInfo->useSortingIdx = 1; } + if( pParse->pIdxEpr ){ + aggregateConvertIndexedExprRefToColumn(pAggInfo); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20, pParse, p, + ("AggInfo function expressions converted to reference index\n")); + sqlite3TreeViewSelect(0, p, 0); + printAggInfo(pAggInfo); + } +#endif + } + /* If the index or temporary table used by the GROUP BY sort ** will naturally deliver rows in the order required by the ORDER BY ** clause, cancel the ephemeral table open coded earlier. From e79cb67c3575d8254b36eba3f35dba2d62124e07 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 19:03:22 +0000 Subject: [PATCH 062/282] Add an experimental OPFS VFS-specific URI flag, opfs-unlock-asap, which tells the VFS to release implicit locks ASAP. This permits higher concurrency but hurts performance considerably. This may or may not be obsoleted by other concurrency-related experimentation. FossilOrigin-Name: d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 --- ext/wasm/api/sqlite3-api-opfs.js | 27 ++++++++++++++++++++++- ext/wasm/api/sqlite3-opfs-async-proxy.js | 25 +++++---------------- ext/wasm/tests/opfs/concurrency/worker.js | 2 +- manifest | 16 +++++++------- manifest.uuid | 2 +- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 53f68a1d58..deb8bf04e8 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -355,6 +355,25 @@ const installOpfsVfs = function callee(options){ toss("Maintenance required: not found:",k); } }); + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). "opfs-unlock-asap=1" enables + this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + If true, any async routine which implicitly acquires a sync + access handle (i.e. an OPFS lock) will release that locks at + the end of the call which acquires it. If false, such + "autolocks" are not released until the VFS is idle for some + brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). + */ + defaultUnlockAsap: false + }); /** Runs the given operation (by name) in the async worker @@ -845,9 +864,15 @@ const installOpfsVfs = function callee(options){ //xSleep is optionally defined below xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ mTimeStart('xOpen'); + let opfsFlags = 0; if(0===zName){ zName = randomFilename(); }else if('number'===typeof zName){ + if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ + /* -----------------------^^^^^ MUST pass the untranslated + C-string here. */ + opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP; + } zName = wasm.cstringToJs(zName); } const fh = Object.create(null); @@ -855,7 +880,7 @@ const installOpfsVfs = function callee(options){ fh.filename = zName; fh.sab = new SharedArrayBuffer(state.fileBufferSize); fh.flags = flags; - const rc = opRun('xOpen', pFile, zName, flags); + const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); if(!rc){ /* Recall that sqlite3_vfs::xClose() will be called, even on error, unless pFile->pMethods is NULL. */ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index cbe0549b3e..b14494a0c9 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -201,18 +201,6 @@ const releaseImplicitLocks = async ()=>{ } }; -/** - If true, any routine which implicitly acquires a sync access handle - (i.e. an OPFS lock) will release that locks at the end of the call - which acquires it. If false, such "autolocks" are not released - until the VFS is idle for some brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). -*/ -state.defaultReleaseImplicitLocks = false; - /** An experiment in improving concurrency by freeing up implicit locks sooner. This is known to impact performance dramatically but it has @@ -536,7 +524,8 @@ const vfsAsyncImpls = { mTimeEnd(); }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, - flags/*SQLITE_OPEN_...*/){ + flags/*SQLITE_OPEN_...*/, + opfsFlags/*OPFS_...*/){ const opName = 'xOpen'; mTimeStart(opName); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); @@ -566,13 +555,8 @@ const vfsAsyncImpls = { deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); fh.releaseImplicitLocks = - state.defaultReleaseImplicitLocks - /* TODO: check URI flags for "opfs-auto-unlock". First we need to - reshape the API a bit to be able to pass those on to here - from the other half of the proxy. */; - /*if(fh.releaseImplicitLocks){ - console.warn("releaseImplicitLocks is ON for",fh); - }*/ + (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) + || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite does but it leads to horrible contention on journal files. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ @@ -887,6 +871,7 @@ navigator.storage.getDirectory().then(function(d){ state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); state.opIds = opt.opIds; state.sq3Codes = opt.sq3Codes; + state.opfsFlags = opt.opfsFlags; Object.keys(vfsAsyncImpls).forEach((k)=>{ if(!Number.isFinite(state.opIds[k])){ toss("Maintenance required: missing state.opIds[",k,"]"); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 9b4898f4c3..1a896e714e 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -44,7 +44,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const run = async function(){ db = new sqlite3.oo1.DB({ - filename: 'file:'+dbName, + filename: 'file:'+dbName,//+'?opfs-unlock-asap=1'/*EXPERIMENTAL*/, flags: 'c', vfs: 'opfs' }); diff --git a/manifest b/manifest index d6e43cfa43..d9cc7cb9d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\sinfrastructure\sfor\sadding\sa\smode\sto\sthe\sOPFS\sVFS\swhich\scauses\simplicit\slocks\sto\sbe\sreleased\sASAP,\swhich\sincreases\sconcurrency\sat\sthe\scost\sof\sperformance. -D 2022-11-23T16:39:07.866 +C Add\san\sexperimental\sOPFS\sVFS-specific\sURI\sflag,\sopfs-unlock-asap,\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP.\sThis\spermits\shigher\sconcurrency\sbut\shurts\sperformance\sconsiderably.\sThis\smay\sor\smay\snot\sbe\sobsoleted\sby\sother\sconcurrency-related\sexperimentation. +D 2022-11-23T19:03:22.450 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 -F ext/wasm/api/sqlite3-api-opfs.js 69a897eae705816a002f44e8a37693e2ceb7b3e32f9889df2302aeba7df24c70 +F ext/wasm/api/sqlite3-api-opfs.js 23b5c51d7c48134eb5a2d23c1dca06315c738aa365f1c9620e649805c62e5781 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 3142ed3a636d9a1a3f4793c6af6373996593c615a3a5fa29de59ba3e0ea45bee +F ext/wasm/api/sqlite3-opfs-async-proxy.js 20030993ccc04e42b4c7111db31d8f22ca02a00879f183a9067738d7bc9f10b9 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js cc43ea47bb59582523e3f27c2198247c4adca2fae9a891f84dd3bccfccef2833 +F ext/wasm/tests/opfs/concurrency/worker.js e1b10dc5d96117ac58f4eedde97a970816adc60e007f0a0f20a41d880ab59ca9 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 -R 5ffa2a11b06c4a44d92d27455a2fc3e8 +P c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 +R 37d44560e41f4eab685876e5b8c66c33 U stephan -Z 70f5adf76d87ddf8a488ff2bcd290afb +Z 1cab17b5c5d56b08d5be52571107da20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 54a2776a7a..8365ac9b1a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 \ No newline at end of file +d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 \ No newline at end of file From 056a71562f192c7fcd4ca73a3133b60dcf5a83fd Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 20:49:08 +0000 Subject: [PATCH 063/282] OPFS concurrency test: add a URL flag to enable/disable unlock-asap mode. FossilOrigin-Name: 1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 --- ext/wasm/tests/opfs/concurrency/test.js | 6 +++++- ext/wasm/tests/opfs/concurrency/worker.js | 11 +++++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 27bc47b19d..cb9d4275be 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -63,6 +63,9 @@ options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 ) || 750; + options.unlockAsap = ( + urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 + ) || 0; const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); @@ -92,12 +95,13 @@ } }; - stdout("Launching",options.workerCount,"workers..."); + stdout("Launching",options.workerCount,"workers. Options:",options); workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + '&interval='+options.interval + '&opfs-verbose='+options.opfsVerbose + + '&opfs-unlock-asap='+options.unlockAsap ); for(let i = 0; i < options.workerCount; ++i){ stdout("Launching worker..."); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 1a896e714e..91aa0fa6fa 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -3,9 +3,12 @@ importScripts( ); self.sqlite3InitModule().then(async function(sqlite3){ const urlArgs = new URL(self.location.href).searchParams; - const wName = urlArgs.get('workerId') || Math.round(Math.random()*10000); + const options = { + workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ + }; const wPost = (type,...payload)=>{ - postMessage({type, worker: wName, payload}); + postMessage({type, worker: options.workerName, payload}); }; const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); @@ -44,7 +47,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const run = async function(){ db = new sqlite3.oo1.DB({ - filename: 'file:'+dbName,//+'?opfs-unlock-asap=1'/*EXPERIMENTAL*/, + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c', vfs: 'opfs' }); @@ -66,7 +69,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ try{ db.exec({ sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", - bind: [wName, new Date().getTime()] + bind: [options.workerName, new Date().getTime()] }); //stdout("Set",prefix); }catch(e){ diff --git a/manifest b/manifest index d9cc7cb9d0..a6b0539bab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\sOPFS\sVFS-specific\sURI\sflag,\sopfs-unlock-asap,\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP.\sThis\spermits\shigher\sconcurrency\sbut\shurts\sperformance\sconsiderably.\sThis\smay\sor\smay\snot\sbe\sobsoleted\sby\sother\sconcurrency-related\sexperimentation. -D 2022-11-23T19:03:22.450 +C OPFS\sconcurrency\stest:\sadd\sa\sURL\sflag\sto\senable/disable\sunlock-asap\smode. +D 2022-11-23T20:49:08.427 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -553,8 +553,8 @@ F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b51073 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js e1b10dc5d96117ac58f4eedde97a970816adc60e007f0a0f20a41d880ab59ca9 +F ext/wasm/tests/opfs/concurrency/test.js 9a937068b66a0cfbb9cb6833cb001ce22f9d0f8f765775e3456860b05db21797 +F ext/wasm/tests/opfs/concurrency/worker.js 2a275c4983016365cac18c9105f1ac7bd2adbc7ad89cc91b363d722f2bb55cb5 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 -R 37d44560e41f4eab685876e5b8c66c33 +P d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 +R ff8c3df82102d217683515f7779a95c4 U stephan -Z 1cab17b5c5d56b08d5be52571107da20 +Z 269769e48cdf286c8fb19455cdae3a52 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8365ac9b1a..bb0d09842b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 \ No newline at end of file +1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 \ No newline at end of file From 875db41afca4447f00ecbb554e5268017a4c8b87 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 21:03:22 +0000 Subject: [PATCH 064/282] Add optional zSchema argument to sqlite3_js_db_export(). FossilOrigin-Name: 9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa --- ext/wasm/api/sqlite3-api-prologue.js | 16 +++++++++++----- ext/wasm/api/sqlite3-wasm.c | 20 +++++++++++--------- manifest | 17 ++++++++--------- manifest.uuid | 2 +- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 8b2ce0936d..f9ad5b5127 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1310,14 +1310,17 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( Serializes the given `sqlite3*` pointer to a Uint8Array, as per sqlite3_serialize(). On success it returns a Uint8Array. On error it throws with a description of the problem. + + schema is the schema to serialize. It may be a WASM C-string + pointer or a JS string. If it is falsy, it defaults to "main". */ - capi.sqlite3_js_db_export = function(pDb){ + capi.sqlite3_js_db_export = function(pDb, schema=0){ if(!pDb) toss3('Invalid sqlite3* argument.'); if(!wasm.bigIntEnabled) toss3('BigInt64 support is not enabled.'); - const stack = wasm.pstack.pointer; + const scope = wasm.scopedAllocPush(); let pOut; try{ - const pSize = wasm.pstack.alloc(8/*i64*/ + wasm.ptrSizeof); + const pSize = wasm.scopedAlloc(8/*i64*/ + wasm.ptrSizeof); const ppOut = pSize + 8; /** Maintenance reminder, since this cost a full hour of grief @@ -1326,8 +1329,11 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( export reads a garbage size because it's not on an 8-byte memory boundary! */ + const zSchema = schema + ? (wasm.isPtr(schema) ? schema : wasm.scopedAllocCString(''+schema)) + : 0; let rc = wasm.exports.sqlite3_wasm_db_serialize( - pDb, ppOut, pSize, 0 + pDb, zSchema, ppOut, pSize, 0 ); if(rc){ toss3("Database serialization failed with code", @@ -1341,7 +1347,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return rc; }finally{ if(pOut) wasm.exports.sqlite3_free(pOut); - wasm.pstack.restore(stack); + wasm.scopedAllocPop(scope); } }; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index af5ed6bf76..67e97c4a03 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -916,25 +916,27 @@ int sqlite3_wasm_db_export_chunked( sqlite3* pDb, } /* -** A proxy for sqlite3_serialize() which serializes the "main" schema +** A proxy for sqlite3_serialize() which serializes the schema zSchema ** of pDb, placing the serialized output in pOut and nOut. nOut may be -** NULL. If pDb or pOut are NULL then SQLITE_MISUSE is returned. If -** allocation of the serialized copy fails, SQLITE_NOMEM is returned. -** On success, 0 is returned and `*pOut` will contain a pointer to the -** memory unless mFlags includes SQLITE_SERIALIZE_NOCOPY and the -** database has no contiguous memory representation, in which case -** `*pOut` will be NULL but 0 will be returned. +** NULL. If zSchema is NULL then "main" is assumed. If pDb or pOut are +** NULL then SQLITE_MISUSE is returned. If allocation of the +** serialized copy fails, SQLITE_NOMEM is returned. On success, 0 is +** returned and `*pOut` will contain a pointer to the memory unless +** mFlags includes SQLITE_SERIALIZE_NOCOPY and the database has no +** contiguous memory representation, in which case `*pOut` will be +** NULL but 0 will be returned. ** ** If `*pOut` is not NULL, the caller is responsible for passing it to ** sqlite3_free() to free it. */ SQLITE_WASM_KEEP -int sqlite3_wasm_db_serialize( sqlite3 *pDb, unsigned char **pOut, +int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, + unsigned char **pOut, sqlite3_int64 *nOut, unsigned int mFlags ){ unsigned char * z; if( !pDb || !pOut ) return SQLITE_MISUSE; if(nOut) *nOut = 0; - z = sqlite3_serialize(pDb, "main", nOut, mFlags); + z = sqlite3_serialize(pDb, zSchema ? zSchema : "main", nOut, mFlags); if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){ *pOut = z; return 0; diff --git a/manifest b/manifest index 3ecb281109..12cb76e8ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMakefile.in\sto\sinclude\snew\starget\s"sqlite3r.c".\sFor\sgenerating\s"sqlite3r.c"\sand\s"sqlite3r.h",\sversions\sof\sthe\samalgamation\sthat\sinclude\sthe\srecover\sextension.\sTo\sbuild\sthe\sshell\stool\sagainst\sthese\sfiles,\sadd\s-DSQLITE_HAVE_SQLITE3R. -D 2022-11-23T16:08:49.765 +C Add\soptional\szSchema\sargument\sto\ssqlite3_js_db_export(). +D 2022-11-23T21:03:22.603 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,12 +503,12 @@ F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b -F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 +F ext/wasm/api/sqlite3-api-prologue.js fa28d080a28bb936348156f7a298e0a001be088fb439905ff2dd85146f4fd2a0 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 +F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2059,9 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -R 6b5395a59674684ae7409c1a67752bd1 -T +closed 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -U dan -Z 08a51ae8157c87edcee2b45d299f4155 +P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 +R f8ada562338905eb066689b281095c3e +U stephan +Z 3d9c890d275324696db830c3a91a1a42 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c19a5133e..b3bb5e9e42 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 \ No newline at end of file +9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa \ No newline at end of file From f1ce4f40c7a01fd272662ab42d24b1f3ba747069 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 21:09:51 +0000 Subject: [PATCH 065/282] Minor JS doc updates. FossilOrigin-Name: 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 --- ext/wasm/api/sqlite3-api-prologue.js | 14 +++++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index f9ad5b5127..1cddc8eaf6 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1307,12 +1307,16 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( }; /** - Serializes the given `sqlite3*` pointer to a Uint8Array, as per - sqlite3_serialize(). On success it returns a Uint8Array. On - error it throws with a description of the problem. + A convenience wrapper around sqlite3_serialize() which serializes + the given `sqlite3*` pointer to a Uint8Array. - schema is the schema to serialize. It may be a WASM C-string - pointer or a JS string. If it is falsy, it defaults to "main". + On success it returns a Uint8Array. If the schema is empty, an + empty array is returned. + + `schema` is the schema to serialize. It may be a WASM C-string + pointer or a JS string. If it is falsy, it defaults to `"main"`. + + On error it throws with a description of the problem. */ capi.sqlite3_js_db_export = function(pDb, schema=0){ if(!pDb) toss3('Invalid sqlite3* argument.'); diff --git a/manifest b/manifest index 12cb76e8ab..c042b8b5eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\soptional\szSchema\sargument\sto\ssqlite3_js_db_export(). -D 2022-11-23T21:03:22.603 +C Minor\sJS\sdoc\supdates. +D 2022-11-23T21:09:51.213 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,7 @@ F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b -F ext/wasm/api/sqlite3-api-prologue.js fa28d080a28bb936348156f7a298e0a001be088fb439905ff2dd85146f4fd2a0 +F ext/wasm/api/sqlite3-api-prologue.js 58caf45e4cc0deec9bdddb051ce17408686f2bc21ea7b3ee0b4c4deaba532605 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 -R f8ada562338905eb066689b281095c3e +P 9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa +R 8f8d491e9ca84b75fecb920dbf623854 U stephan -Z 3d9c890d275324696db830c3a91a1a42 +Z a1e4b87d56bdef1be072540c12f0e1a7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b3bb5e9e42..c0426c7035 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa \ No newline at end of file +27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 \ No newline at end of file From 9b80cb5f39af66075762057df127ffbbb2c6ef2a Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 01:40:20 +0000 Subject: [PATCH 066/282] Add explanatory comment to the new optimization. And add a test case. FossilOrigin-Name: e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 --- manifest | 13 ++++--- manifest.uuid | 2 +- src/select.c | 5 +++ test/tkt-99378177930f87bd.test | 69 ++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 test/tkt-99378177930f87bd.test diff --git a/manifest b/manifest index 5a5586e90e..9476f82d3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Aggregates\swith\sGROUP\sBY\snow\smake\suse\sof\sexpressions\son\sindexes.\s\sThis\scode\nworks\sand\sgets\sthe\scorrect\sanswer\sfor\sthe\stest\scase\sin\sthe\sticket.\s\sLots\smore\ntesting\sand\sdocumentation\sis\sneeded,\showever. -D 2022-11-23T18:51:04.363 +C Add\sexplanatory\scomment\sto\sthe\snew\soptimization.\s\sAnd\sadd\sa\stest\scase. +D 2022-11-24T01:40:20.262 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c f253214cbd2458f744b95fc6bf8dee103cecf4b051f620fd5f62b43dbab92ec1 +F src/select.c 7fcbbc0db92d0082e3a10b4492457932d8589e9027e843ac2b972a4ba0233136 F src/shell.c.in 7d1705f139e6762e8c0fe254a8ebf3ab77aec6d8366f033cdd5f5ebadefbbb20 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1628,6 +1628,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 +F test/tkt-99378177930f87bd.test f33bf2e038025941eb2f5495db4d4e3f5a614cdf6c44e0fe4d420d41197330c1 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2059,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84c06023f4a1606664fdb9811312603b31f7c94a43d0e443ba7dde7fdba029e3 -R 2344df85f68d9987b55d50d38bac47c1 +P 8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f +R 9274a8d11a500d33c0e780bbd7f350f0 U drh -Z 4c924f435822db601e489fbe4dcd4a9e +Z b2115b9d996407ff47bc262d3bc9ad3e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 56c84edce5..cdcf19b4bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dcf9f2031c16f296d187fe876d4204c71fc96fec120984ff11b6d8b03d58a5f \ No newline at end of file +e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 \ No newline at end of file diff --git a/src/select.c b/src/select.c index dcb878dde9..dcf899e4bb 100644 --- a/src/select.c +++ b/src/select.c @@ -7703,6 +7703,11 @@ int sqlite3Select( pAggInfo->useSortingIdx = 1; } + /* If there entries in pAgggInfo->aFunc[] that contain subexpressions + ** that are indexed (and that were previously identified and tagged + ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions + ** must now be converted into a TK_AGG_COLUMN node so that the value + ** is correctly pulled from the index rather than being recomputed. */ if( pParse->pIdxEpr ){ aggregateConvertIndexedExprRefToColumn(pAggInfo); #if TREETRACE_ENABLED diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test new file mode 100644 index 0000000000..aad0a6dc21 --- /dev/null +++ b/test/tkt-99378177930f87bd.test @@ -0,0 +1,69 @@ +# 2022-11-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests to verify that the enhancement +# request documented by ticket 99378177930f87bd is working. +# +# The enhancement is that if an aggregate query with a GROUP BY clause +# uses subexpressions in the arguments to aggregate functions that are +# also columns of an index, then the values are pulled from the index +# rather than being recomputed. This has the potential to make some +# indexed queries works as if the index were covering. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test tkt-99378-100 { + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} {} +do_execsql_test tkt-99378-110 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} + +# The proof that the index on the expression is being used is in the +# fact that the byte code contains no "Function" opcodes. In other words, +# the ->> operator (which is implemented by a function) is never invoked. +# Instead, the b->>'x' value is pulled out of the index. +# +do_execsql_test tkt-99378-120 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + +finish_test From f46091d73fb059de729fe780a8d2eb4ff8056ddc Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 24 Nov 2022 02:35:03 +0000 Subject: [PATCH 067/282] Add sqlite3.oo1.DB.prototype.checkRc() and tests for both that method and its class-level counterpart. FossilOrigin-Name: f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 --- ext/wasm/api/sqlite3-api-oo1.js | 27 +++++++++++++++++++-------- ext/wasm/tester1.c-pp.js | 10 ++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 02ce9c0ced..fc0f7372b8 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -55,12 +55,13 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(sqliteResultCode){ if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; toss3( - "sqlite result code",sqliteResultCode+":", + "sqlite3 result code",sqliteResultCode+":", (dbPtr ? capi.sqlite3_errmsg(dbPtr) : capi.sqlite3_errstr(sqliteResultCode)) ); } + return arguments[0]; }; /** @@ -462,14 +463,16 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Expects to be given a DB instance or an `sqlite3*` pointer (may be null) and an sqlite3 API result code. If the result code is not falsy, this function throws an SQLite3Error with an error - message from sqlite3_errmsg(), using dbPtr as the db handle, or - sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a - non-error code like SQLITE_ROW or SQLITE_DONE, it will still - throw but the error string might be "Not an error." The various - non-0 non-error codes need to be checked for in - client code where they are expected. + message from sqlite3_errmsg(), using db (or, if db is-a DB, + db.pointer) as the db handle, or sqlite3_errstr() if db is + falsy. Note that if it's passed a non-error code like SQLITE_ROW + or SQLITE_DONE, it will still throw but the error string might be + "Not an error." The various non-0 non-error codes need to be + checked for in client code where they are expected. + + If it does not throw, it returns its first argument. */ - DB.checkRc = checkSqlite3Rc; + DB.checkRc = (db,resultCode)=>checkSqlite3Rc(db,resultCode); DB.prototype = { /** Returns true if this db handle is open, else false. */ @@ -1130,6 +1133,14 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this.exec("ROLLBACK to SAVEPOINT oo1; RELEASE SAVEPOINT oo1"); throw e; } + }, + + /** + A convenience form of DB.checkRc(this,resultCode). If it does + not throw, it returns this object. + */ + checkRc: function(resultCode){ + return DB.checkRc(this, resultCode); } }/*DB.prototype*/; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 2a5da8407c..5bb5959396 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1163,6 +1163,16 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) .assert(dbFile === db.dbFilename()) .assert(!db.dbFilename('nope')); + //Sanity check DB.checkRc()... + let ex; + try{db.checkRc(rc)} + catch(e){ex = e} + T.assert(ex instanceof sqlite3.SQLite3Error) + .assert(0===ex.message.indexOf("sqlite3 result code")) + .assert(ex.message.indexOf("Invalid SQL")>0); + T.assert(db === db.checkRc(0)) + .assert(db === sqlite3.oo1.DB.checkRc(db,0)) + .assert(null === sqlite3.oo1.DB.checkRc(null,0)) }) //////////////////////////////////////////////////////////////////// diff --git a/manifest b/manifest index c042b8b5eb..a47d3eccc2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sJS\sdoc\supdates. -D 2022-11-23T21:09:51.213 +C Add\ssqlite3.oo1.DB.prototype.checkRc()\sand\stests\sfor\sboth\sthat\smethod\sand\sits\sclass-level\scounterpart. +D 2022-11-24T02:35:03.924 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,7 +501,7 @@ F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b387181 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 -F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed +F ext/wasm/api/sqlite3-api-oo1.js dec6c14994317ad0011714890426cdc211f4eab451c9766ea88c7ac4f535287e F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b F ext/wasm/api/sqlite3-api-prologue.js 58caf45e4cc0deec9bdddb051ce17408686f2bc21ea7b3ee0b4c4deaba532605 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f @@ -551,7 +551,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d +F ext/wasm/tester1.c-pp.js 3b91f192c159088004fba6fe3441edea58421a8b88bccf3dd20978a077648d19 F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9c23644b1e5bf44bfb431a35fd1674c11ccb99e9eb0989f10175b0cb2a858eaa -R 8f8d491e9ca84b75fecb920dbf623854 +P 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 +R 25ba86c410fa1767dbcf638c18b7bc32 U stephan -Z a1e4b87d56bdef1be072540c12f0e1a7 +Z 6ecc8a9a17b94657e9cbb62c36aba3fc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0426c7035..1c21bd8831 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 \ No newline at end of file +f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 \ No newline at end of file From 9b9017def4edc224e0f4f773b775cfa575cf198b Mon Sep 17 00:00:00 2001 From: larrybr Date: Thu, 24 Nov 2022 02:59:33 +0000 Subject: [PATCH 068/282] Speed up base64 conversions, and add test with more data for the baseNN conversion to grind. FossilOrigin-Name: 6c84ae4ba83713c751fddff8be41686bbcb525ac8135e1520436c62d0bc23d2c --- ext/misc/base64.c | 70 +++++++++++++++++++++++++++-------------------- manifest | 14 +++++----- manifest.uuid | 2 +- test/basexx1.test | 14 ++++++++++ 4 files changed, 62 insertions(+), 38 deletions(-) mode change 100644 => 100755 ext/misc/base64.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c old mode 100644 new mode 100755 index 6e3a6b5cb7..7bb7683d1f --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -92,42 +92,46 @@ static const char b64Numerals[64+1] #define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80) #define IS_BX_WS(bdp) ((bdp)==WS) #define IS_BX_PAD(bdp) ((bdp)==PC) -#define BX_NUMERAL(dv) (b64Numerals[dv]) +#define BX_NUMERAL(dv) (b64Numerals[(ubyte)(dv)]) /* Width of base64 lines. Should be an integer multiple of 4. */ #define B64_DARK_MAX 72 -/* Encode a byte buffer into base64 text. If pSep!=0, it's a C string -** to be appended to encoded groups to limit their length to B64_DARK_MAX -** or to terminate the last group (to aid concatenation.) +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. */ -static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ +static char* toBase64( ubyte *pIn, int nbIn, char *pOut ){ int nCol = 0; - *pOut = 0; - while( nbIn > 0 ){ - static signed char ncio[] = { 0, 2, 3, 4 }; - int nbi = (nbIn > 3)? 3 : nbIn; - signed char nc; - int nbe; - unsigned long qv = (ubyte)*pIn++; - for( nbe=1; nbe<3; ++nbe ){ - ubyte b = (nbe= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ - char ce = (nbe>= 6; pOut[nbe] = ce; } pOut += 4; - if( pSep && ((nCol += 4)>=B64_DARK_MAX || nbIn<=0) ){ - char *p = pSep; - while( *p ) *pOut++ = *p++; - nCol = 0; - } - *pOut = 0; + *pOut++ = '\n'; } + *pOut = 0; return pOut; } @@ -157,11 +161,12 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ ubyte bdp = BX_DV_PROTO(c); switch( bdp ){ case ND: - /* Treat non-digits as pad, but they terminate decode too. */ + /* Treat dark non-digits as pad, but they terminate decode too. */ ncIn = 0; /* fall thru */ case WS: - /* Treat whitespace as pad */ + /* Treat whitespace as pad and terminate this group.*/ + nti = nac; /* fall thru */ case PC: bdp = 0; @@ -172,10 +177,15 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ break; } } - nti = 2; - while( nbo-- > 0 ){ - *pOut++ = (qv >> (8*nti--))&0xff; + switch( nbo ){ + case 3: + pOut[2] = (qv) & 0xff; + case 2: + pOut[1] = (qv>>8) & 0xff; + case 1: + pOut[0] = (qv>>16) & 0xff; } + pOut += nbo; } return pOut; } @@ -200,7 +210,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; bBuf = (ubyte*)sqlite3_value_blob(av[0]); - nc = (int)(toBase64(bBuf, nb, cBuf, "\n") - cBuf); + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; case SQLITE_TEXT: diff --git a/manifest b/manifest index 6d54a98758..56f984f662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smore\sbaseNN\stests,\sget\soversize\serror\strapping\sworking,\sand\ssync\sw/trunk -D 2022-11-22T22:46:41.866 +C Speed\sup\sbase64\sconversions,\sand\sadd\stest\swith\smore\sdata\sfor\sthe\sbaseNN\sconversion\sto\sgrind. +D 2022-11-24T02:59:33.748 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,7 +289,7 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c 0472f388e1a6c168912a363dfbe88d2d588325fba7ce6101f726d01b54fe6d3b +F ext/misc/base64.c 6333194e5c2e85b0748116ad4004bf3e070347cc09984aaa8d462fb3fc0566b6 x F ext/misc/base85.c 9005549904fc06ec2f3ff96970709f92f76e2d9ec2b785553ac32908ddc1baa0 F ext/misc/basexx.c 678dcc83894f78c26fd3662b322886777cc26bf2b40809236cd2abdad532a33c F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a @@ -816,7 +816,7 @@ F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f310fd3b24a491b6b77bccdf14923b85d6ebcce751068c180d93a6b8ff854399 -F test/basexx1.test 9b12557d2b5bd017f9f8a8698239438ced5899c3ee55d9a549f74d90b16e51a6 +F test/basexx1.test d8a50f0744b93dca656625597bcd3499ff4b9a4ea2a82432b119b7d46e3e0c08 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c F test/bestindex1.test 856a453dff8c68b4568601eed5a8b5e20b4763af9229f3947c215729ed878db0 F test/bestindex2.test 394ff8fbf34703391247116d6a44e1c50ee7282236ee77909044573cefc37bc0 @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff67460e1a3d21c9ca7cbd171fbc5e6cbdb3797de359887f851776b73b732fdf 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 -R 36565c16629af64673142ebb24820a85 +P 03819e9368fd9f78f351147a1dc865743f9634893e43a9d1e3d7cbaf4c966069 +R f94c977ea1969bc60efbcf2158a80ba4 U larrybr -Z 3e47c0e49ec42d7b0a58333b563577b4 +Z 312bc23355f0a0c3b5b7524b3c0fa709 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 825fd93e96..11257f9dab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03819e9368fd9f78f351147a1dc865743f9634893e43a9d1e3d7cbaf4c966069 \ No newline at end of file +6c84ae4ba83713c751fddff8be41686bbcb525ac8135e1520436c62d0bc23d2c \ No newline at end of file diff --git a/test/basexx1.test b/test/basexx1.test index 8855a78634..947a5678f3 100644 --- a/test/basexx1.test +++ b/test/basexx1.test @@ -138,4 +138,18 @@ do_catchsql_test 116 { SELECT is_base85(x'00'); } {1 {is_base85 accepts only text or NULL}} +# Round-trip many bigger random blobs. + +do_execsql_test 117 { + CREATE TABLE bs(b blob, num); + INSERT INTO bs SELECT randomblob(4000 + n%3), n + FROM ( + WITH RECURSIVE seq(n) AS ( + VALUES(1) UNION ALL SELECT n+1 + FROM seq WHERE n<100 + ) SELECT n FROM seq); + SELECT num FROM bs WHERE base64(base64(b))!=b; + SELECT num FROM bs WHERE base85(base85(b))!=b; +} {} + finish_test From b669bb5e2bd130b2a8d8a87304449cfb3b27f9d9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 13:19:25 +0000 Subject: [PATCH 069/282] New test cases. Fix the logic so that it works for GROUP BY aggregates that do not require sorting. FossilOrigin-Name: ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef --- manifest | 14 +++--- manifest.uuid | 2 +- src/expr.c | 5 ++ test/tkt-99378177930f87bd.test | 85 ++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 19b2b20356..7f64f61129 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sagg-with-indexed-expr\sbranch\sto\nsimplify\sdiffs. -D 2022-11-24T01:41:44.524 +C New\stest\scases.\s\sFix\sthe\slogic\sso\sthat\sit\sworks\sfor\sGROUP\sBY\saggregates\nthat\sdo\snot\srequire\ssorting. +D 2022-11-24T13:19:25.140 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 141af8139174010ab37591df6234a647ecc38b41f72ac6e2c128ebcf167e0bc0 +F src/expr.c 44a7f638eebe915ca7522464f4753eb767bf37139c0010017eae1c60cb02345f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -1628,7 +1628,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 -F test/tkt-99378177930f87bd.test f33bf2e038025941eb2f5495db4d4e3f5a614cdf6c44e0fe4d420d41197330c1 +F test/tkt-99378177930f87bd.test 0f932e85fa1d41f30532cb7be9718d82e491e953123b8c4c85cf025f36ffe34b F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6c20f61de7d048eee65c8e74a3eb36760ab9747ebd1ab50e49642b777c10306 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 -R beb6109a205ba56e52e9073d9db7b007 +P 38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 +R fe9857b5384581f396eba27025a5561f U drh -Z 9965e8c3c2c486ad5a3aae007bdf6965 +Z d9a080fe6c44b0ca440456485b55fa7a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6fe2b265a2..016d1fad45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 \ No newline at end of file +ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c34a28dc47..2e909dd8ca 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4145,6 +4145,11 @@ expr_code_doover: } } return target; + }else if( pExpr->y.pTab==0 ){ + /* This case happens when the argument to an aggregate function + ** is rewritten by aggregateConvertIndexedExprRefToColumn() */ + sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target); + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ /* no break */ deliberate_fall_through diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index aad0a6dc21..b21be3f67f 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -66,4 +66,89 @@ do_execsql_test tkt-99378-120 { GROUP BY a; } {~/Function/} + +do_execsql_test tkt-99378-130 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-140 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} {~/Function/} + +do_execsql_test tkt-99378-200 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(a, d, b->>'x', c); +} +do_execsql_test tkt-99378-210 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-220 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} +do_execsql_test tkt-99378-230 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-240 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + + + + finish_test From c25f5ea6e838dabbd0721947a432c4d9717bda74 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 15:04:23 +0000 Subject: [PATCH 070/282] Add NEVER() and ALWAYS() macros on branches that are believed to be unreachable. FossilOrigin-Name: 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 4 ++-- src/select.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7f64f61129..c36f686e3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases.\s\sFix\sthe\slogic\sso\sthat\sit\sworks\sfor\sGROUP\sBY\saggregates\nthat\sdo\snot\srequire\ssorting. -D 2022-11-24T13:19:25.140 +C Add\sNEVER()\sand\sALWAYS()\smacros\son\sbranches\sthat\sare\sbelieved\sto\sbe\nunreachable. +D 2022-11-24T15:04:23.564 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 44a7f638eebe915ca7522464f4753eb767bf37139c0010017eae1c60cb02345f +F src/expr.c 07d4a0f36cd0fd52862ed25dffd4a746c4e12b6c9ceda3f87772ae373fb436a5 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7fcbbc0db92d0082e3a10b4492457932d8589e9027e843ac2b972a4ba0233136 +F src/select.c 6df5dd2a0434f0921f82cd23449e75138f05435025aed37ad96f344a7a31dcdb F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38c3d3f1ed0fd2bb62aa8a7e5a27f2b247123e094e2fdb0a2475d788c3dfbc04 -R fe9857b5384581f396eba27025a5561f +P ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef +R eb963117dcb4b03d7c27aa3277dea1bb U drh -Z d9a080fe6c44b0ca440456485b55fa7a +Z 65f0cab32e4b61b60d1aa8b06e205831 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 016d1fad45..17ef3a10aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef \ No newline at end of file +3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 2e909dd8ca..dc918e0bd6 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6335,14 +6335,14 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ int iDataCur = pIEpr->iDataCur; if( iDataCur<0 ) continue; - if( pParse->iSelfTab ){ + if( NEVER(pParse->iSelfTab) ){ if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; iDataCur = -1; } if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; } if( pIEpr==0 ) break; - if( !ExprUseYTab(pExpr) ) break; + if( NEVER(!ExprUseYTab(pExpr)) ) break; /* If we reach this point, it means that expression pExpr can be ** translated into a reference to an index column as described by diff --git a/src/select.c b/src/select.c index dcf899e4bb..e9b6296713 100644 --- a/src/select.c +++ b/src/select.c @@ -6282,7 +6282,7 @@ static void optimizeAggregateUseOfIndexedExpr( NameContext *pNC /* Name context used to resolve agg-func args */ ){ pAggInfo->nColumn = pAggInfo->nAccumulator; - if( pAggInfo->nSortingColumn>0 ){ + if( ALWAYS(pAggInfo->nSortingColumn>0) ){ if( pAggInfo->nColumn==0 ){ pAggInfo->nSortingColumn = 0; }else{ From 647b0dd12d23c084eaa037abc99ce0fb097d3b51 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Nov 2022 15:32:00 +0000 Subject: [PATCH 071/282] Update multiplex3.test to account for the fact that the multiplexor xDelete method may return an error even if it manages to delete the first chunk of a file. FossilOrigin-Name: 1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/multiplex3.test | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a47d3eccc2..f025974fe5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssqlite3.oo1.DB.prototype.checkRc()\sand\stests\sfor\sboth\sthat\smethod\sand\sits\sclass-level\scounterpart. -D 2022-11-24T02:35:03.924 +C Update\smultiplex3.test\sto\saccount\sfor\sthe\sfact\sthat\sthe\smultiplexor\sxDelete\smethod\smay\sreturn\san\serror\seven\sif\sit\smanages\sto\sdelete\sthe\sfirst\schunk\sof\sa\sfile. +D 2022-11-24T15:32:00.589 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1340,7 +1340,7 @@ F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a -F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 +F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f92f6a1f7 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 F test/mutex1.test 177db2e4edb530f2ff21edc52ac79a412dbe63e4c47c3ae9504d3fb4f1ce81fa F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27efd63ad7fb3bf8d0d07f2c9f48652c8cacc4e697c229c8085120a8e6b50a39 -R 25ba86c410fa1767dbcf638c18b7bc32 -U stephan -Z 6ecc8a9a17b94657e9cbb62c36aba3fc +P f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 +R f39101951334943bf67c026388719509 +U dan +Z db7c150da52ab80c381cc3a21444c8d8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1c21bd8831..92bbfc67cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 \ No newline at end of file +1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 \ No newline at end of file diff --git a/test/multiplex3.test b/test/multiplex3.test index c1e741acdb..3188350163 100644 --- a/test/multiplex3.test +++ b/test/multiplex3.test @@ -82,6 +82,8 @@ do_faultsim_test 1 -prep { multiplex_restore_db sqlite3 db file:test.db?8_3_names=1 sqlite3_multiplex_control db main chunk_size [expr 256*1024] + execsql { PRAGMA journal_mode = truncate } + execsql { PRAGMA synchronous = off } } -body { execsql { UPDATE t1 SET a=randomblob(12), b=randomblob(1500) WHERE (rowid%32)=0 From 6b86b33793421dc4136114fd6a7bd4f8e46f84c5 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 24 Nov 2022 16:03:49 +0000 Subject: [PATCH 072/282] JS documentation cleanups. No code changes. FossilOrigin-Name: 18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 --- ext/wasm/api/sqlite3-api-prologue.js | 102 ++++++--------------------- manifest | 14 ++-- manifest.uuid | 2 +- 3 files changed, 29 insertions(+), 89 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 1cddc8eaf6..726f47e18d 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -11,76 +11,18 @@ *********************************************************************** This file is intended to be combined at build-time with other - related code, most notably a header and footer which wraps this whole - file into an Emscripten Module.postRun() handler which has a parameter - named "Module" (the Emscripten Module object). The exact requirements, - conventions, and build process are very much under construction and - will be (re)documented once they've stopped fluctuating so much. + related code, most notably a header and footer which wraps this + whole file into an Emscripten Module.postRun() handler which has a + parameter named "Module" (the Emscripten Module object). The sqlite3 + JS API has no hard requirements on Emscripten, and does not expose + any Emscripten APIs to clients. It is structured such that its build + can be tweaked to include it in arbitrary WASM environments which + supply the necessary underlying features (e.g. a POSIX file I/O + layer). - Project home page: https://sqlite.org + Main project home page: https://sqlite.org Documentation home page: https://sqlite.org/wasm - - Specific goals of this subproject: - - - Except where noted in the non-goals, provide a more-or-less - feature-complete wrapper to the sqlite3 C API, insofar as WASM - feature parity with C allows for. In fact, provide at least 4 - APIs... - - 1) 1-to-1 bindings as exported from WASM, with no automatic - type conversions between JS and C. - - 2) A binding of (1) which provides certain JS/C type conversions - to greatly simplify its use. - - 3) A higher-level API, more akin to sql.js and node.js-style - implementations. This one speaks directly to the low-level - API. This API must be used from the same thread as the - low-level API. - - 4) A second higher-level API which speaks to the previous APIs via - worker messages. This one is intended for use in the main - thread, with the lower-level APIs installed in a Worker thread, - and talking to them via Worker messages. Because Workers are - asynchronouns and have only a single message channel, some - acrobatics are needed here to feed async work results back to - the client (as we cannot simply pass around callbacks between - the main and Worker threads). - - - Insofar as possible, support client-side storage using JS - filesystem APIs. As of this writing, such things are still very - much under development. - - Specific non-goals of this project: - - - As WASM is a web-centric technology and UTF-8 is the King of - Encodings in that realm, there are no currently plans to support - the UTF16-related sqlite3 APIs. They would add a complication to - the bindings for no appreciable benefit. Though web-related - implementation details take priority, and the JavaScript - components of the API specifically focus on browser clients, the - lower-level WASM module "should" work in non-web WASM - environments. - - - Supporting old or niche-market platforms. WASM is built for a - modern web and requires modern platforms. - - - Though scalar User-Defined Functions (UDFs) may be created in - JavaScript, there are currently no plans to add support for - aggregate and window functions. - - Attribution: - - This project is endebted to the work of sql.js: - - https://github.com/sql-js/sql.js - - sql.js was an essential stepping stone in this code's development as - it demonstrated how to handle some of the WASM-related voodoo (like - handling pointers-to-pointers and adding JS implementations of - C-bound callback functions). These APIs have a considerably - different shape than sql.js's, however. */ /** @@ -90,6 +32,10 @@ for the current environment, and then optionally be removed from the global object using `delete self.sqlite3ApiBootstrap`. + This function is not intended for client-level use. It is intended + for use in creating bundles configured for specific WASM + environments. + This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so that it can be used without any _direct_ dependency on @@ -126,11 +72,11 @@ environment. Defaults to `"free"`. - `wasmfsOpfsDir`[^1]: if the environment supports persistent - storage, this directory names the "mount point" for that - directory. It must be prefixed by `/` and may contain only a - single directory-name part. Using the root directory name is not - supported by any current persistent backend. This setting is - only used in WASMFS-enabled builds. + storage using OPFS-over-WASMFS , this directory names the "mount + point" for that directory. It must be prefixed by `/` and may + contain only a single directory-name part. Using the root + directory name is not supported by any current persistent + backend. This setting is only used in WASMFS-enabled builds. [^1] = This property may optionally be a function, in which case this @@ -191,11 +137,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( const capi = Object.create(null); /** Holds state which are specific to the WASM-related - infrastructure and glue code. It is not expected that client - code will normally need these, but they're exposed here in case - it does. These APIs are _not_ to be considered an - official/stable part of the sqlite3 WASM API. They may change - as the developers' experience suggests appropriate changes. + infrastructure and glue code. Note that a number of members of this object are injected dynamically after the api object is fully constructed, so @@ -228,7 +170,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( result of sqlite3.capi.sqlite3_js_rc_str() or (if that returns falsy) a synthesized string which contains that integer. - - If passed 2 arguments and the 2nd is a object, it bevaves + - If passed 2 arguments and the 2nd is a object, it behaves like the Error(string,object) constructor except that the first argument is subject to the is-integer semantics from the previous point. @@ -686,9 +628,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** The WASM IR (Intermediate Representation) value for pointer-type values. It MUST refer to a value type of the - size described by this.ptrSizeof _or_ it may be any value - which ends in '*', which Emscripten's glue code internally - translates to i32. + size described by this.ptrSizeof. */ ptrIR: config.wasmPtrIR || "i32", /** diff --git a/manifest b/manifest index f025974fe5..a78a33a4bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\smultiplex3.test\sto\saccount\sfor\sthe\sfact\sthat\sthe\smultiplexor\sxDelete\smethod\smay\sreturn\san\serror\seven\sif\sit\smanages\sto\sdelete\sthe\sfirst\schunk\sof\sa\sfile. -D 2022-11-24T15:32:00.589 +C JS\sdocumentation\scleanups.\sNo\scode\schanges. +D 2022-11-24T16:03:49.104 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,7 @@ F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js dec6c14994317ad0011714890426cdc211f4eab451c9766ea88c7ac4f535287e F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b -F ext/wasm/api/sqlite3-api-prologue.js 58caf45e4cc0deec9bdddb051ce17408686f2bc21ea7b3ee0b4c4deaba532605 +F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f7eaa6ba2147bfd6dbdc2444d0f919d846aa7f9b68cccab17ef585ffdacf3d60 -R f39101951334943bf67c026388719509 -U dan -Z db7c150da52ab80c381cc3a21444c8d8 +P 1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 +R abd9b2d5004cf0e1068a5fc8037a67b4 +U stephan +Z 59d8b0c4ff23bb85893a1a0d379c6144 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 92bbfc67cc..7fa3372b8a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 \ No newline at end of file +18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 \ No newline at end of file From df5d06d03eca407aa84f12fce477a0c210bc4375 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 24 Nov 2022 17:53:09 +0000 Subject: [PATCH 073/282] More work on the OPFS concurrency testing app. FossilOrigin-Name: c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 --- ext/wasm/api/sqlite3-api-opfs.js | 3 +++ ext/wasm/api/sqlite3-opfs-async-proxy.js | 28 +++++----------------- ext/wasm/tests/opfs/concurrency/index.html | 11 +++++---- ext/wasm/tests/opfs/concurrency/test.js | 28 ++++++++++++++++++---- ext/wasm/tests/opfs/concurrency/worker.js | 8 +++---- manifest | 20 ++++++++-------- manifest.uuid | 2 +- 7 files changed, 54 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index deb8bf04e8..5537df89fb 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -1171,6 +1171,9 @@ const installOpfsVfs = function callee(options){ //TODO to support fiddle and worker1 db upload: //opfsUtil.createFile = function(absName, content=undefined){...} + //We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this + //purpose but its interface and name are still under + //consideration. if(sqlite3.oo1){ opfsUtil.OpfsDb = function(...args){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index b14494a0c9..8a3db9c648 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -558,33 +558,17 @@ const vfsAsyncImpls = { (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite - does but it leads to horrible contention on journal files. */ + does but it leads to immediate contention on journal files. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ /* sqlite does not lock these files, so go ahead and grab an OPFS lock. - Regarding "immutable": that flag is not _really_ applicable - here. It's intended for use on read-only media. If, - however, a file is opened with that flag but changes later - (which can happen if we _don't_ grab a sync handle here) - then sqlite may misbehave. - - Regarding "nolock": ironically, the nolock flag forces us - to lock the file up front. "nolock" tells sqlite to _not_ - use its locking API, but OPFS requires a lock to perform - most of the operations performed in this file. If we don't - grab that lock up front, another handle could end up grabbing - it and mutating the database out from under our nolocked'd - handle. In the interest of preventing corruption, at the cost - of decreased concurrency, we have to lock it for the duration - of this file handle. - https://www.sqlite.org/uri.html */ - fh.xLock = "atOpen"/* Truthy value to keep entry from getting - flagged as auto-locked. String value so - that we can easily distinguish is later - if needed. */; + fh.xLock = "xOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; await getSyncHandle(fh,'xOpen'); } __openFiles[fid] = fh; @@ -824,7 +808,7 @@ const waitLoop = async function f(){ to do other things. If this is too high (e.g. 500ms) then even two workers/tabs can easily run into locking errors. */ - const waitTime = 150; + const waitTime = 100; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index a082dfe997..e19f6a8da6 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -24,10 +24,13 @@

    URL flags: pass a number of workers using - the workers=N URL flag and the worker work interval - as interval=N (milliseconds). Enable OPFS VFS - verbosity with verbose=1-3 (output goes to the - dev console). + the workers=N URL flag. Set the time between each + workload with interval=N (milliseconds). Set the + number of worker iterations with iterations=N. + Enable OPFS VFS verbosity with verbose=1-3 (output + goes to the dev console). Enable/disable "unlock ASAP" mode + (higher concurrency, lower speed) + with unlock-asap=0-1.

    Achtung: if it does not start to do anything within a couple of seconds, check the dev console: Chrome often fails with "cannot allocate diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index cb9d4275be..044d343745 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -56,13 +56,16 @@ options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); options.workerCount = ( urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 - ) || 3; + ) || 4; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 ) || 1; options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 - ) || 750; + ) || 1000; + options.iterations = ( + urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10 + ) || 10; options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; @@ -70,15 +73,25 @@ workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); }; - workers.loadedCount = 0; + workers.counts = {loaded: 0, passed: 0, failed: 0}; + const checkFinished = function(){ + if(workers.counts.passed + workers.counts.failed !== workers.length){ + return; + } + if(workers.counts.failed>0){ + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + }else{ + logCss('tests-pass',"All",workers.length,"workers finished."); + } + }; workers.onmessage = function(msg){ msg = msg.data; const prefix = 'Worker #'+msg.worker+':'; switch(msg.type){ case 'loaded': stdout(prefix,"loaded"); - if(++workers.loadedCount === workers.length){ - stdout("All workers loaded. Telling them to run..."); + if(++workers.counts.loaded === workers.length){ + stdout("All",workers.length,"workers loaded. Telling them to run..."); workers.post('run'); } break; @@ -86,10 +99,14 @@ case 'stderr': stderr(prefix,...msg.payload); break; case 'error': stderr(prefix,"ERROR:",...msg.payload); break; case 'finished': + ++workers.counts.passed; logCss('tests-pass',prefix,...msg.payload); + checkFinished(); break; case 'failed': + ++workers.counts.failed; logCss('tests-fail',prefix,"FAILED:",...msg.payload); + checkFinished(); break; default: logCss('error',"Unhandled message type:",msg); break; } @@ -100,6 +117,7 @@ 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + '&interval='+options.interval + + '&iterations='+options.iterations + '&opfs-verbose='+options.opfsVerbose + '&opfs-unlock-asap='+options.unlockAsap ); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 91aa0fa6fa..95fefa1954 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -46,10 +46,9 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.oo1.DB({ + db = new sqlite3.opfs.OpfsDb({ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c', - vfs: 'opfs' + flags: 'c' }); sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ @@ -59,7 +58,8 @@ self.sqlite3InitModule().then(async function(sqlite3){ ]); }); - const maxIterations = 10; + const maxIterations = + urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); diff --git a/manifest b/manifest index a6b0539bab..602f6943ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sconcurrency\stest:\sadd\sa\sURL\sflag\sto\senable/disable\sunlock-asap\smode. -D 2022-11-23T20:49:08.427 +C More\swork\son\sthe\sOPFS\sconcurrency\stesting\sapp. +D 2022-11-24T17:53:09.937 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 -F ext/wasm/api/sqlite3-api-opfs.js 23b5c51d7c48134eb5a2d23c1dca06315c738aa365f1c9620e649805c62e5781 +F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18dd87fab2ce05284a10d7 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 20030993ccc04e42b4c7111db31d8f22ca02a00879f183a9067738d7bc9f10b9 +F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -552,9 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d -F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js 9a937068b66a0cfbb9cb6833cb001ce22f9d0f8f765775e3456860b05db21797 -F ext/wasm/tests/opfs/concurrency/worker.js 2a275c4983016365cac18c9105f1ac7bd2adbc7ad89cc91b363d722f2bb55cb5 +F ext/wasm/tests/opfs/concurrency/index.html e8fec75ea6eddc600c8a382da7ea2579feece2263a2fb4417f2cf3e9d451744c +F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a +F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 -R ff8c3df82102d217683515f7779a95c4 +P 1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 +R b591bd74ceba5bd36575b86f818be346 U stephan -Z 269769e48cdf286c8fb19455cdae3a52 +Z f64db814b4972f9f3353d092900f9ddf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bb0d09842b..11f565569e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 \ No newline at end of file +c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 \ No newline at end of file From eb84c81c53b5b53f8ac259705a0702c57629e8bc Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Nov 2022 17:58:55 +0000 Subject: [PATCH 074/282] Fix a test case in fts3expr4.test to account for different locales. FossilOrigin-Name: a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts3expr4.test | 11 ++++++++++- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a78a33a4bb..b3aa3b73da 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C JS\sdocumentation\scleanups.\sNo\scode\schanges. -D 2022-11-24T16:03:49.104 +C Fix\sa\stest\scase\sin\sfts3expr4.test\sto\saccount\sfor\sdifferent\slocales. +D 2022-11-24T17:58:55.503 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1094,7 +1094,7 @@ F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test ebae205a7a89446c32583bcd492dcb817b9f6b31819bb4dde2583bb99c77e526 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8 -F test/fts3expr4.test f5b2832549f01b1f7f73389fa21d4b875499bc95bf7c8b36271844888c6a0938 +F test/fts3expr4.test 6c7675bbdbffe6ffc95e9e861500b8ac3f739c4d004ffda812f138eeb1b45529 F test/fts3expr5.test a5b9a053becbdb8e973fbf4d6d3abaabeb42d511d1848bd57931f3e0a1cf983e F test/fts3f.test 8c438d5e1cab526b0021988fb1dc70cf3597b006a33ffd6c955ee89929077fe3 F test/fts3fault.test f4e1342acfe6d216a001490e8cd52afac1f9ffe4a11bbcdcb296129a45c5df45 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a7f3254735054ed8ca32d5ec7c8cde9195a64702638bdc50392007e396fead2 -R abd9b2d5004cf0e1068a5fc8037a67b4 -U stephan -Z 59d8b0c4ff23bb85893a1a0d379c6144 +P 18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 +R e3089d3062e617db155d3fe3a44f9c32 +U dan +Z c38105809dd01fc34e6f6bf40716c164 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7fa3372b8a..bea5150fe5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 \ No newline at end of file +a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 \ No newline at end of file diff --git a/test/fts3expr4.test b/test/fts3expr4.test index 4a6bd293c4..b9227aef55 100644 --- a/test/fts3expr4.test +++ b/test/fts3expr4.test @@ -50,7 +50,16 @@ do_icu_expr_test 1.6 { "(x OR y)" } {PHRASE 3 0 ( x or y )} # is passed to the tokenizer. # do_icu_expr_test 1.7 {a:word} {PHRASE 0 0 word} -do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word} +# do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word} +do_test 1.8 { + set res [ + db one {SELECT fts3_exprtest('icu en_US', 'd:word', 'a', 'b', 'c')} + ] + expr { + $res=="PHRASE 3 0 d:word" || + $res=="AND {AND {PHRASE 3 0 d} {PHRASE 3 0 :}} {PHRASE 3 0 word}" + } +} 1 set sqlite_fts3_enable_parentheses 0 From 594357f2e100fb0b504a9fc1625d8f739c75b1d4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 18:45:56 +0000 Subject: [PATCH 075/282] Change a NEVER() into an assert(). FossilOrigin-Name: a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c36f686e3e..e69a4d81dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sNEVER()\sand\sALWAYS()\smacros\son\sbranches\sthat\sare\sbelieved\sto\sbe\nunreachable. -D 2022-11-24T15:04:23.564 +C Change\sa\sNEVER()\sinto\san\sassert(). +D 2022-11-24T18:45:56.655 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 07d4a0f36cd0fd52862ed25dffd4a746c4e12b6c9ceda3f87772ae373fb436a5 +F src/expr.c 4de4d488850fdf28ec2237401594af36713bec2fa739f6eda87e29ae8515fbb3 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef6ebe7922f56c1584a005deedc85ca1070b4fe5082ada8bbf8d06df54f1c9ef -R eb963117dcb4b03d7c27aa3277dea1bb +P 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 +R fd9fef17e074036cfa72866de0a2e639 U drh -Z 65f0cab32e4b61b60d1aa8b06e205831 +Z bd1dc748d429f9c2f3da3ec49c4b6323 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 17ef3a10aa..91406edc13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 \ No newline at end of file +a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index dc918e0bd6..f5587395a7 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6330,15 +6330,12 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ default: { IndexedExpr *pIEpr; Expr tmp; + assert( pParse->iSelfTab==0 ); if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; if( pParse->pIdxEpr==0 ) break; for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ int iDataCur = pIEpr->iDataCur; if( iDataCur<0 ) continue; - if( NEVER(pParse->iSelfTab) ){ - if( pIEpr->iDataCur!=pParse->iSelfTab-1 ) continue; - iDataCur = -1; - } if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; } if( pIEpr==0 ) break; From e644f7c24bd97b24a0c1be9a82b11c13a316fafd Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 19:02:40 +0000 Subject: [PATCH 076/282] Take care not to try to add comments to a TK_AGG_COLUMN opcode that does not have an associated Table object because it is a reference to an indexed expression. FossilOrigin-Name: 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e69a4d81dd..9d1dae173c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\sNEVER()\sinto\san\sassert(). -D 2022-11-24T18:45:56.655 +C Take\scare\snot\sto\stry\sto\sadd\scomments\sto\sa\sTK_AGG_COLUMN\sopcode\sthat\sdoes\nnot\shave\san\sassociated\sTable\sobject\sbecause\sit\sis\sa\sreference\sto\san\sindexed\nexpression. +D 2022-11-24T19:02:40.883 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 4de4d488850fdf28ec2237401594af36713bec2fa739f6eda87e29ae8515fbb3 +F src/expr.c af9e80644a53ba437851659ba753e8e049e32c40cd584eece3796357fab52e9e F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a901e88c87fc76c7fe42e47a976a5706830f0dbd6027605663e4d55f4f33590 -R fd9fef17e074036cfa72866de0a2e639 +P a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f +R 54c8f77b091e9c38ba79b231dfbd47a4 U drh -Z bd1dc748d429f9c2f3da3ec49c4b6323 +Z df0c8e1baf68d2353d611c7c44676419 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 91406edc13..979e5c34b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f \ No newline at end of file +5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f5587395a7..d7f8cfa917 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4135,9 +4135,11 @@ expr_code_doover: Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - if( pCol->iColumn<0 ){ + if( pTab==0 ){ + /* No comment added */ + }else if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( pTab!=0 ){ + }else{ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ From 8683c0928111707903654278e6c87634cbee5d8c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 24 Nov 2022 23:35:27 +0000 Subject: [PATCH 077/282] Correctly deal with IF-NULL-ROW operators when dealing with indexed expressions in aggregates. FossilOrigin-Name: 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9d1dae173c..b0b6894258 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\scare\snot\sto\stry\sto\sadd\scomments\sto\sa\sTK_AGG_COLUMN\sopcode\sthat\sdoes\nnot\shave\san\sassociated\sTable\sobject\sbecause\sit\sis\sa\sreference\sto\san\sindexed\nexpression. -D 2022-11-24T19:02:40.883 +C Correctly\sdeal\swith\sIF-NULL-ROW\soperators\swhen\sdealing\swith\nindexed\sexpressions\sin\saggregates. +D 2022-11-24T23:35:27.975 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6df5dd2a0434f0921f82cd23449e75138f05435025aed37ad96f344a7a31dcdb +F src/select.c c1cf437e18022df196bfe6e1f34a5fc1e871706665a0588863df5082d2e26563 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a0fd44f4e5a0ec83465203a0009f307ca50223833575895a3ebf8a289515714f -R 54c8f77b091e9c38ba79b231dfbd47a4 +P 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b +R 9699b80135f783c01dc948dbd9568fdf U drh -Z df0c8e1baf68d2353d611c7c44676419 +Z b4118428f98b75714904c3dd7e4b5c40 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 979e5c34b9..5c7c74dae1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b \ No newline at end of file +939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 \ No newline at end of file diff --git a/src/select.c b/src/select.c index e9b6296713..2e6ad45614 100644 --- a/src/select.c +++ b/src/select.c @@ -6318,6 +6318,7 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ if( pExpr->pAggInfo==0 ) return WRC_Continue; if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; pAggInfo = pExpr->pAggInfo; assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); pCol = &pAggInfo->aCol[pExpr->iAgg]; From 7960da03466d96209f53defe088fc16916ba26fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:08:20 +0000 Subject: [PATCH 078/282] Improved comments. Add assert()s to verify that the AggInfo structure is unchanged after registers have been assigned. FossilOrigin-Name: 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 2 ++ src/select.c | 16 ++++++++++++++++ src/sqliteInt.h | 11 ++++++++--- 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index b0b6894258..77a2dfc4d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\sdeal\swith\sIF-NULL-ROW\soperators\swhen\sdealing\swith\nindexed\sexpressions\sin\saggregates. -D 2022-11-24T23:35:27.975 +C Improved\scomments.\s\sAdd\sassert()s\sto\sverify\sthat\sthe\sAggInfo\sstructure\nis\sunchanged\safter\sregisters\shave\sbeen\sassigned. +D 2022-11-25T13:08:20.992 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c af9e80644a53ba437851659ba753e8e049e32c40cd584eece3796357fab52e9e +F src/expr.c 9e7fadc664b938c18f006be0d4f6669888f9302756ee204420c7eccaed5435a6 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c c1cf437e18022df196bfe6e1f34a5fc1e871706665a0588863df5082d2e26563 +F src/select.c 6168029423615a01de361024c56261f2f39d85adcd6da06137010f7ceea759b4 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h f4917d663170308601d794cdff7b8cf9704c4bafe03d7e926a156be835e29f29 +F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5fc23863e4a6388884ce061cd69546757b4c39d731bbb58a8121b92fd5144f7b -R 9699b80135f783c01dc948dbd9568fdf +P 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 +R 85b4d1494a033b09f8257ada7ad96c3c U drh -Z b4118428f98b75714904c3dd7e4b5c40 +Z f7018e8b2be8d601c79b6c13f2f54a26 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c7c74dae1..07bc523f18 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 \ No newline at end of file +5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index d7f8cfa917..d30ae97668 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6265,6 +6265,7 @@ static void findOrCreateAggInfoColumn( struct AggInfo_col *pCol; int k; + assert( pAggInfo->iFirstReg==0 ); pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ if( pCol->iTable==pExpr->iTable @@ -6328,6 +6329,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ AggInfo *pAggInfo = pNC->uNC.pAggInfo; assert( pNC->ncFlags & NC_UAggInfo ); + assert( pAggInfo->iFirstReg==0 ); switch( pExpr->op ){ default: { IndexedExpr *pIEpr; diff --git a/src/select.c b/src/select.c index 2e6ad45614..e76198772d 100644 --- a/src/select.c +++ b/src/select.c @@ -6253,6 +6253,7 @@ static void analyzeAggFuncArgs( ){ int i; assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); pNC->ncFlags |= NC_InAggFunc; for(i=0; inFunc; i++){ Expr *pExpr = pAggInfo->aFunc[i].pFExpr; @@ -6281,6 +6282,7 @@ static void optimizeAggregateUseOfIndexedExpr( AggInfo *pAggInfo, /* The aggregate info */ NameContext *pNC /* Name context used to resolve agg-func args */ ){ + assert( pAggInfo->iFirstReg==0 ); pAggInfo->nColumn = pAggInfo->nAccumulator; if( ALWAYS(pAggInfo->nSortingColumn>0) ){ if( pAggInfo->nColumn==0 ){ @@ -6348,6 +6350,18 @@ static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ ** Allocate a block of registers so that there is one register for each ** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first ** register in this block is stored in pAggInfo->iFirstReg. +** +** This routine may only be called once for each AggInfo object. Prior +** to calling this routine: +** +** * The aCol[] and aFunc[] arrays may be modified +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used +** +** After clling this routine: +** +** * The aCol[] and aFunc[] arrays are fixed +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used +** */ static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ assert( pAggInfo!=0 ); @@ -6369,6 +6383,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int i; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + assert( pAggInfo->iFirstReg>0 ); assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; @@ -6436,6 +6451,7 @@ static void updateAccumulator( struct AggInfo_func *pF; struct AggInfo_col *pC; + assert( pAggInfo->iFirstReg>0 ); pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ecdb35fee..db65082bbc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2766,10 +2766,15 @@ struct AggInfo { }; /* -** Macros to compute aCol[] and aFunc[] register numbers: +** Macros to compute aCol[] and aFunc[] register numbers. +** +** These macros should not be used prior to the call to +** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. +** The assert()s that are part of this macro verify that constraint. */ -#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) -#define AggInfoFuncReg(A,I) ((A)->iFirstReg+(A)->nColumn+(I)) +#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. From f5d0656f167b4b9b7730e0af0caefcb7893b4432 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:15:48 +0000 Subject: [PATCH 079/282] Improve the robustness of the updateAccumulator() routine against OOM. FossilOrigin-Name: d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 77a2dfc4d6..d326feb02b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments.\s\sAdd\sassert()s\sto\sverify\sthat\sthe\sAggInfo\sstructure\nis\sunchanged\safter\sregisters\shave\sbeen\sassigned. -D 2022-11-25T13:08:20.992 +C Improve\sthe\srobustness\sof\sthe\supdateAccumulator()\sroutine\sagainst\sOOM. +D 2022-11-25T13:15:48.547 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6168029423615a01de361024c56261f2f39d85adcd6da06137010f7ceea759b4 +F src/select.c 40a7d7b49b087789cdbf000e7b9328d3bbd29ee4ea585fad75e8b29fa567497c F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 939cb47025354e2df047de7654c0b06f791957cfe4e904abe8892207cea90215 -R 85b4d1494a033b09f8257ada7ad96c3c +P 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 +R d91874ddf96958a005f34b5d280b17b3 U drh -Z f7018e8b2be8d601c79b6c13f2f54a26 +Z b1915dcd60d8f6304cf98a15b81e844e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 07bc523f18..7938fd8387 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 \ No newline at end of file +d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 \ No newline at end of file diff --git a/src/select.c b/src/select.c index e76198772d..215acd6a01 100644 --- a/src/select.c +++ b/src/select.c @@ -6430,8 +6430,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ /* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. +** Generate code that will update the accumulator memory cells for an +** aggregate based on the current cursor position. ** ** If regAcc is non-zero and there are no min() or max() aggregates ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator @@ -6452,6 +6452,7 @@ static void updateAccumulator( struct AggInfo_col *pC; assert( pAggInfo->iFirstReg>0 ); + if( pParse->nErr ) return; pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; From bedbb1bfabc0ba78206aa7897f874d168aa0af13 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 13:26:51 +0000 Subject: [PATCH 080/282] Remove a redundant assert() statement. FossilOrigin-Name: e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d326feb02b..728b14b688 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\srobustness\sof\sthe\supdateAccumulator()\sroutine\sagainst\sOOM. -D 2022-11-25T13:15:48.547 +C Remove\sa\sredundant\sassert()\sstatement. +D 2022-11-25T13:26:51.747 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 40a7d7b49b087789cdbf000e7b9328d3bbd29ee4ea585fad75e8b29fa567497c +F src/select.c f29c90b07067263139fa08f266860ba0877f7a1398775572c3996d3b67c2fe6e F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5200b84195ee1ccaa387f7032eae3d463724c48cb53ba0251bbc79e927dd9752 -R d91874ddf96958a005f34b5d280b17b3 +P d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 +R b674915df2317c83a8051b3fae0cee5f U drh -Z b1915dcd60d8f6304cf98a15b81e844e +Z 8a9f5f62db508168a49cc0d82c6f6183 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7938fd8387..d164374f87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2844f577b69fdc16a0a2568c0958fc3d8aff33e9a0ef80e0f58e92f01097432 \ No newline at end of file +e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 215acd6a01..187beac7fb 100644 --- a/src/select.c +++ b/src/select.c @@ -6387,7 +6387,6 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; - assert( pAggInfo->iFirstReg>0 ); if( pParse->nErr ) return; sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, pAggInfo->iFirstReg+nReg-1); From 23f61a4ba80695ee6e9d88e56294d0dae24d5a41 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 15:52:00 +0000 Subject: [PATCH 081/282] Add restriction (9) to the push-down optimization: If the subquery is a compound then all arms of the compound must have the same affinity. dbsqlfuzz 3a548de406a50e896c1bf7142692d35d339d697f. FossilOrigin-Name: 1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f --- manifest | 16 +++++++------- manifest.uuid | 2 +- src/select.c | 53 ++++++++++++++++++++++++++++++++++------------ test/pushdown.test | 27 ++++++++++++++++++++++- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index b3aa3b73da..64d80c2f59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sin\sfts3expr4.test\sto\saccount\sfor\sdifferent\slocales. -D 2022-11-24T17:58:55.503 +C Add\srestriction\s(9)\sto\sthe\spush-down\soptimization:\s\sIf\sthe\ssubquery\sis\na\scompound\sthen\sall\sarms\sof\sthe\scompound\smust\shave\sthe\ssame\saffinity.\ndbsqlfuzz\s3a548de406a50e896c1bf7142692d35d339d697f. +D 2022-11-25T15:52:00.241 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 4c48373abb4e67129c36bc15d1f5a99a0dfd9534afeb539a2169a09ae91ccec9 +F src/select.c bafe6424e942aad558b2d4be8dbcf5e35ce427ce3cf66d7f0c0ac37c366b00c6 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1403,7 +1403,7 @@ F test/printf.test 390d0d7fcffc3c4ea3c1bb4cbb267444e32b33b048ae21895f23a291844fe F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc +F test/pushdown.test c69f0970ea17e0afc674b89741f60c172cb6f761d81665fc71015f674f0f66ba F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18e89a436daa18a8c972caf06b298da43c97a6ea3e2e5229dccb6920c27bfdb9 -R e3089d3062e617db155d3fe3a44f9c32 -U dan -Z c38105809dd01fc34e6f6bf40716c164 +P a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 +R 9c0acbc69bb427be679e111e4ec839ec +U drh +Z ae1fb8eb841aaf444344209ddd2c7ea7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bea5150fe5..4e3e4aa9d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 \ No newline at end of file +1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f \ No newline at end of file diff --git a/src/select.c b/src/select.c index 4d5bde9a31..65a0a06c33 100644 --- a/src/select.c +++ b/src/select.c @@ -4050,6 +4050,34 @@ static ExprList *findLeftmostExprlist(Select *pSel){ return pSel->pEList; } +/* +** Return true if any of the result-set columns in the compound query +** have incompatible affinities on one or more arms of the compound. +*/ +static int compoundHasDifferentAffinities(Select *p){ + int ii; + ExprList *pList; + assert( p!=0 ); + assert( p->pEList!=0 ); + assert( p->pPrior!=0 ); + pList = p->pEList; + for(ii=0; iinExpr; ii++){ + char aff; + Select *pSub1; + assert( pList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pList->a[ii].pExpr); + for(pSub1=p->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 1; + } + } + } + return 0; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -4153,7 +4181,8 @@ static ExprList *findLeftmostExprlist(Select *pSel){ ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) ** (17h) The corresponding result set expressions in all arms of the -** compound must have the same affinity. +** compound must have the same affinity. (See restriction (9) +** on the push-down optimization.) ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -4372,19 +4401,7 @@ static int flattenSubquery( if( (p->selFlags & SF_Recursive) ) return 0; /* Restriction (17h) */ - for(ii=0; iipEList->nExpr; ii++){ - char aff; - assert( pSub->pEList->a[ii].pExpr!=0 ); - aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr); - for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){ - assert( pSub1->pEList!=0 ); - assert( pSub1->pEList->nExpr>ii ); - assert( pSub1->pEList->a[ii].pExpr!=0 ); - if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ - return 0; - } - } - } + if( compoundHasDifferentAffinities(pSub) ) return 0; if( pSrc->nSrc>1 ){ if( pParse->nSelect>500 ) return 0; @@ -5036,6 +5053,11 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** But it is a lot of work to check that case for an obscure and ** minor optimization, so we omit it for now.) ** +** (9) If the subquery is a compound, then all arms of the compound must +** have the same affinity. (This is the same as restriction (17h) +** for query flattening.) +** +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -5061,6 +5083,9 @@ static int pushDownWhereTerms( if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ if( pSel->pWin ) return 0; /* restriction (6b) */ } + if( compoundHasDifferentAffinities(pSubq) ){ + return 0; /* restriction (9) */ + } }else{ if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; } diff --git a/test/pushdown.test b/test/pushdown.test index af5162495b..7c9b107841 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -86,6 +86,31 @@ do_test 2.2 { set L } {three} - +# 2022-11-25 dbsqlfuzz crash-3a548de406a50e896c1bf7142692d35d339d697f +# Disable the push-down optimization for compound subqueries if any +# arm of the compound has an incompatible affinity. +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t0(c0 INT); + INSERT INTO t0 VALUES(0); + CREATE TABLE t1_a(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1_a VALUES(1,'one'); --,(4,'four'); + CREATE TABLE t1_b(c INTEGER PRIMARY KEY, d TEXT); + INSERT INTO t1_b VALUES(2,'two'); --,(5,'five'); + CREATE VIEW v0 AS SELECT CAST(t0.c0 AS INTEGER) AS c0 FROM t0; + CREATE VIEW t1 AS SELECT a, b FROM t1_a UNION ALL SELECT c, 0 FROM t1_b; + SELECT t1.a, quote(t1.b), t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,t1; +} { + 1 'one' 0 + 2 '0' 0 +} +do_execsql_test 3.2 { + SELECT a, quote(b), cd FROM ( + SELECT t1.a, t1.b, t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,t1 + ) WHERE a=2 AND b='0' AND cd=0; +} { + 2 '0' 0 +} finish_test From 49d400b0912fe99f5d6c0bc9e361f132c4fa338e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 16:32:59 +0000 Subject: [PATCH 082/282] Fix #ifdefs so that restrictions (8) and (9) of the push-down optimization are still enforced even if compiled with SQLITE_OMIT_WINDOWFUNC. This fixes a bug introduced by check-in [346a3b12b861ce7b]. FossilOrigin-Name: 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/select.c | 6 ++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 76e28676a0..8852b478ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Immprove\sthe\squery\splanner\ssuch\sthat\sit\sis\sable\sto\smake\suse\sof\sindexed\nexpressions\swithin\san\saggregate\squery\swith\sGROUP\sBY.\s\sThis\simplements\nenhancement\srequest\s[99378177930f87bd]. -D 2022-11-25T16:10:48.068 +C Fix\s#ifdefs\sso\sthat\srestrictions\s(8)\sand\s(9)\sof\sthe\spush-down\soptimization\nare\sstill\senforced\seven\sif\scompiled\swith\sSQLITE_OMIT_WINDOWFUNC.\s\sThis\nfixes\sa\sbug\sintroduced\sby\scheck-in\s[346a3b12b861ce7b]. +D 2022-11-25T16:32:59.610 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7dcab394efe24c6016c6ed7958d426bfcdef57f50c2520dcccda27d9df28fdb6 +F src/select.c e0a04b10826637f826e0f9edb9c6cebdc0031f4093cec2553030f39e0b751109 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,9 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 -R eb048b7302ca70762f9994708c895609 -T +closed e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 +P b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 +R ba0c31d84a1008c246f9efb185042449 U drh -Z 6fdb43be076439dd6a7f7f9247393df9 +Z 98b0ad17c9260369ac5243663e0dc610 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff7ccc702c..1de8ddd5c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 \ No newline at end of file +09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 067c98b4a3..bc9ae25ce8 100644 --- a/src/select.c +++ b/src/select.c @@ -5073,7 +5073,6 @@ static int pushDownWhereTerms( if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0; -#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pPrior ){ Select *pSel; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ @@ -5081,15 +5080,18 @@ static int pushDownWhereTerms( assert( op==TK_ALL || op==TK_SELECT || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ +#endif } if( compoundHasDifferentAffinities(pSubq) ){ return 0; /* restriction (9) */ } }else{ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; - } #endif + } #ifdef SQLITE_DEBUG /* Only the first term of a compound can have a WITH clause. But make From 902ea8392536b729c4c4553613dae9f65a4c1bf3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Nov 2022 17:05:55 +0000 Subject: [PATCH 083/282] Relax restriction (8) on the push-down optimization so that it only applies if one or more columns uses a collating sequence other than BINARY. See [forum:/forumpost/3824ced748baa808|forum post 3824ced748baa808] and check-in [346a3b12b861ce7b]. FossilOrigin-Name: adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 30 +++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 8852b478ad..3a6ab9396c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\s#ifdefs\sso\sthat\srestrictions\s(8)\sand\s(9)\sof\sthe\spush-down\soptimization\nare\sstill\senforced\seven\sif\scompiled\swith\sSQLITE_OMIT_WINDOWFUNC.\s\sThis\nfixes\sa\sbug\sintroduced\sby\scheck-in\s[346a3b12b861ce7b]. -D 2022-11-25T16:32:59.610 +C Relax\srestriction\s(8)\son\sthe\spush-down\soptimization\sso\sthat\sit\sonly\sapplies\nif\sone\sor\smore\scolumns\suses\sa\scollating\ssequence\sother\sthan\sBINARY.\nSee\s[forum:/forumpost/3824ced748baa808|forum\spost\s3824ced748baa808]\sand\ncheck-in\s[346a3b12b861ce7b]. +D 2022-11-25T17:05:55.692 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c e0a04b10826637f826e0f9edb9c6cebdc0031f4093cec2553030f39e0b751109 +F src/select.c 934c37455762579dcef5ce37f6abf04facaa61ef04cde207ac170b28c4780113 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 -R ba0c31d84a1008c246f9efb185042449 +P 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 +R 051f7bebeb695c93dfdb45030ad5b532 U drh -Z 98b0ad17c9260369ac5243663e0dc610 +Z 0a4a218169578e69e7b7469760b002aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1de8ddd5c0..76e9e654c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 \ No newline at end of file +adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf \ No newline at end of file diff --git a/src/select.c b/src/select.c index bc9ae25ce8..2bedfb9da0 100644 --- a/src/select.c +++ b/src/select.c @@ -5046,12 +5046,9 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** be materialized. (This restriction is implemented in the calling ** routine.) ** -** (8) The subquery may not be a compound that uses UNION, INTERSECT, -** or EXCEPT. (We could, perhaps, relax this restriction to allow -** this case if none of the comparisons operators between left and -** right arms of the compound use a collation other than BINARY. -** But it is a lot of work to check that case for an obscure and -** minor optimization, so we omit it for now.) +** (8) If the subquery is a compound that uses UNION, INTERSECT, +** or EXCEPT, then all of the result set columns for all arms of +** the compound must use the BINARY collating sequence. ** ** (9) If the subquery is a compound, then all arms of the compound must ** have the same affinity. (This is the same as restriction (17h) @@ -5075,11 +5072,14 @@ static int pushDownWhereTerms( if( pSubq->pPrior ){ Select *pSel; + int notUnionAll = 0; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ u8 op = pSel->op; assert( op==TK_ALL || op==TK_SELECT || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); - if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ + if( op!=TK_ALL && op!=TK_SELECT ){ + notUnionAll = 1; + } #ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ #endif @@ -5087,6 +5087,22 @@ static int pushDownWhereTerms( if( compoundHasDifferentAffinities(pSubq) ){ return 0; /* restriction (9) */ } + if( notUnionAll ){ + /* If any of the compound arms are connected using UNION, INTERSECT, + ** or EXCEPT, then we must ensure that none of the columns use a + ** non-BINARY collating sequence. */ + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + int ii; + const ExprList *pList = pSel->pEList; + assert( pList!=0 ); + for(ii=0; iinExpr; ii++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[ii].pExpr); + if( !sqlite3IsBinary(pColl) ){ + return 0; /* Restriction (8) */ + } + } + } + } }else{ #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; From 6e711869042b45d84fd6c8fe8243db3849287137 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 14:19:47 +0000 Subject: [PATCH 084/282] Experimental changes to help the query planner detect when an expression index is coverting. Works somewhat, but there are tests that fail. FossilOrigin-Name: 968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 --- manifest | 15 ++++++----- manifest.uuid | 2 +- src/where.c | 69 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 3a6ab9396c..a56bd219e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Relax\srestriction\s(8)\son\sthe\spush-down\soptimization\sso\sthat\sit\sonly\sapplies\nif\sone\sor\smore\scolumns\suses\sa\scollating\ssequence\sother\sthan\sBINARY.\nSee\s[forum:/forumpost/3824ced748baa808|forum\spost\s3824ced748baa808]\sand\ncheck-in\s[346a3b12b861ce7b]. -D 2022-11-25T17:05:55.692 +C Experimental\schanges\sto\shelp\sthe\squery\splanner\sdetect\swhen\san\sexpression\nindex\sis\scoverting.\s\sWorks\ssomewhat,\sbut\sthere\sare\stests\sthat\sfail. +D 2022-11-26T14:19:47.899 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -728,7 +728,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c ea0f518df9e00aa44013a1d384090b4b3a499ee11d4daa0a7d99c4eb9f7ab4ba +F src/where.c 550388b75a29c2af2c2182e56e2a51bc6b92aef302eb072f177b3526394d6020 F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2060,8 +2060,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 -R 051f7bebeb695c93dfdb45030ad5b532 +P adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf +R b02bf662fd2aa7710f36a7d19197692a +T *branch * covering-indexed-expr +T *sym-covering-indexed-expr * +T -sym-trunk * U drh -Z 0a4a218169578e69e7b7469760b002aa +Z e78db49817bfc2647feb4d11db6b69e2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 76e9e654c1..036305084f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf \ No newline at end of file +968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index ae3d513cae..5f35c1e791 100644 --- a/src/where.c +++ b/src/where.c @@ -3247,6 +3247,26 @@ static int whereUsablePartialIndex( return 0; } +/* +** pIdx is an index containing expressions. Check it see if any of the +** expressions in the index match the pExpr expression. +*/ +static int exprIsCoveredByIndex( + const Expr *pExpr, + const Index *pIdx, + int iTabCur +){ + int i; + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]==XN_EXPR + && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0 + ){ + return 1; + } + } + return 0; +} + /* ** Structure passed to the whereIsCoveringIndex Walker callback. */ @@ -3256,7 +3276,7 @@ struct CoveringIndexCheck { }; /* -** Information passed in is pWalk->u.pCovIdxCk. Call is pCk. +** Information passed in is pWalk->u.pCovIdxCk. Call it pCk. ** ** If the Expr node references the table with cursor pCk->iTabCur, then ** make sure that column is covered by the index pCk->pIdx. We know that @@ -3268,23 +3288,32 @@ struct CoveringIndexCheck { ** ** If this node does not disprove that the index can be a covering index, ** then just return WRC_Continue, to continue the search. +** +** If pCk->pIdx contains indexed expressions and one of those expressions +** matches pExpr, then prune the search. */ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ int i; /* Loop counter */ const Index *pIdx; /* The index of interest */ const i16 *aiColumn; /* Columns contained in the index */ u16 nColumn; /* Number of columns in the index */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue; - if( pExpr->iColumn<(BMS-1) ) return WRC_Continue; - if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; pIdx = pWalk->u.pCovIdxCk->pIdx; - aiColumn = pIdx->aiColumn; - nColumn = pIdx->nColumn; - for(i=0; iiColumn ) return WRC_Continue; + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ + /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ + if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; iiColumn ) return WRC_Continue; + } + pWalk->eCode = 1; + return WRC_Abort; + }else if( pIdx->bHasExpr + && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + return WRC_Prune; } - pWalk->eCode = 1; - return WRC_Abort; + return WRC_Continue; } @@ -3315,14 +3344,16 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex( ** if pIdx is covering. Assume it is not. */ return 1; } - for(i=0; inColumn; i++){ - if( pIdx->aiColumn[i]>=BMS-1 ) break; - } - if( i>=pIdx->nColumn ){ - /* pIdx does not index any columns greater than 62, but we know from - ** colMask that columns greater than 62 are used, so this is not a - ** covering index */ - return 1; + if( pIdx->bHasExpr==0 ){ + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 1; + } } ck.pIdx = pIdx; ck.iTabCur = iTabCur; @@ -3552,7 +3583,7 @@ static int whereLoopAddBtree( m = 0; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT ){ + if( m==TOPBIT || pProbe->bHasExpr ){ m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); } pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; From 743c9ec4a5f3f00a85ad99c582b012c189a0d34f Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:12:02 +0000 Subject: [PATCH 085/282] Fix an incorrect assertion-fault in the "TREETRACE" debugging logic on select.c. The problem does not affect production builds. FossilOrigin-Name: f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f14f6424d5..a2d2f52121 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sVFS:\sadd\sthe\sopfs-unlock-asap=1\sURI\sflag\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP\sinstead\sof\sduring\sVFS\sidle\stime.\sThis\simproves\sconcurrency\snotably\sin\sthe\stest\sapp\sbut\sbrings\sa\ssignificant\sperformance\spenalty\sin\sspeedtest1\s(roughly\s4x\sslowdown).\sThis\sis\snot\sthe\sfinal\sword\sin\sOPFS\sconcurrency,\sbut\sgets\sus\sa\sstep\sfurther. -D 2022-11-26T15:24:58.092 +C Fix\san\sincorrect\sassertion-fault\sin\sthe\s"TREETRACE"\sdebugging\slogic\son\nselect.c.\s\sThe\sproblem\sdoes\snot\saffect\sproduction\sbuilds. +D 2022-11-26T20:12:02.257 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 934c37455762579dcef5ce37f6abf04facaa61ef04cde207ac170b28c4780113 +F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,9 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 -R ef0be46e6ac8ee603582dd679fae7434 -T +closed c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 Closed\sby\sintegrate-merge. -U stephan -Z 3b8060cf3f2afe36aab99fa36bba89db +P 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 +R 581898efb4aefcef8555315298dbd7f7 +U drh +Z f6493a79b992a3696ede08a261d497d4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e97840e0fc..d4e96fca06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 \ No newline at end of file +f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2bedfb9da0..10498310d4 100644 --- a/src/select.c +++ b/src/select.c @@ -6259,7 +6259,7 @@ static void printAggInfo(AggInfo *pAggInfo){ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" " iSorterColumn=%d %s\n", ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), + pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, pCol->iSorterColumn, ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); From 217e77d24008f5bd813c719e4a08da93500a2218 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:48:56 +0000 Subject: [PATCH 086/282] Show which AggInfo.aCol an Expr node represents in the tree trace debugging logic. Does not affect production builds. FossilOrigin-Name: b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/treeview.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a2d2f52121..54ca01a2ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\sassertion-fault\sin\sthe\s"TREETRACE"\sdebugging\slogic\son\nselect.c.\s\sThe\sproblem\sdoes\snot\saffect\sproduction\sbuilds. -D 2022-11-26T20:12:02.257 +C Show\swhich\sAggInfo.aCol\san\sExpr\snode\srepresents\sin\sthe\stree\strace\sdebugging\nlogic.\s\sDoes\snot\saffect\sproduction\sbuilds. +D 2022-11-26T20:48:56.633 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b -F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 +F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 -R 581898efb4aefcef8555315298dbd7f7 +P f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e +R 77aba841340d0be9ea0f0036dfeb9ac7 U drh -Z f6493a79b992a3696ede08a261d497d4 +Z cf507f961f29b58691309417f6fc3d94 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d4e96fca06..4df4cac51e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e \ No newline at end of file +b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 89a128dff6..9df16a12bf 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -487,7 +487,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", @@ -504,6 +504,9 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; From 9d34152bcf72c20ed260d2c124dfde292cf4d5f0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:52:38 +0000 Subject: [PATCH 087/282] Better reporting of when the WHERE clause analysis thinks that an index is covering. FossilOrigin-Name: 17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/treeview.c | 5 ++++- src/where.c | 9 ++++++++- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1e9ca78250..e31ab14280 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sinto\sthe\scovering-indexed-expr\sbranch. -D 2022-11-26T20:13:54.129 +C Better\sreporting\sof\swhen\sthe\sWHERE\sclause\sanalysis\sthinks\sthat\san\sindex\nis\scovering. +D 2022-11-26T20:52:38.117 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b -F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 +F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 @@ -728,7 +728,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 550388b75a29c2af2c2182e56e2a51bc6b92aef302eb072f177b3526394d6020 +F src/where.c 1f64631ab2c94b3c11de00ad1d08be9d51ee73acd613c54c4367526f5e2ac241 F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e -R b11982e41348628501c0b91820bd7cd7 +P 9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e +R 1dffa8ce2c066e6a4b41c4abd6aac427 U drh -Z 5e6dcb1170e4740f335f77683444a169 +Z 2a27a64568a31340c97915e2beb3adb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 25771bfcb7..7f494549c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e \ No newline at end of file +17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 89a128dff6..9df16a12bf 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -487,7 +487,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", @@ -504,6 +504,9 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; diff --git a/src/where.c b/src/where.c index 5f35c1e791..6ab9230d83 100644 --- a/src/where.c +++ b/src/where.c @@ -3583,8 +3583,15 @@ static int whereLoopAddBtree( m = 0; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT || pProbe->bHasExpr ){ + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol) ){ m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + WHERETRACE(0xff, + ("-> %s %s a covering index according to whereIsCoveringIndex()\n", + pProbe->zName, m==0 ? "is" : "is not")); + }else{ + WHERETRACE(0xff, + ("-> %s %s a covering index according to bitmasks\n", + pProbe->zName, m==0 ? "is" : "is not")); } pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } From ca9309cc565da99c6a8e8dea84852b9c4ee478a4 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 27 Nov 2022 00:57:56 +0000 Subject: [PATCH 088/282] Document an OPFS API change in Chrome v108 which does not break our code but does change several formerly async methods to synchronous. No code changes. FossilOrigin-Name: e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 16 ++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 8a3db9c648..8e60969bc6 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -29,6 +29,22 @@ This file represents an implementation detail of a larger piece of code, and not a public interface. Its details may change at any time and are not intended to be used by any client-level code. + + 2022-11-27: Chrome v108 changes some async methods to synchronous, as + documented at: + + https://developer.chrome.com/blog/sync-methods-for-accesshandles/ + + We cannot change to the sync forms at this point without breaking + clients who use Chrome v104-ish or higher. truncate(), getSize(), + flush(), and close() are now (as of v108) synchronous. Calling them + with an "await", as we have to for the async forms, is still legal + with the sync forms but is superfluous. Calling the async forms with + theFunc().then(...) is not compatible with the change to + synchronous, but we do do not use those APIs that way. i.e. we don't + _need_ to change anything for this, but at some point (after Chrome + versions (approximately) 104-107 are extinct) should change our + usage of those methods to remove the "await". */ "use strict"; const toss = function(...args){throw new Error(args.join(' '))}; diff --git a/manifest b/manifest index 54ca01a2ed..37ec0eef0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\swhich\sAggInfo.aCol\san\sExpr\snode\srepresents\sin\sthe\stree\strace\sdebugging\nlogic.\s\sDoes\snot\saffect\sproduction\sbuilds. -D 2022-11-26T20:48:56.633 +C Document\san\sOPFS\sAPI\schange\sin\sChrome\sv108\swhich\sdoes\snot\sbreak\sour\scode\sbut\sdoes\schange\sseveral\sformerly\sasync\smethods\sto\ssynchronous.\sNo\scode\schanges. +D 2022-11-27T00:57:56.426 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -506,7 +506,7 @@ F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18d F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a +F ext/wasm/api/sqlite3-opfs-async-proxy.js 798383f6b46fd5dac122d6e35962d25b10401ddb825b5c66df1d21e6b1d8aacc F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e -R 77aba841340d0be9ea0f0036dfeb9ac7 -U drh -Z cf507f961f29b58691309417f6fc3d94 +P b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 +R 4c0998f1bfd61979c1006462c1fac102 +U stephan +Z 9e17173a8e8b4b20b4b9a5a815e1b3d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4df4cac51e..852ac7c5b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 \ No newline at end of file +e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 \ No newline at end of file From c1ca183abfeb54a5735f4f3bd4c0e1994645c3f7 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 28 Nov 2022 02:28:44 +0000 Subject: [PATCH 089/282] Conform CLI .trace arg handling to its help. FossilOrigin-Name: 31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c.in | 2 +- test/shell4.test | 6 ++++++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 37ec0eef0c..107bf9f15b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Document\san\sOPFS\sAPI\schange\sin\sChrome\sv108\swhich\sdoes\snot\sbreak\sour\scode\sbut\sdoes\schange\sseveral\sformerly\sasync\smethods\sto\ssynchronous.\sNo\scode\schanges. -D 2022-11-27T00:57:56.426 +C Conform\sCLI\s.trace\sarg\shandling\sto\sits\shelp. +D 2022-11-28T02:28:44.889 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 -F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 +F src/shell.c.in 334b1661abb6453e4f5d1778ce9c80dddef83470867f1fba07e5ee400792adbe F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -1497,7 +1497,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test e4b4de56f454708e0747b52915135baa2cbfec4965406d6eaf02a4a5c22a9880 F test/shell2.test c536c2aab4852608f8a606262330797abc4d964a4c2c782a7760f54ea1f17a6a F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a -F test/shell4.test 7dc8a515705bc093d8ffe381670e8fa7a969661e8ed177c35c847e3c6dfc35e2 +F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 -R 4c0998f1bfd61979c1006462c1fac102 -U stephan -Z 9e17173a8e8b4b20b4b9a5a815e1b3d6 +P e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 +R ce11368d09e97cb20c99cf53926be781 +U larrybr +Z 2b24f85901c30dde8df5f7afa27a353b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 852ac7c5b3..2d0d9deb13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 \ No newline at end of file +31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index e1f0ecfcd4..ab47dba8c9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10658,7 +10658,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); + p->traceOut = output_file_open(z, 0); } } if( p->traceOut==0 ){ diff --git a/test/shell4.test b/test/shell4.test index 068072202d..193dc8b69b 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -125,6 +125,12 @@ SELECT * FROM t1;}} do_test shell4-2.5 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;" } {0 {SELECT * FROM t1;}} +do_test shell4-2.6 { + catchcmd ":memory:" { +CREATE TABLE t1(x); +.trace --stmt stdout +SELECT * FROM t1;} +} {0 {SELECT * FROM t1;}} } do_test shell4-3.1 { From abbfa7aa17307f31011751072454850e6da4d435 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Nov 2022 13:47:27 +0000 Subject: [PATCH 090/282] Ensure the RETURNING clause is honoured when a row of a temp table is updated by an ON CONFLICT clause. FossilOrigin-Name: a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/trigger.c | 2 +- test/returning1.test | 24 ++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 107bf9f15b..91afe6c7c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Conform\sCLI\s.trace\sarg\shandling\sto\sits\shelp. -D 2022-11-28T02:28:44.889 +C Ensure\sthe\sRETURNING\sclause\sis\shonoured\swhen\sa\srow\sof\sa\stemp\stable\sis\supdated\sby\san\sON\sCONFLICT\sclause. +D 2022-11-28T13:47:27.194 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -707,7 +707,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 -F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 +F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 @@ -1421,7 +1421,7 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2 F test/releasetest_data.tcl 0db8aee0c348090fd06da47020ab4ed8ec692e0723427b2f3947d4dfb806f3b0 F test/resetdb.test 8062cf10a09d8c048f8de7711e94571c38b38168db0e5877ba7561789e5eeb2b F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test c43b8370a351f77aec6d71f4a2cde59b849369ed1933261a2c2c69e23e34ff5e +F test/returning1.test 1366e04566cfe1a082d17b1e0f195ec64473c79374b3a5d4ae00c43d885dea31 F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 -R ce11368d09e97cb20c99cf53926be781 -U larrybr -Z 2b24f85901c30dde8df5f7afa27a353b +P 31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c +R cab683454f2ed9db2c50c1dc1f9667b9 +U dan +Z f330cc4347042764fbf120efeed28ded # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d0d9deb13..0bd7375f33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c \ No newline at end of file +a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa \ No newline at end of file diff --git a/src/trigger.c b/src/trigger.c index ca64d6145b..02d8540237 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -61,7 +61,7 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ if( pTrig->pTabSchema==pTab->pSchema && pTrig->table && 0==sqlite3StrICmp(pTrig->table, pTab->zName) - && pTrig->pTabSchema!=pTmpSchema + && (pTrig->pTabSchema!=pTmpSchema || pTrig->bReturning) ){ pTrig->pNext = pList; pList = pTrig; diff --git a/test/returning1.test b/test/returning1.test index fb058a6438..92e10ee9f4 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -376,4 +376,28 @@ do_execsql_test 16.1 { SELECT * FROM t2; } {1 2 3 a b c} + +foreach {tn temp} { + 1 "" + 2 TEMP +} { + reset_db + do_execsql_test 17.$tn.0 " + CREATE $temp TABLE foo ( + fooid INTEGER PRIMARY KEY, + fooval INTEGER NOT NULL UNIQUE, + refcnt INTEGER NOT NULL DEFAULT 1 + ); + " + do_execsql_test 17.$tn.1 { + INSERT INTO foo (fooval) VALUES (17), (4711), (17) + ON CONFLICT DO + UPDATE SET refcnt = refcnt+1 + RETURNING fooid; + } { + 1 2 1 + } +} + + finish_test From 84606c973757cbb5a276b54c5bfad67943288722 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 15:23:53 +0000 Subject: [PATCH 091/282] Rework the covering index checking routine, whereIsCoveringIndex(), so that it can return a "maybe" result for aggregate queries where we are not exactly sure. The index is scored as if it is covering, but the main table is still opened. FossilOrigin-Name: b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a --- manifest | 14 +++---- manifest.uuid | 2 +- src/where.c | 103 ++++++++++++++++++++++++++++++++++--------------- src/whereInt.h | 1 + 4 files changed, 81 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index e31ab14280..f0f6a8af9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\sreporting\sof\swhen\sthe\sWHERE\sclause\sanalysis\sthinks\sthat\san\sindex\nis\scovering. -D 2022-11-26T20:52:38.117 +C Rework\sthe\scovering\sindex\schecking\sroutine,\swhereIsCoveringIndex(),\sso\sthat\nit\scan\sreturn\sa\s"maybe"\sresult\sfor\saggregate\squeries\swhere\swe\sare\snot\sexactly\nsure.\s\sThe\sindex\sis\sscored\sas\sif\sit\sis\scovering,\sbut\sthe\smain\stable\sis\nstill\sopened. +D 2022-11-28T15:23:53.775 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -728,8 +728,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 1f64631ab2c94b3c11de00ad1d08be9d51ee73acd613c54c4367526f5e2ac241 -F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f +F src/where.c 81422870d17fdf3880fdb9d3b3965e1aa5efb7a72c6933cec3f1bc8081c463db +F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e -R 1dffa8ce2c066e6a4b41c4abd6aac427 +P 17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 +R 254ab4db9b1e239ec16eb2e1a31dc657 U drh -Z 2a27a64568a31340c97915e2beb3adb7 +Z 489a805b1f3c740affbc27a0febf881c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7f494549c9..d0a052a41d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 \ No newline at end of file +b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6ab9230d83..589300c6ba 100644 --- a/src/where.c +++ b/src/where.c @@ -3094,7 +3094,7 @@ static int whereLoopAddBtreeIndex( assert( pSrc->pTab->szTabRow>0 ); rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); @@ -3270,9 +3270,12 @@ static int exprIsCoveredByIndex( /* ** Structure passed to the whereIsCoveringIndex Walker callback. */ +typedef struct CoveringIndexCheck CoveringIndexCheck; struct CoveringIndexCheck { Index *pIdx; /* The index */ int iTabCur; /* Cursor number for the corresponding table */ + u8 bExpr; /* Uses an indexed expression */ + u8 bUnidx; /* Uses an unindexed column not within an indexed expr */ }; /* @@ -3293,24 +3296,28 @@ struct CoveringIndexCheck { ** matches pExpr, then prune the search. */ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ - int i; /* Loop counter */ - const Index *pIdx; /* The index of interest */ - const i16 *aiColumn; /* Columns contained in the index */ - u16 nColumn; /* Number of columns in the index */ - pIdx = pWalk->u.pCovIdxCk->pIdx; + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + CoveringIndexCheck *pCk; /* Info about this search */ + + pCk = pWalk->u.pCovIdxCk; + pIdx = pCk->pIdx; if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ - if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; + if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue; pIdx = pWalk->u.pCovIdxCk->pIdx; aiColumn = pIdx->aiColumn; nColumn = pIdx->nColumn; for(i=0; iiColumn ) return WRC_Continue; } - pWalk->eCode = 1; + pCk->bUnidx = 1; return WRC_Abort; }else if( pIdx->bHasExpr && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + pCk->bExpr = 1; return WRC_Prune; } return WRC_Continue; @@ -3319,30 +3326,39 @@ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ /* ** pIdx is an index that covers all of the low-number columns used by -** pWInfo->pSelect (columns from 0 through 62). But there are columns -** in pWInfo->pSelect beyond 62. This routine tries to answer the question -** of whether pIdx covers *all* columns in the query. +** pWInfo->pSelect (columns from 0 through 62) or an index that has +** expressions terms. Hence, we cannot determine whether or not it is +** a covering index by using the colUsed bitmasks. We have to do a search +** to see if the index is covering. This routine does that search. ** -** Return 0 if pIdx is a covering index. Return non-zero if pIdx is -** not a covering index or if we are unable to determine if pIdx is a -** covering index. +** The return value is one of these: ** -** This routine is an optimization. It is always safe to return non-zero. -** But returning zero when non-zero should have been returned can lead to -** incorrect bytecode and assertion faults. +** 0 The index is definitely not a covering index +** +** WHERE_IDX_ONLY The index is definitely a covering index +** +** WHERE_EXPRIDX The index is likely a covering index, but it is +** difficult to determine precisely because of the +** expressions that are indexed. Score it as a +** covering index, but still keep the main table open +** just in case we need it. +** +** This routine is an optimization. It is always safe to return zero. +** But returning one of the other two values when zero should have been +** returned can lead to incorrect bytecode and assertion faults. */ static SQLITE_NOINLINE u32 whereIsCoveringIndex( WhereInfo *pWInfo, /* The WHERE clause context */ Index *pIdx, /* Index that is being tested */ int iTabCur /* Cursor for the table being indexed */ ){ - int i; + int i, rc; struct CoveringIndexCheck ck; Walker w; if( pWInfo->pSelect==0 ){ /* We don't have access to the full query, so we cannot check to see ** if pIdx is covering. Assume it is not. */ - return 1; + return 0; } if( pIdx->bHasExpr==0 ){ for(i=0; inColumn; i++){ @@ -3352,18 +3368,26 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex( /* pIdx does not index any columns greater than 62, but we know from ** colMask that columns greater than 62 are used, so this is not a ** covering index */ - return 1; + return 0; } } ck.pIdx = pIdx; ck.iTabCur = iTabCur; + ck.bExpr = 0; + ck.bUnidx = 0; memset(&w, 0, sizeof(w)); w.xExprCallback = whereIsCoveringIndexWalkCallback; w.xSelectCallback = sqlite3SelectWalkNoop; w.u.pCovIdxCk = &ck; - w.eCode = 0; sqlite3WalkSelect(&w, pWInfo->pSelect); - return w.eCode; + if( ck.bUnidx ){ + rc = 0; + }else if( ck.bExpr ){ + rc = WHERE_EXPRIDX; + }else{ + rc = WHERE_IDX_ONLY; + } + return rc; } /* @@ -3579,21 +3603,38 @@ static int whereLoopAddBtree( }else{ Bitmask m; if( pProbe->isCovering ){ - pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol) ){ - m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + pNew->wsFlags = WHERE_INDEXED; + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ + u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + if( isCov==0 ){ + WHERETRACE(0xff, + ("-> %s is not a covering index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + assert( m!=0 ); + }else{ + m = 0; + pNew->wsFlags |= isCov; + if( isCov & WHERE_IDX_ONLY ){ + WHERETRACE(0xff, + ("-> %s is a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + }else{ + assert( isCov==WHERE_EXPRIDX ); + WHERETRACE(0xff, + ("-> %s might be a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + } + } + }else if( m==0 ){ WHERETRACE(0xff, - ("-> %s %s a covering index according to whereIsCoveringIndex()\n", - pProbe->zName, m==0 ? "is" : "is not")); - }else{ - WHERETRACE(0xff, - ("-> %s %s a covering index according to bitmasks\n", + ("-> %s a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; } - pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } /* Full scan via index */ diff --git a/src/whereInt.h b/src/whereInt.h index 28ede5be66..b89a4513e3 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -634,5 +634,6 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ #define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */ +#define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ From cbb958580575c4f4c57dc2019993d44653e04f41 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 18:17:42 +0000 Subject: [PATCH 092/282] Test cases added derived from the enhancment ticket. FossilOrigin-Name: c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46 --- manifest | 12 +++---- manifest.uuid | 2 +- test/indexexpr1.test | 83 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index c83d4173b2..c76ad7a312 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\scovering-index-expr\sbranch. -D 2022-11-28T16:11:10.036 +C Test\scases\sadded\sderived\sfrom\sthe\senhancment\sticket. +D 2022-11-28T18:17:42.151 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1213,7 +1213,7 @@ F test/index7.test b238344318e0b4e42126717f6554f0e7dfd0b39cecad4b736039b43e1e3b6 F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a912a3 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 -F test/indexexpr1.test 3360c2a29a8844e7c4b13293567025281257f9e13a31854cfff6959cede11502 +F test/indexexpr1.test 1cd460113e3999a1477754fd0e3ac0d7bcbbbc742b427b418945f28af708c695 F test/indexexpr2.test 2c7abe3c48f8aaa5a448615ab4d13df3662185d28419c00999670834a3f0b484 F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a b44ab10c49bc2895483a9d40813be3798710ee713cc4bf04e449dce55a68452a -R 5944640ae5b4a3237e1d8a4c46c48db4 +P 89d775ada39aed4dc532374ace02156d07dc06e8ae54a194608af0c5a582d20f +R e73156e3b8e8d088eb2bbd902f358640 U drh -Z 9aa95e661b7edb77eed478b829e7796d +Z b96271db8dd85961a10ccefd57c768e1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e7c719444e..a270d0f31b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89d775ada39aed4dc532374ace02156d07dc06e8ae54a194608af0c5a582d20f \ No newline at end of file +c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46 \ No newline at end of file diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 042132b81d..a5d1514bcd 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -420,7 +420,7 @@ do_execsql_test indexexpr1-1500 { # 2018-01-03 OSSFuzz discovers another test case for the same problem # above. # -do_execsql_test indexexpr-1510 { +do_execsql_test indexexpr1-1510 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(a PRIMARY KEY,b UNIQUE); REPLACE INTO t1 VALUES(2, 1); @@ -434,18 +434,18 @@ do_execsql_test indexexpr-1510 { # a numeric table column, trouble can arise since there are multiple # string that can map to the same numeric value. (Ex: 123, 0123, 000123). # -do_execsql_test indexexpr-1600 { +do_execsql_test indexexpr1-1600 { DROP TABLE IF EXISTS t1; CREATE TABLE t1 (a INTEGER, b); CREATE INDEX idx1 ON t1 (lower(a)); INSERT INTO t1 VALUES('0001234',3); PRAGMA integrity_check; } {ok} -do_execsql_test indexexpr-1610 { +do_execsql_test indexexpr1-1610 { INSERT INTO t1 VALUES('1234',0),('001234',2),('01234',1); SELECT b FROM t1 WHERE lower(a)='1234' ORDER BY +b; } {0 1 2 3} -do_execsql_test indexexpr-1620 { +do_execsql_test indexexpr1-1620 { SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b; } {} @@ -453,7 +453,7 @@ do_execsql_test indexexpr-1620 { # ExprImpliesExpr theorem prover bug: # "(NULL IS FALSE) IS FALSE" does not imply "NULL IS NULL" # -do_execsql_test indexexpr-1700 { +do_execsql_test indexexpr1-1700 { DROP TABLE IF EXISTS t0; CREATE TABLE t0(c0); INSERT INTO t0(c0) VALUES (0); @@ -468,17 +468,17 @@ do_execsql_test indexexpr-1700 { # computing the expression. # ifcapable like_match_blobs { - do_execsql_test indexexpr-1800 { + do_execsql_test indexexpr1-1800 { DROP TABLE IF EXISTS t0; CREATE TABLE t0(c0 REAL, c1 TEXT); CREATE INDEX i0 ON t0(+c0, c0); INSERT INTO t0(c0) VALUES(0); SELECT CAST(+ t0.c0 AS BLOB) LIKE 0 FROM t0; } {0} - do_execsql_test indexexpr-1810 { + do_execsql_test indexexpr1-1810 { SELECT CAST(+ t0.c0 AS BLOB) LIKE '0.0' FROM t0; } {1} - do_execsql_test indexexpr-1820 { + do_execsql_test indexexpr1-1820 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(x REAL); CREATE INDEX t1x ON t1(x, +x); @@ -491,19 +491,80 @@ ifcapable like_match_blobs { # Assertion fault during a DELETE INDEXED BY. # reset_db -do_execsql_test indexexpr-1900 { +do_execsql_test indexexpr1-1900 { CREATE TABLE t1(x TEXT PRIMARY KEY, y TEXT, z INT); INSERT INTO t1(x,y,z) VALUES('alpha','ALPHA',1),('bravo','charlie',1); CREATE INDEX i1 ON t1(+y COLLATE NOCASE); SELECT * FROM t1; } {alpha ALPHA 1 bravo charlie 1} -do_execsql_test indexexpr-1910 { +do_execsql_test indexexpr1-1910 { DELETE FROM t1 INDEXED BY i1 WHERE x IS +y COLLATE NOCASE IN (SELECT z FROM t1) RETURNING *; } {alpha ALPHA 1} -do_execsql_test indexexpr-1920 { +do_execsql_test indexexpr1-1920 { SELECT * FROM t1; } {bravo charlie 1} +# 2022-11-28 Ticket 695a1a53de +# Improved ability to recognize that an index on an expression is a +# covering index. +# +reset_db +do_execsql_test indexexpr1-2000 { + CREATE TABLE t1(a INT, b TEXT); + INSERT INTO t1(a,b) VALUES + (10, '{"one":5,"two":6}'), + (10, '{"one":50,"two":60}'), + (10, '{"three":99}'), + (11, '{"one":100,"two":200}'); + CREATE INDEX t1_one ON t1(a, b->>'one'); + CREATE INDEX t1_two ON t1(a, b->>'two'); +} +do_execsql_test indexexpr1-2010 { + EXPLAIN QUERY PLAN + SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */ +} {/.* t1_one .*/} +do_execsql_test indexexpr1-2011 { + SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */ +} {55} +do_execsql_test indexexpr1-2020 { + EXPLAIN QUERY PLAN + SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */ +} {/.* t1_two .*/} +do_execsql_test indexexpr1-2021 { + SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */ +} {66} +do_execsql_test indexexpr1-2030 { + DROP TABLE t1; + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} +do_execsql_test indexexpr1-2030 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1; +} {1 6 4 54 46} +do_execsql_test indexexpr1-2030 { + explain query plan + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1; +} {/.*SCAN t1 USING INDEX t1x.*/} + + + finish_test From 1b3d13e65e125500d1034e889b37363baa8f1e38 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Nov 2022 18:41:41 +0000 Subject: [PATCH 093/282] Add the SQLITE_FCNTL_RESET_CACHE verb. Use it to ensure that the page cache is purged before and after a the recovery extension is run. FossilOrigin-Name: 6db0bc4bc0d272b610bef2aeeae43f539ed6e7cc0a9cc767d5af85ecb0019d5f --- ext/recover/recover1.test | 41 ++++++++++++++++++++++++++++++++++++ ext/recover/sqlite3recover.c | 6 +++++- manifest | 24 ++++++++++----------- manifest.uuid | 2 +- src/btree.c | 11 ++++++++++ src/btree.h | 2 ++ src/main.c | 3 +++ src/sqlite.h.in | 7 ++++++ 8 files changed, 82 insertions(+), 14 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 75f5dba1ff..3e8a691492 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -273,5 +273,46 @@ do_execsql_test 15.1 { } {} do_recover_test 15 +#------------------------------------------------------------------------- +reset_db +do_execsql_test 16.1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1), (2), (3); +} {wal} +do_test 16.2 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} +do_execsql_test 16.3 { + SELECT * FROM t1; +} {1 2 3} + +do_execsql_test 16.4 { + BEGIN; + SELECT * FROM t1; +} {1 2 3} +do_test 16.5 { + set R [sqlite3_recover_init db main test.db2] + $R run + list [catch { $R finish } msg] $msg +} {1 {cannot start a transaction within a transaction}} +do_execsql_test 16.6 { + SELECT * FROM t1; +} {1 2 3} +do_execsql_test 16.7 { + INSERT INTO t1 VALUES(4); +} +do_test 16.8 { + set R [sqlite3_recover_init db main test.db2] + $R run + list [catch { $R finish } msg] $msg +} {1 {cannot start a transaction within a transaction}} +do_execsql_test 16.9 { + SELECT * FROM t1; + COMMIT; +} {1 2 3 4} + finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 30260f014e..e62aba752f 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -2017,6 +2017,7 @@ static void recoverFinalCleanup(sqlite3_recover *p){ p->pTblList = 0; sqlite3_finalize(p->pGetPage); p->pGetPage = 0; + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); { #ifndef NDEBUG @@ -2315,6 +2316,7 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ ** ** + first freelist page (32-bits at offset 32) ** + size of freelist (32-bits at offset 36) + ** + the wal-mode flags (16-bits at offset 18) ** ** We also try to preserve the auto-vacuum, incr-value, user-version ** and application-id fields - all 32 bit quantities at offsets @@ -2378,7 +2380,8 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ if( p->pPage1Cache ){ p->pPage1Disk = &p->pPage1Cache[nByte]; memcpy(p->pPage1Disk, aBuf, nByte); - + aHdr[18] = a[18]; + aHdr[19] = a[19]; recoverPutU32(&aHdr[28], dbsz); recoverPutU32(&aHdr[56], enc); recoverPutU16(&aHdr[105], pgsz-nReserve); @@ -2574,6 +2577,7 @@ static void recoverStep(sqlite3_recover *p){ recoverOpenOutput(p); /* Open transactions on both the input and output databases. */ + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); recoverExec(p, p->dbIn, "BEGIN"); if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; diff --git a/manifest b/manifest index 477c836f77..a159026919 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s(optional)\sbase64\sand\sbase85\sUDF\sextensions. -D 2022-11-28T14:51:50.439 +C Add\sthe\sSQLITE_FCNTL_RESET_CACHE\sverb.\sUse\sit\sto\sensure\sthat\sthe\spage\scache\sis\spurged\sbefore\sand\safter\sa\sthe\srecovery\sextension\sis\srun. +D 2022-11-28T18:41:41.586 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -390,7 +390,7 @@ F ext/rbu/sqlite3rbu.c c4ba7901b2d3e0c7845f30840e3ffb35c6f999d6da0d80f121866491f F ext/rbu/sqlite3rbu.h 02d981e2d39c151391759e1a400e29c7388730812957ac3db8dad7f6c9f9cfc8 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 F ext/recover/dbdata.c 8f1f75d636431de69d7977ec50fc41bfdd0c48c510d5ee7eae0cbd4164e1429a -F ext/recover/recover1.test 02004eb8f9ec2825ba77e24742c18e45162cb21d27e76a3a435b83a759a1131a +F ext/recover/recover1.test 2a2df2943d6696f9487e75868feae4b1511c4a511b102854ba0d2af0326d9dfb F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49 @@ -402,7 +402,7 @@ F ext/recover/recoverpgsz.test 3658ab8e68475b1bb87d6af88baa04551c84b73280a566a1b F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411 F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c 1e4de9cd30daa67e5c1cd0cafb764ddf183087ce037ab47b7c1a7e2921c90d37 +F ext/recover/sqlite3recover.c 6d37a422c8917b793548a7b2c6e7ff8e1af15ba5453eef37bb69331828b73186 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c 1a34e2d04533d919a30ae4d5caeb1643f6684e9ccd7597ca27721d8af81f4ade F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -583,8 +583,8 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 522df0f1173495e06c5589f0b17a61f53a212017b82be53171133fcfc0e1e90a -F src/btree.h 4fcbb0b041013071dd5e9f53c538d49916c092e6ad8842185985e5270a0792de +F src/btree.c d7d74e6c87a2d947bbf37c705a5ac11b0dc7b3848e76d0ff7c940bb644bba291 +F src/btree.h 49da925329574798be3cbb745a49d069a9e67c99900d8a0d04b1e934d60394ea F src/btreeInt.h 88ad499c92b489afedbfefc3f067c4d15023ec021afe622db240dc9d2277cfa5 F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a @@ -607,7 +607,7 @@ F src/insert.c 90a32bc7faa755cd5292ade21d2b3c6edba8fd1d70754a364caccabfde2c3bb2 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 25663175950c5c4404b9377840b7b4c6fe5c53b415caf43634c62f442c02a9a7 -F src/main.c fa53bb2ae09549dab5629271c3cfd681f89059f5192afaaaf5c0d396bb3957fe +F src/main.c 954490392b74fb215378af3c75a9e1f4f559f19cb1567e5d77f3fbbb63909b4d F src/malloc.c 3d4ec162214024ee071d85711b93bec25cd3371280aee3702b63bcf312ca8238 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -646,7 +646,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c -F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca +F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa b8345630a2a322234bda49ee4b996f6ba20e2b080621e229a2ec5e820892a663 -R b25d12f698cc565856f0ef09be44e05e -U larrybr -Z dca654d0507bf9deef15f6de6a6be007 +P b44ab10c49bc2895483a9d40813be3798710ee713cc4bf04e449dce55a68452a +R 013988a2db147f7833ada18759e80a5f +U dan +Z 0974da01907b06c2d097fc0b34ec52e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d5baba61d..030eff1202 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b44ab10c49bc2895483a9d40813be3798710ee713cc4bf04e449dce55a68452a \ No newline at end of file +6db0bc4bc0d272b610bef2aeeae43f539ed6e7cc0a9cc767d5af85ecb0019d5f \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index cabcf675e3..c949001b89 100644 --- a/src/btree.c +++ b/src/btree.c @@ -11083,6 +11083,17 @@ int sqlite3BtreeIsReadonly(Btree *p){ */ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } +/* +** If no transaction is active and the database is not a temp-db, clear +** the in-memory pager cache. +*/ +void sqlite3BtreeClearCache(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->inTransaction==TRANS_NONE ){ + sqlite3PagerClearCache(pBt->pPager); + } +} + #if !defined(SQLITE_OMIT_SHARED_CACHE) /* ** Return true if the Btree passed as the only argument is sharable. diff --git a/src/btree.h b/src/btree.h index 7f31c6020f..c9b9d80175 100644 --- a/src/btree.h +++ b/src/btree.h @@ -368,6 +368,8 @@ void sqlite3BtreeCursorList(Btree*); int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64); +void sqlite3BtreeClearCache(Btree*); + /* ** If we are not using shared cache, then there is no need to ** use mutexes to access the BtShared structures. So make the diff --git a/src/main.c b/src/main.c index b0645efe61..67dd60ae7e 100644 --- a/src/main.c +++ b/src/main.c @@ -3950,6 +3950,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); } rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESET_CACHE ){ + sqlite3BtreeClearCache(pBtree); + rc = SQLITE_OK; }else{ int nSave = db->busyHandler.nBusy; rc = sqlite3OsFileControl(fd, op, pArg); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c2fc4e5a6a..369d6b6022 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1192,6 +1192,12 @@ struct sqlite3_io_methods { ** **

  • [[SQLITE_FCNTL_CKSM_FILE]] ** Used by the cksmvfs VFS module only. +** +**
  • [[SQLITE_FCNTL_RESET_CACHE]] +** If there is currently no transaction open on the database, and the +** database is not a temp db, then this file-control purges the contents +** of the in-memory page cache. If there is an open transaction, or if +** the db is a temp-db, it is a no-op, not an error. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1234,6 +1240,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CKPT_START 39 #define SQLITE_FCNTL_EXTERNAL_READER 40 #define SQLITE_FCNTL_CKSM_FILE 41 +#define SQLITE_FCNTL_RESET_CACHE 42 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE From 5b046dadfcececc6416535de6a4e2a2dd25e3201 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 20:08:15 +0000 Subject: [PATCH 094/282] Fix harmless compiler warnings. FossilOrigin-Name: 8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 +++--- src/select.c | 9 +++++---- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d8cdd7a1e9..9e7f070df8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implementation\sof\senhancement\srequest\s[695a1a53dea6b240]:\sThe\squery\splanner\nhas\simproved\sawareness\sof\swhen\sindexes\son\sexpressions\sare\scovering\sand\sadjusts\ntheir\scosts\saccordingly. -D 2022-11-28T19:42:48.433 +C Fix\sharmless\scompiler\swarnings. +D 2022-11-28T20:08:15.505 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -583,7 +583,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c d7d74e6c87a2d947bbf37c705a5ac11b0dc7b3848e76d0ff7c940bb644bba291 +F src/btree.c bd4375adcc5e45b3267549ba201a04b2f41fbb0a581d33513fb186a208d4c81d F src/btree.h 49da925329574798be3cbb745a49d069a9e67c99900d8a0d04b1e934d60394ea F src/btreeInt.h 88ad499c92b489afedbfefc3f067c4d15023ec021afe622db240dc9d2277cfa5 F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -644,7 +644,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 +F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6db0bc4bc0d272b610bef2aeeae43f539ed6e7cc0a9cc767d5af85ecb0019d5f c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46 -R cf4309a5723ca7c0ed03679edb5024d5 +P e3474d79b27298e96e7686e5a6f9a8e736b8a6568a0a08fb1abe5bcca038ca63 +R a17c4713fb6c6661faa720f3ee28899c U drh -Z 287040c8f243d0f300388e5a2dad7f24 +Z eb9319d9dfe7dee1c2083bff54e6e44f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e27e9da559..3479de78d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3474d79b27298e96e7686e5a6f9a8e736b8a6568a0a08fb1abe5bcca038ca63 \ No newline at end of file +8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c949001b89..e71531c730 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7127,12 +7127,12 @@ static int insertCell( assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ - int rc = SQLITE_OK; + int rc2 = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pPage, pCell, &rc); - if( rc ) return rc; + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc; } #endif } diff --git a/src/select.c b/src/select.c index 10498310d4..0cd28b543e 100644 --- a/src/select.c +++ b/src/select.c @@ -6290,7 +6290,6 @@ static void printAggInfo(AggInfo *pAggInfo){ ** (or recomputing) those aCol[] entries. */ static void analyzeAggFuncArgs( - Parse *pParse, AggInfo *pAggInfo, NameContext *pNC ){ @@ -6335,7 +6334,7 @@ static void optimizeAggregateUseOfIndexedExpr( pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; } } - analyzeAggFuncArgs(pParse, pAggInfo, pNC); + analyzeAggFuncArgs(pAggInfo, pNC); #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ IndexedExpr *pIEpr; @@ -6350,7 +6349,8 @@ static void optimizeAggregateUseOfIndexedExpr( printAggInfo(pAggInfo); } #else - (void)pSelect; /* Suppress unused-parameter warnings */ + UNUSED_PARAMETER(pSelect); + UNUSED_PARAMETER(pParse); #endif } @@ -6360,6 +6360,7 @@ static void optimizeAggregateUseOfIndexedExpr( static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ AggInfo *pAggInfo; struct AggInfo_col *pCol; + UNUSED_PARAMETER(pWalker); if( pExpr->pAggInfo==0 ) return WRC_Continue; if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; @@ -7611,7 +7612,7 @@ int sqlite3Select( }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - analyzeAggFuncArgs(pParse, pAggInfo, &sNC); + analyzeAggFuncArgs(pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ From 0b0070f1bdf78f7c9ff8315f1aa635592205f9bd Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 28 Nov 2022 21:09:49 +0000 Subject: [PATCH 095/282] Get ext/misc/basexx.c into the testfixture.exe build for MSC. FossilOrigin-Name: bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c --- Makefile.msc | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 191701cdba..0312b0dbaf 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1561,6 +1561,7 @@ TESTEXT = \ $(TOP)\ext\expert\test_expert.c \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\appendvfs.c \ + $(TOP)\ext\misc\basexx.c \ $(TOP)\ext\misc\carray.c \ $(TOP)\ext\misc\cksumvfs.c \ $(TOP)\ext\misc\closure.c \ diff --git a/manifest b/manifest index 9e7f070df8..895ea017fd 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Fix\sharmless\scompiler\swarnings. -D 2022-11-28T20:08:15.505 +C Get\sext/misc/basexx.c\sinto\sthe\stestfixture.exe\sbuild\sfor\sMSC. +D 2022-11-28T21:09:49.816 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in c223963d7b0828f26cb62ea3e0f583d26839b7d3ef0d1cca87f35c4b222ff01b F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc e7a564ceec71f0d9666031d5638cf0d4f88b050b44e8df5d32125137cd259ac0 +F Makefile.msc e86ec21328921721dd6f777da435c62a5469cda1a654f5ecefacf87ddb3dfeb3 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 413ec94920a487ae32c9a2a8819544d690662d6f7c7ce025c0d0b8a1e74fa9db F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e3474d79b27298e96e7686e5a6f9a8e736b8a6568a0a08fb1abe5bcca038ca63 -R a17c4713fb6c6661faa720f3ee28899c -U drh -Z eb9319d9dfe7dee1c2083bff54e6e44f +P 8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 +R 6f236bfe43856f6e65bad104d646e92b +U larrybr +Z a8a43d921cd6c44f25932dfa5b1108ac # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3479de78d9..46da65e1fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 \ No newline at end of file +bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c \ No newline at end of file From d3fc2e63aa45f0a5a64baac15242aa369a74cfb3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 21:17:30 +0000 Subject: [PATCH 096/282] Check-in [8d5b76593d82b3a5] contained an error that was causing some obscure error codes to be lost. Fixed here. FossilOrigin-Name: 46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 895ea017fd..04e653dcec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sext/misc/basexx.c\sinto\sthe\stestfixture.exe\sbuild\sfor\sMSC. -D 2022-11-28T21:09:49.816 +C Check-in\s[8d5b76593d82b3a5]\scontained\san\serror\sthat\swas\scausing\ssome\sobscure\nerror\scodes\sto\sbe\slost.\s\sFixed\shere. +D 2022-11-28T21:17:30.250 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -583,7 +583,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c bd4375adcc5e45b3267549ba201a04b2f41fbb0a581d33513fb186a208d4c81d +F src/btree.c 2f794c217e52fdf4322bf37ee7778331b4d93aed2c00b5d67f914c0239a9edcc F src/btree.h 49da925329574798be3cbb745a49d069a9e67c99900d8a0d04b1e934d60394ea F src/btreeInt.h 88ad499c92b489afedbfefc3f067c4d15023ec021afe622db240dc9d2277cfa5 F src/build.c d3e43e950e4e377c1d451a4862556792acdef1faba14a03f899d30d09731c48b @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d5b76593d82b3a57bc904ced33c24cd49369868b5dd2fe7f2c7b114f5aee2f6 -R 6f236bfe43856f6e65bad104d646e92b -U larrybr -Z a8a43d921cd6c44f25932dfa5b1108ac +P bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c +R edc56f3db34be700dc2fe2c48f4c3e31 +U drh +Z fbd7482d8e22e645fc260e0f000cee41 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 46da65e1fd..2763f271ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c \ No newline at end of file +46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e71531c730..d38b7a551a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7132,7 +7132,7 @@ static int insertCell( ** the entry for the overflow page into the pointer map. */ ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); - if( rc2 ) return rc; + if( rc2 ) return rc2; } #endif } From e6b0154138801768597d572c298af899ffebddeb Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Nov 2022 02:23:12 +0000 Subject: [PATCH 097/282] Add an explicit warning about the current API-instability of the sqlite3.opfs namespace, which may need to be eliminated based on re-thinking of how the OPFS sqlite3_vfs is registered. Comment changes only - no code. FossilOrigin-Name: 0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 --- ext/wasm/api/sqlite3-api-opfs.js | 7 +++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 5537df89fb..458448175b 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -128,6 +128,10 @@ const installOpfsVfs = function callee(options){ /** Generic utilities for working with OPFS. This will get filled out by the Promise setup and, on success, installed as sqlite3.opfs. + + ACHTUNG: do not rely on these APIs in client code. They are + experimental and subject to change or removal as the + OPFS-specific sqlite3_vfs evolves. */ const opfsUtil = Object.create(null); /** @@ -186,8 +190,7 @@ const installOpfsVfs = function callee(options){ const dVfs = pDVfs ? new sqlite3_vfs(pDVfs) : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. Though we cannot currently handle - that case, the hope is to eventually be able to. */; + SQLITE_OS_OTHER. */; const opfsVfs = new sqlite3_vfs(); const opfsIoMethods = new sqlite3_io_methods(); opfsVfs.$iVersion = 2/*yes, two*/; diff --git a/manifest b/manifest index 04e653dcec..01aaf2a316 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Check-in\s[8d5b76593d82b3a5]\scontained\san\serror\sthat\swas\scausing\ssome\sobscure\nerror\scodes\sto\sbe\slost.\s\sFixed\shere. -D 2022-11-28T21:17:30.250 +C Add\san\sexplicit\swarning\sabout\sthe\scurrent\sAPI-instability\sof\sthe\ssqlite3.opfs\snamespace,\swhich\smay\sneed\sto\sbe\seliminated\sbased\son\sre-thinking\sof\show\sthe\sOPFS\ssqlite3_vfs\sis\sregistered.\sComment\schanges\sonly\s-\sno\scode. +D 2022-11-29T02:23:12.943 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,7 +505,7 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18dd87fab2ce05284a10d7 +F ext/wasm/api/sqlite3-api-opfs.js 3cdae7e98c500f89f9468a260e2a0e1b528c845a107bf72d368e5222769214d3 F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb2c5d088e4784f6763eb0ea2a8542ad2129529f181fa92963c7231d7a68f25c -R edc56f3db34be700dc2fe2c48f4c3e31 -U drh -Z fbd7482d8e22e645fc260e0f000cee41 +P 46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 +R 3870d04bfd54da096e986662fe29b1c8 +U stephan +Z ad4a8c5f45a34a10f588c0c6dc455846 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2763f271ba..361de4178e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 \ No newline at end of file +0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 \ No newline at end of file From 04184761de820ac763036157ff07b2f22a89db77 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Nov 2022 05:25:08 +0000 Subject: [PATCH 098/282] Internal restructuring of the OPFS sqlite3_vfs in order to facilitate certain experimentation and improve error reporting/hints if it cannot be activated. Deprecate the name sqlite3.opfs.OpfsDb, preferring sqlite3.oo1.OpfsDb for consistency with JsStorageDb and any future DB subclasses. FossilOrigin-Name: 0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 --- ext/wasm/api/sqlite3-api-opfs.js | 79 +- ext/wasm/api/sqlite3-opfs-async-proxy.js | 1655 ++++++++++---------- ext/wasm/tester1.c-pp.js | 34 +- ext/wasm/tests/opfs/concurrency/index.html | 7 +- manifest | 18 +- manifest.uuid | 2 +- 6 files changed, 919 insertions(+), 876 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 458448175b..cef15305d4 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -76,15 +76,25 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `opfs` property, containing several OPFS-specific utilities. */ const installOpfsVfs = function callee(options){ - if(!self.SharedArrayBuffer || - !self.Atomics || - !self.FileSystemHandle || - !self.FileSystemDirectoryHandle || - !self.FileSystemFileHandle || - !self.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator.storage.getDirectory){ + if(!self.SharedArrayBuffer + || !self.Atomics){ return Promise.reject( - new Error("This environment does not have OPFS support.") + new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ + "The server must emit the COOP/COEP response headers to enable those. "+ + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") + ); + }else if(self.window===self && self.document){ + return Promise.reject( + new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ + "because it requires Atomics.wait().") + ); + }else if(!self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + return Promise.reject( + new Error("Missing required OPFS APIs.") ); } if(!options || 'object'!==typeof options){ @@ -134,6 +144,18 @@ const installOpfsVfs = function callee(options){ OPFS-specific sqlite3_vfs evolves. */ const opfsUtil = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + const thisThreadHasOPFS = ()=>{ + return self.FileSystemHandle && + self.FileSystemDirectoryHandle && + self.FileSystemFileHandle && + self.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator.storage.getDirectory; + }; + /** Not part of the public API. Solely for internal/development use. @@ -1179,12 +1201,16 @@ const installOpfsVfs = function callee(options){ //consideration. if(sqlite3.oo1){ - opfsUtil.OpfsDb = function(...args){ + const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); opt.vfs = opfsVfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; - opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsDb = + opfsUtil.OpfsDb /* sqlite3.opfs.OpfsDb => deprecated name - + will be phased out Real Soon */ = + OpfsDb; + OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, [ @@ -1206,13 +1232,6 @@ const installOpfsVfs = function callee(options){ ); } - /** - Potential TODOs: - - - Expose one or both of the Worker objects via opfsUtil and - publish an interface for proxying the higher-level OPFS - features like getting a directory listing. - */ const sanityCheck = function(){ const scope = wasm.scopedAllocPush(); const sq3File = new sqlite3_file(); @@ -1282,6 +1301,11 @@ const installOpfsVfs = function callee(options){ W.onmessage = function({data}){ //log("Worker.onmessage:",data); switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; case 'opfs-async-loaded': /*Arrives as soon as the asyc proxy finishes loading. Pass our config and shared state on to the async worker.*/ @@ -1308,14 +1332,18 @@ const installOpfsVfs = function callee(options){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); } - navigator.storage.getDirectory().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); + if(thisThreadHasOPFS()){ + navigator.storage.getDirectory().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + sqlite3.opfs = opfsUtil; + opfsUtil.rootDirectory = d; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(sqlite3); + }); + }else{ promiseResolve(sqlite3); - }); + } }catch(e){ error(e); promiseReject(e); @@ -1334,9 +1362,6 @@ const installOpfsVfs = function callee(options){ installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - if(sqlite3.scriptInfo && !sqlite3.scriptInfo.isWorker){ - return; - } try{ let proxyJs = installOpfsVfs.defaultProxyUri; if(sqlite3.scriptInfo.sqlite3Dir){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 8e60969bc6..1ba6e9bdbb 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -47,854 +47,871 @@ usage of those methods to remove the "await". */ "use strict"; -const toss = function(...args){throw new Error(args.join(' '))}; -if(self.window === self){ - toss("This code cannot run from the main thread.", - "Load it as a Worker from a separate Worker."); -}else if(!navigator.storage.getDirectory){ - toss("This API requires navigator.storage.getDirectory."); -} - -/** - Will hold state copied to this object from the syncronous side of - this API. -*/ -const state = Object.create(null); - -/** - verbose: - - 0 = no logging output - 1 = only errors - 2 = warnings and errors - 3 = debug, warnings, and errors -*/ -state.verbose = 1; - -const loggers = { - 0:console.error.bind(console), - 1:console.warn.bind(console), - 2:console.log.bind(console) -}; -const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); -}; -const log = (...args)=>logImpl(2, ...args); -const warn = (...args)=>logImpl(1, ...args); -const error = (...args)=>logImpl(0, ...args); -const metrics = Object.create(null); -metrics.reset = ()=>{ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); +const wPost = (type,...args)=>postMessage({type, payload:args}); +const installAsyncProxy = function(self){ + const toss = function(...args){throw new Error(args.join(' '))}; + if(self.window === self){ + toss("This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); + }else if(!navigator.storage.getDirectory){ + toss("This API requires navigator.storage.getDirectory."); } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; -}; -metrics.dump = ()=>{ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - } - console.log(self.location.href, - "metrics for",self.location.href,":\n", - metrics, - "\nTotal of",n,"op(s) for",t,"ms", - "approx",w,"ms spent waiting on OPFS APIs."); - console.log("Serialization metrics:",metrics.s11n); -}; -/** - __openFiles is a map of sqlite3_file pointers (integers) to - metadata related to a given OPFS file handles. The pointers are, in - this side of the interface, opaque file handle IDs provided by the - synchronous part of this constellation. Each value is an object - with a structure demonstrated in the xOpen() impl. -*/ -const __openFiles = Object.create(null); -/** - __implicitLocks is a Set of sqlite3_file pointers (integers) which were - "auto-locked". i.e. those for which we obtained a sync access - handle without an explicit xLock() call. Such locks will be - released during db connection idle time, whereas a sync access - handle obtained via xLock(), or subsequently xLock()'d after - auto-acquisition, will not be released until xUnlock() is called. - - Maintenance reminder: if we relinquish auto-locks at the end of the - operation which acquires them, we pay a massive performance - penalty: speedtest1 benchmarks take up to 4x as long. By delaying - the lock release until idle time, the hit is negligible. -*/ -const __implicitLocks = new Set(); - -/** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg is - true, the result is returned as an array of path elements, else an - absolute path string is returned. -*/ -const getResolvedPath = function(filename,splitIt){ - const p = new URL( - filename, 'file://irrelevant' - ).pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; -}; - -/** - Takes the absolute path to a filesystem element. Returns an array - of [handleOfContainingDir, filename]. If the 2nd argument is truthy - then each directory element leading to the file is created along - the way. Throws if any creation or resolution fails. -*/ -const getDirForFilename = async function f(absFilename, createDirs = false){ - const path = getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = state.rootDir; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; -}; - -/** - If the given file-holding object has a sync handle attached to it, - that handle is remove and asynchronously closed. Though it may - sound sensible to continue work as soon as the close() returns - (noting that it's asynchronous), doing so can cause operations - performed soon afterwards, e.g. a call to getSyncHandle() to fail - because they may happen out of order from the close(). OPFS does - not guaranty that the actual order of operations is retained in - such cases. i.e. always "await" on the result of this function. -*/ -const closeSyncHandle = async (fh)=>{ - if(fh.syncHandle){ - log("Closing sync handle for",fh.filenameAbs); - const h = fh.syncHandle; - delete fh.syncHandle; - delete fh.xLock; - __implicitLocks.delete(fh.fid); - return h.close(); - } -}; - -/** - A proxy for closeSyncHandle() which is guaranteed to not throw. - - This function is part of a lock/unlock step in functions which - require a sync access handle but may be called without xLock() - having been called first. Such calls need to release that - handle to avoid locking the file for all of time. This is an - _attempt_ at reducing cross-tab contention but it may prove - to be more of a problem than a solution and may need to be - removed. -*/ -const closeSyncHandleNoThrow = async (fh)=>{ - try{await closeSyncHandle(fh)} - catch(e){ - warn("closeSyncHandleNoThrow() ignoring:",e,fh); - } -}; - -/* Release all auto-locks. */ -const releaseImplicitLocks = async ()=>{ - if(__implicitLocks.size){ - /* Release all auto-locks. */ - for(const fid of __implicitLocks){ - const fh = __openFiles[fid]; - await closeSyncHandleNoThrow(fh); - log("Auto-unlocked",fid,fh.filenameAbs); - } - } -}; - -/** - An experiment in improving concurrency by freeing up implicit locks - sooner. This is known to impact performance dramatically but it has - also shown to improve concurrency considerably. - - If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, - this routine returns closeSyncHandleNoThrow(), else it is a no-op. -*/ -const releaseImplicitLock = async (fh)=>{ - if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ - return closeSyncHandleNoThrow(fh); - } -}; - -/** - An error class specifically for use with getSyncHandle(), the goal - of which is to eventually be able to distinguish unambiguously - between locking-related failures and other types, noting that we - cannot currently do so because createSyncAccessHandle() does not - define its exceptions in the required level of detail. -*/ -class GetSyncHandleError extends Error { - constructor(errorObject, ...msg){ - super(); - this.error = errorObject; - this.message = [ - ...msg, ': Original exception ['+errorObject.name+']:', - errorObject.message - ].join(' '); - this.name = 'GetSyncHandleError'; - } -}; -GetSyncHandleError.convertRc = (e,rc)=>{ - if(0){ - /* This approach makes the very wild assumption that such a - failure _is_ a locking error. In practice that appears to be - the most common error, by far, but we cannot unambiguously - distinguish that from other errors. - - This approach is highly questionable. - */ - return (e instanceof GetSyncHandleError) - ? state.sq3Codes.SQLITE_IOERR_LOCK - : rc; - }else{ - return rc; - } -} -/** - Returns the sync access handle associated with the given file - handle object (which must be a valid handle object, as created by - xOpen()), lazily opening it if needed. - - In order to help alleviate cross-tab contention for a dabase, - if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to 3 times. If acquisition - still fails at that point it will give up and propagate the - exception. -*/ -const getSyncHandle = async (fh,opName)=>{ - if(!fh.syncHandle){ - const t = performance.now(); - log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, msBase = 300; - let i = 1, ms = msBase; - for(; true; ms = msBase * ++i){ - try { - //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); - //TODO? A config option which tells it to throw here - //randomly every now and then, for testing purposes. - fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); - break; - }catch(e){ - if(i === maxTries){ - throw new GetSyncHandleError( - e, "Error getting sync handle for",opName+"().",maxTries, - "attempts failed.",fh.filenameAbs - ); - } - warn("Error getting sync handle for",opName+"(). Waiting",ms, - "ms and trying again.",fh.filenameAbs,e); - //await releaseImplicitLocks(); - Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); - } - } - log("Got",opName+"() sync handle for",fh.filenameAbs, - 'in',performance.now() - t,'ms'); - if(!fh.xLock){ - __implicitLocks.add(fh.fid); - log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); - } - } - return fh.syncHandle; -}; - -/** - Stores the given value at state.sabOPView[state.opIds.rc] and then - Atomics.notify()'s it. -*/ -const storeAndNotify = (opName, value)=>{ - log(opName+"() => notify(",value,")"); - Atomics.store(state.sabOPView, state.opIds.rc, value); - Atomics.notify(state.sabOPView, state.opIds.rc); -}; - -/** - Throws if fh is a file-holding object which is flagged as read-only. -*/ -const affirmNotRO = function(opName,fh){ - if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); -}; -const affirmLocked = function(opName,fh){ - //if(!fh.syncHandle) toss(opName+"(): File does not have a lock: "+fh.filenameAbs); /** - Currently a no-op, as speedtest1 triggers xRead() without a - lock (that seems like a bug but it's currently uninvestigated). - This means, however, that some OPFS VFS routines may trigger - acquisition of a lock but never let it go until xUnlock() is - called (which it likely won't be if xLock() was not called). + Will hold state copied to this object from the syncronous side of + this API. */ -}; + const state = Object.create(null); -/** - We track 2 different timers: the "metrics" timer records how much - time we spend performing work. The "wait" timer records how much - time we spend waiting on the underlying OPFS timer. See the calls - to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() - throughout this file to see how they're used. -*/ -const __mTimer = Object.create(null); -__mTimer.op = undefined; -__mTimer.start = undefined; -const mTimeStart = (op)=>{ - __mTimer.start = performance.now(); - __mTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); - ++metrics[op].count; -}; -const mTimeEnd = ()=>( - metrics[__mTimer.op].time += performance.now() - __mTimer.start -); -const __wTimer = Object.create(null); -__wTimer.op = undefined; -__wTimer.start = undefined; -const wTimeStart = (op)=>{ - __wTimer.start = performance.now(); - __wTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); -}; -const wTimeEnd = ()=>( - metrics[__wTimer.op].wait += performance.now() - __wTimer.start -); + /** + verbose: -/** - Gets set to true by the 'opfs-async-shutdown' command to quit the - wait loop. This is only intended for debugging purposes: we cannot - inspect this file's state while the tight waitLoop() is running and - need a way to stop that loop for introspection purposes. -*/ -let flagAsyncShutdown = false; + 0 = no logging output + 1 = only errors + 2 = warnings and errors + 3 = debug, warnings, and errors + */ + state.verbose = 1; - -/** - Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods - methods, as well as helpers like mkdir(). Maintenance reminder: - members are in alphabetical order to simplify finding them. -*/ -const vfsAsyncImpls = { - 'opfs-async-metrics': async ()=>{ - mTimeStart('opfs-async-metrics'); - metrics.dump(); - storeAndNotify('opfs-async-metrics', 0); - mTimeEnd(); - }, - 'opfs-async-shutdown': async ()=>{ - flagAsyncShutdown = true; - storeAndNotify('opfs-async-shutdown', 0); - }, - mkdir: async (dirname)=>{ - mTimeStart('mkdir'); - let rc = 0; - wTimeStart('mkdir'); - try { - await getDirForFilename(dirname+"/filepart", true); - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); + const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) + }; + const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const metrics = Object.create(null); + metrics.reset = ()=>{ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); } - storeAndNotify('mkdir', rc); - mTimeEnd(); - }, - xAccess: async (filename)=>{ - mTimeStart('xAccess'); - /* OPFS cannot support the full range of xAccess() queries sqlite3 - calls for. We can essentially just tell if the file is - accessible, but if it is it's automatically writable (unless - it's locked, which we cannot(?) know without trying to open - it). OPFS does not have the notion of read-only. - - The return semantics of this function differ from sqlite3's - xAccess semantics because we are limited in what we can - communicate back to our synchronous communication partner: 0 = - accessible, non-0 means not accessible. - */ - let rc = 0; - wTimeStart('xAccess'); - try{ - const [dh, fn] = await getDirForFilename(filename); - await dh.getFileHandle(fn); - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + }; + metrics.dump = ()=>{ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; } - storeAndNotify('xAccess', rc); - mTimeEnd(); - }, - xClose: async function(fid/*sqlite3_file pointer*/){ - const opName = 'xClose'; - mTimeStart(opName); - __implicitLocks.delete(fid); - const fh = __openFiles[fid]; - let rc = 0; - wTimeStart(opName); - if(fh){ - delete __openFiles[fid]; - await closeSyncHandle(fh); - if(fh.deleteOnClose){ - try{ await fh.dirHandle.removeEntry(fh.filenamePart) } - catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + console.log(self.location.href, + "metrics for",self.location.href,":\n", + metrics, + "\nTotal of",n,"op(s) for",t,"ms", + "approx",w,"ms spent waiting on OPFS APIs."); + console.log("Serialization metrics:",metrics.s11n); + }; + + /** + __openFiles is a map of sqlite3_file pointers (integers) to + metadata related to a given OPFS file handles. The pointers are, in + this side of the interface, opaque file handle IDs provided by the + synchronous part of this constellation. Each value is an object + with a structure demonstrated in the xOpen() impl. + */ + const __openFiles = Object.create(null); + /** + __implicitLocks is a Set of sqlite3_file pointers (integers) which were + "auto-locked". i.e. those for which we obtained a sync access + handle without an explicit xLock() call. Such locks will be + released during db connection idle time, whereas a sync access + handle obtained via xLock(), or subsequently xLock()'d after + auto-acquisition, will not be released until xUnlock() is called. + + Maintenance reminder: if we relinquish auto-locks at the end of the + operation which acquires them, we pay a massive performance + penalty: speedtest1 benchmarks take up to 4x as long. By delaying + the lock release until idle time, the hit is negligible. + */ + const __implicitLocks = new Set(); + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg is + true, the result is returned as an array of path elements, else an + absolute path string is returned. + */ + const getResolvedPath = function(filename,splitIt){ + const p = new URL( + filename, 'file://irrelevant' + ).pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an array + of [handleOfContainingDir, filename]. If the 2nd argument is truthy + then each directory element leading to the file is created along + the way. Throws if any creation or resolution fails. + */ + const getDirForFilename = async function f(absFilename, createDirs = false){ + const path = getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = state.rootDir; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); } + } + return [dh, filename]; + }; + + /** + If the given file-holding object has a sync handle attached to it, + that handle is remove and asynchronously closed. Though it may + sound sensible to continue work as soon as the close() returns + (noting that it's asynchronous), doing so can cause operations + performed soon afterwards, e.g. a call to getSyncHandle() to fail + because they may happen out of order from the close(). OPFS does + not guaranty that the actual order of operations is retained in + such cases. i.e. always "await" on the result of this function. + */ + const closeSyncHandle = async (fh)=>{ + if(fh.syncHandle){ + log("Closing sync handle for",fh.filenameAbs); + const h = fh.syncHandle; + delete fh.syncHandle; + delete fh.xLock; + __implicitLocks.delete(fh.fid); + return h.close(); + } + }; + + /** + A proxy for closeSyncHandle() which is guaranteed to not throw. + + This function is part of a lock/unlock step in functions which + require a sync access handle but may be called without xLock() + having been called first. Such calls need to release that + handle to avoid locking the file for all of time. This is an + _attempt_ at reducing cross-tab contention but it may prove + to be more of a problem than a solution and may need to be + removed. + */ + const closeSyncHandleNoThrow = async (fh)=>{ + try{await closeSyncHandle(fh)} + catch(e){ + warn("closeSyncHandleNoThrow() ignoring:",e,fh); + } + }; + + /* Release all auto-locks. */ + const releaseImplicitLocks = async ()=>{ + if(__implicitLocks.size){ + /* Release all auto-locks. */ + for(const fid of __implicitLocks){ + const fh = __openFiles[fid]; + await closeSyncHandleNoThrow(fh); + log("Auto-unlocked",fid,fh.filenameAbs); + } + } + }; + + /** + An experiment in improving concurrency by freeing up implicit locks + sooner. This is known to impact performance dramatically but it has + also shown to improve concurrency considerably. + + If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, + this routine returns closeSyncHandleNoThrow(), else it is a no-op. + */ + const releaseImplicitLock = async (fh)=>{ + if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ + return closeSyncHandleNoThrow(fh); + } + }; + + /** + An error class specifically for use with getSyncHandle(), the goal + of which is to eventually be able to distinguish unambiguously + between locking-related failures and other types, noting that we + cannot currently do so because createSyncAccessHandle() does not + define its exceptions in the required level of detail. + */ + class GetSyncHandleError extends Error { + constructor(errorObject, ...msg){ + super(); + this.error = errorObject; + this.message = [ + ...msg, ': Original exception ['+errorObject.name+']:', + errorObject.message + ].join(' '); + this.name = 'GetSyncHandleError'; + } + }; + GetSyncHandleError.convertRc = (e,rc)=>{ + if(0){ + /* This approach makes the very wild assumption that such a + failure _is_ a locking error. In practice that appears to be + the most common error, by far, but we cannot unambiguously + distinguish that from other errors. + + This approach is highly questionable. + */ + return (e instanceof GetSyncHandleError) + ? state.sq3Codes.SQLITE_IOERR_LOCK + : rc; }else{ - state.s11n.serialize(); - rc = state.sq3Codes.SQLITE_NOTFOUND; + return rc; } - wTimeEnd(); - storeAndNotify(opName, rc); - mTimeEnd(); - }, - xDelete: async function(...args){ - mTimeStart('xDelete'); - const rc = await vfsAsyncImpls.xDeleteNoWait(...args); - storeAndNotify('xDelete', rc); - mTimeEnd(); - }, - xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ - /* The syncDir flag is, for purposes of the VFS API's semantics, - ignored here. However, if it has the value 0x1234 then: after - deleting the given file, recursively try to delete any empty - directories left behind in its wake (ignoring any errors and - stopping at the first failure). + } + /** + Returns the sync access handle associated with the given file + handle object (which must be a valid handle object, as created by + xOpen()), lazily opening it if needed. - That said: we don't know for sure that removeEntry() fails if - the dir is not empty because the API is not documented. It has, - however, a "recursive" flag which defaults to false, so - presumably it will fail if the dir is not empty and that flag - is false. + In order to help alleviate cross-tab contention for a dabase, + if an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to 3 times. If acquisition + still fails at that point it will give up and propagate the + exception. + */ + const getSyncHandle = async (fh,opName)=>{ + if(!fh.syncHandle){ + const t = performance.now(); + log("Acquiring sync handle for",fh.filenameAbs); + const maxTries = 6, msBase = 300; + let i = 1, ms = msBase; + for(; true; ms = msBase * ++i){ + try { + //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); + //TODO? A config option which tells it to throw here + //randomly every now and then, for testing purposes. + fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); + break; + }catch(e){ + if(i === maxTries){ + throw new GetSyncHandleError( + e, "Error getting sync handle for",opName+"().",maxTries, + "attempts failed.",fh.filenameAbs + ); + } + warn("Error getting sync handle for",opName+"(). Waiting",ms, + "ms and trying again.",fh.filenameAbs,e); + //await releaseImplicitLocks(); + Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); + } + } + log("Got",opName+"() sync handle for",fh.filenameAbs, + 'in',performance.now() - t,'ms'); + if(!fh.xLock){ + __implicitLocks.add(fh.fid); + log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); + } + } + return fh.syncHandle; + }; + + /** + Stores the given value at state.sabOPView[state.opIds.rc] and then + Atomics.notify()'s it. + */ + const storeAndNotify = (opName, value)=>{ + log(opName+"() => notify(",value,")"); + Atomics.store(state.sabOPView, state.opIds.rc, value); + Atomics.notify(state.sabOPView, state.opIds.rc); + }; + + /** + Throws if fh is a file-holding object which is flagged as read-only. + */ + const affirmNotRO = function(opName,fh){ + if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); + }; + const affirmLocked = function(opName,fh){ + //if(!fh.syncHandle) toss(opName+"(): File does not have a lock: "+fh.filenameAbs); + /** + Currently a no-op, as speedtest1 triggers xRead() without a + lock (that seems like a bug but it's currently uninvestigated). + This means, however, that some OPFS VFS routines may trigger + acquisition of a lock but never let it go until xUnlock() is + called (which it likely won't be if xLock() was not called). */ - let rc = 0; - wTimeStart('xDelete'); - try { - while(filename){ - const [hDir, filenamePart] = await getDirForFilename(filename, false); - if(!filenamePart) break; - await hDir.removeEntry(filenamePart, {recursive}); - if(0x1234 !== syncDir) break; - recursive = false; - filename = getResolvedPath(filename, true); - filename.pop(); - filename = filename.join('/'); - } - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR_DELETE; - } - wTimeEnd(); - return rc; - }, - xFileSize: async function(fid/*sqlite3_file pointer*/){ - mTimeStart('xFileSize'); - const fh = __openFiles[fid]; - let rc; - wTimeStart('xFileSize'); - try{ - affirmLocked('xFileSize',fh); - const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); - state.s11n.serialize(Number(sz)); - rc = 0; - }catch(e){ - state.s11n.storeException(2,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); - } - await releaseImplicitLock(fh); - wTimeEnd(); - storeAndNotify('xFileSize', rc); - mTimeEnd(); - }, - xLock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xLock'); - const fh = __openFiles[fid]; - let rc = 0; - const oldLockType = fh.xLock; - fh.xLock = lockType; - if( !fh.syncHandle ){ - wTimeStart('xLock'); - try { - await getSyncHandle(fh,'xLock'); - __implicitLocks.delete(fid); - }catch(e){ - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); - fh.xLock = oldLockType; - } - wTimeEnd(); - } - storeAndNotify('xLock',rc); - mTimeEnd(); - }, - xOpen: async function(fid/*sqlite3_file pointer*/, filename, - flags/*SQLITE_OPEN_...*/, - opfsFlags/*OPFS_...*/){ - const opName = 'xOpen'; - mTimeStart(opName); - const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); - wTimeStart('xOpen'); - try{ - let hDir, filenamePart; - try { - [hDir, filenamePart] = await getDirForFilename(filename, !!create); - }catch(e){ - state.s11n.storeException(1,e); - storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); - mTimeEnd(); - wTimeEnd(); - return; - } - const hFile = await hDir.getFileHandle(filenamePart, {create}); - wTimeEnd(); - const fh = Object.assign(Object.create(null),{ - fid: fid, - filenameAbs: filename, - filenamePart: filenamePart, - dirHandle: hDir, - fileHandle: hFile, - sabView: state.sabFileBufView, - readOnly: create - ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), - deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) - }); - fh.releaseImplicitLocks = - (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) - || state.opfsFlags.defaultUnlockAsap; - if(0 /* this block is modelled after something wa-sqlite - does but it leads to immediate contention on journal files. */ - && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ - /* sqlite does not lock these files, so go ahead and grab an OPFS - lock. + }; - https://www.sqlite.org/uri.html - */ - fh.xLock = "xOpen"/* Truthy value to keep entry from getting - flagged as auto-locked. String value so - that we can easily distinguish is later - if needed. */; - await getSyncHandle(fh,'xOpen'); - } - __openFiles[fid] = fh; - storeAndNotify(opName, 0); - }catch(e){ - wTimeEnd(); - error(opName,e); - state.s11n.storeException(1,e); - storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); - } - mTimeEnd(); - }, - xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xRead'); - let rc = 0, nRead; - const fh = __openFiles[fid]; - try{ - affirmLocked('xRead',fh); - wTimeStart('xRead'); - nRead = (await getSyncHandle(fh,'xRead')).read( - fh.sabView.subarray(0, n), - {at: Number(offset64)} - ); - wTimeEnd(); - if(nRead < n){/* Zero-fill remaining bytes */ - fh.sabView.fill(0, nRead, n); - rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; - } - }catch(e){ - if(undefined===nRead) wTimeEnd(); - error("xRead() failed",e,fh); - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); - } - await releaseImplicitLock(fh); - storeAndNotify('xRead',rc); - mTimeEnd(); - }, - xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ - mTimeStart('xSync'); - const fh = __openFiles[fid]; - let rc = 0; - if(!fh.readOnly && fh.syncHandle){ + /** + We track 2 different timers: the "metrics" timer records how much + time we spend performing work. The "wait" timer records how much + time we spend waiting on the underlying OPFS timer. See the calls + to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() + throughout this file to see how they're used. + */ + const __mTimer = Object.create(null); + __mTimer.op = undefined; + __mTimer.start = undefined; + const mTimeStart = (op)=>{ + __mTimer.start = performance.now(); + __mTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + ++metrics[op].count; + }; + const mTimeEnd = ()=>( + metrics[__mTimer.op].time += performance.now() - __mTimer.start + ); + const __wTimer = Object.create(null); + __wTimer.op = undefined; + __wTimer.start = undefined; + const wTimeStart = (op)=>{ + __wTimer.start = performance.now(); + __wTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + }; + const wTimeEnd = ()=>( + metrics[__wTimer.op].wait += performance.now() - __wTimer.start + ); + + /** + Gets set to true by the 'opfs-async-shutdown' command to quit the + wait loop. This is only intended for debugging purposes: we cannot + inspect this file's state while the tight waitLoop() is running and + need a way to stop that loop for introspection purposes. + */ + let flagAsyncShutdown = false; + + + /** + Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods + methods, as well as helpers like mkdir(). Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const vfsAsyncImpls = { + 'opfs-async-metrics': async ()=>{ + mTimeStart('opfs-async-metrics'); + metrics.dump(); + storeAndNotify('opfs-async-metrics', 0); + mTimeEnd(); + }, + 'opfs-async-shutdown': async ()=>{ + flagAsyncShutdown = true; + storeAndNotify('opfs-async-shutdown', 0); + }, + mkdir: async (dirname)=>{ + mTimeStart('mkdir'); + let rc = 0; + wTimeStart('mkdir'); try { - wTimeStart('xSync'); - await fh.syncHandle.flush(); + await getDirForFilename(dirname+"/filepart", true); }catch(e){ state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR_FSYNC; + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('mkdir', rc); + mTimeEnd(); + }, + xAccess: async (filename)=>{ + mTimeStart('xAccess'); + /* OPFS cannot support the full range of xAccess() queries sqlite3 + calls for. We can essentially just tell if the file is + accessible, but if it is it's automatically writable (unless + it's locked, which we cannot(?) know without trying to open + it). OPFS does not have the notion of read-only. + + The return semantics of this function differ from sqlite3's + xAccess semantics because we are limited in what we can + communicate back to our synchronous communication partner: 0 = + accessible, non-0 means not accessible. + */ + let rc = 0; + wTimeStart('xAccess'); + try{ + const [dh, fn] = await getDirForFilename(filename); + await dh.getFileHandle(fn); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('xAccess', rc); + mTimeEnd(); + }, + xClose: async function(fid/*sqlite3_file pointer*/){ + const opName = 'xClose'; + mTimeStart(opName); + __implicitLocks.delete(fid); + const fh = __openFiles[fid]; + let rc = 0; + wTimeStart(opName); + if(fh){ + delete __openFiles[fid]; + await closeSyncHandle(fh); + if(fh.deleteOnClose){ + try{ await fh.dirHandle.removeEntry(fh.filenamePart) } + catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + } + }else{ + state.s11n.serialize(); + rc = state.sq3Codes.SQLITE_NOTFOUND; } wTimeEnd(); - } - storeAndNotify('xSync',rc); - mTimeEnd(); - }, - xTruncate: async function(fid/*sqlite3_file pointer*/,size){ - mTimeStart('xTruncate'); - let rc = 0; - const fh = __openFiles[fid]; - wTimeStart('xTruncate'); - try{ - affirmLocked('xTruncate',fh); - affirmNotRO('xTruncate', fh); - await (await getSyncHandle(fh,'xTruncate')).truncate(size); - }catch(e){ - error("xTruncate():",e,fh); - state.s11n.storeException(2,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); - } - await releaseImplicitLock(fh); - wTimeEnd(); - storeAndNotify('xTruncate',rc); - mTimeEnd(); - }, - xUnlock: async function(fid/*sqlite3_file pointer*/, + storeAndNotify(opName, rc); + mTimeEnd(); + }, + xDelete: async function(...args){ + mTimeStart('xDelete'); + const rc = await vfsAsyncImpls.xDeleteNoWait(...args); + storeAndNotify('xDelete', rc); + mTimeEnd(); + }, + xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ + /* The syncDir flag is, for purposes of the VFS API's semantics, + ignored here. However, if it has the value 0x1234 then: after + deleting the given file, recursively try to delete any empty + directories left behind in its wake (ignoring any errors and + stopping at the first failure). + + That said: we don't know for sure that removeEntry() fails if + the dir is not empty because the API is not documented. It has, + however, a "recursive" flag which defaults to false, so + presumably it will fail if the dir is not empty and that flag + is false. + */ + let rc = 0; + wTimeStart('xDelete'); + try { + while(filename){ + const [hDir, filenamePart] = await getDirForFilename(filename, false); + if(!filenamePart) break; + await hDir.removeEntry(filenamePart, {recursive}); + if(0x1234 !== syncDir) break; + recursive = false; + filename = getResolvedPath(filename, true); + filename.pop(); + filename = filename.join('/'); + } + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_DELETE; + } + wTimeEnd(); + return rc; + }, + xFileSize: async function(fid/*sqlite3_file pointer*/){ + mTimeStart('xFileSize'); + const fh = __openFiles[fid]; + let rc; + wTimeStart('xFileSize'); + try{ + affirmLocked('xFileSize',fh); + const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); + state.s11n.serialize(Number(sz)); + rc = 0; + }catch(e){ + state.s11n.storeException(2,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); + } + await releaseImplicitLock(fh); + wTimeEnd(); + storeAndNotify('xFileSize', rc); + mTimeEnd(); + }, + xLock: async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xUnlock'); - let rc = 0; - const fh = __openFiles[fid]; - if( state.sq3Codes.SQLITE_LOCK_NONE===lockType - && fh.syncHandle ){ - wTimeStart('xUnlock'); - try { await closeSyncHandle(fh) } - catch(e){ + mTimeStart('xLock'); + const fh = __openFiles[fid]; + let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; + if( !fh.syncHandle ){ + wTimeStart('xLock'); + try { + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); + }catch(e){ + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); + fh.xLock = oldLockType; + } + wTimeEnd(); + } + storeAndNotify('xLock',rc); + mTimeEnd(); + }, + xOpen: async function(fid/*sqlite3_file pointer*/, filename, + flags/*SQLITE_OPEN_...*/, + opfsFlags/*OPFS_...*/){ + const opName = 'xOpen'; + mTimeStart(opName); + const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); + wTimeStart('xOpen'); + try{ + let hDir, filenamePart; + try { + [hDir, filenamePart] = await getDirForFilename(filename, !!create); + }catch(e){ + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); + mTimeEnd(); + wTimeEnd(); + return; + } + const hFile = await hDir.getFileHandle(filenamePart, {create}); + wTimeEnd(); + const fh = Object.assign(Object.create(null),{ + fid: fid, + filenameAbs: filename, + filenamePart: filenamePart, + dirHandle: hDir, + fileHandle: hFile, + sabView: state.sabFileBufView, + readOnly: create + ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), + deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) + }); + fh.releaseImplicitLocks = + (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) + || state.opfsFlags.defaultUnlockAsap; + if(0 /* this block is modelled after something wa-sqlite + does but it leads to immediate contention on journal files. */ + && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ + /* sqlite does not lock these files, so go ahead and grab an OPFS + lock. + + https://www.sqlite.org/uri.html + */ + fh.xLock = "xOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; + await getSyncHandle(fh,'xOpen'); + } + __openFiles[fid] = fh; + storeAndNotify(opName, 0); + }catch(e){ + wTimeEnd(); + error(opName,e); state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); } + mTimeEnd(); + }, + xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xRead'); + let rc = 0, nRead; + const fh = __openFiles[fid]; + try{ + affirmLocked('xRead',fh); + wTimeStart('xRead'); + nRead = (await getSyncHandle(fh,'xRead')).read( + fh.sabView.subarray(0, n), + {at: Number(offset64)} + ); + wTimeEnd(); + if(nRead < n){/* Zero-fill remaining bytes */ + fh.sabView.fill(0, nRead, n); + rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; + } + }catch(e){ + if(undefined===nRead) wTimeEnd(); + error("xRead() failed",e,fh); + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); + } + await releaseImplicitLock(fh); + storeAndNotify('xRead',rc); + mTimeEnd(); + }, + xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ + mTimeStart('xSync'); + const fh = __openFiles[fid]; + let rc = 0; + if(!fh.readOnly && fh.syncHandle){ + try { + wTimeStart('xSync'); + await fh.syncHandle.flush(); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_FSYNC; + } + wTimeEnd(); + } + storeAndNotify('xSync',rc); + mTimeEnd(); + }, + xTruncate: async function(fid/*sqlite3_file pointer*/,size){ + mTimeStart('xTruncate'); + let rc = 0; + const fh = __openFiles[fid]; + wTimeStart('xTruncate'); + try{ + affirmLocked('xTruncate',fh); + affirmNotRO('xTruncate', fh); + await (await getSyncHandle(fh,'xTruncate')).truncate(size); + }catch(e){ + error("xTruncate():",e,fh); + state.s11n.storeException(2,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); + } + await releaseImplicitLock(fh); wTimeEnd(); - } - storeAndNotify('xUnlock',rc); - mTimeEnd(); - }, - xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xWrite'); - let rc; - const fh = __openFiles[fid]; - wTimeStart('xWrite'); - try{ - affirmLocked('xWrite',fh); - affirmNotRO('xWrite', fh); - rc = ( - n === (await getSyncHandle(fh,'xWrite')) - .write(fh.sabView.subarray(0, n), - {at: Number(offset64)}) - ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; - }catch(e){ - error("xWrite():",e,fh); - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); - } - await releaseImplicitLock(fh); - wTimeEnd(); - storeAndNotify('xWrite',rc); - mTimeEnd(); - } -}/*vfsAsyncImpls*/; - -const initS11n = ()=>{ - /** - ACHTUNG: this code is 100% duplicated in the other half of this - proxy! The documentation is maintained in the "synchronous half". - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; + storeAndNotify('xTruncate',rc); + mTimeEnd(); + }, + xUnlock: async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + mTimeStart('xUnlock'); + let rc = 0; + const fh = __openFiles[fid]; + if( state.sq3Codes.SQLITE_LOCK_NONE===lockType + && fh.syncHandle ){ + wTimeStart('xUnlock'); + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; } - rc.push(v); + wTimeEnd(); } + storeAndNotify('xUnlock',rc); + mTimeEnd(); + }, + xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xWrite'); + let rc; + const fh = __openFiles[fid]; + wTimeStart('xWrite'); + try{ + affirmLocked('xWrite',fh); + affirmNotRO('xWrite', fh); + rc = ( + n === (await getSyncHandle(fh,'xWrite')) + .write(fh.sabView.subarray(0, n), + {at: Number(offset64)}) + ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + }catch(e){ + error("xWrite():",e,fh); + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); + } + await releaseImplicitLock(fh); + wTimeEnd(); + storeAndNotify('xWrite',rc); + mTimeEnd(); } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; + }/*vfsAsyncImpls*/; + + const initS11n = ()=>{ + /** + ACHTUNG: this code is 100% duplicated in the other half of this + proxy! The documentation is maintained in the "synchronous half". + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; + }; + state.s11n.deserialize = function(clear=false){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - - state.s11n.storeException = state.asyncS11nExceptions - ? ((priority,e)=>{ - if(priority<=state.asyncS11nExceptions){ - state.s11n.serialize([e.name,': ',e.message].join("")); - } - }) - : ()=>{}; - - return state.s11n; -}/*initS11n()*/; - -const waitLoop = async function f(){ - const opHandlers = Object.create(null); - for(let k of Object.keys(state.opIds)){ - const vi = vfsAsyncImpls[k]; - if(!vi) continue; - const o = Object.create(null); - opHandlers[state.opIds[k]] = o; - o.key = k; - o.f = vi; - } - /** - waitTime is how long (ms) to wait for each Atomics.wait(). - We need to wake up periodically to give the thread a chance - to do other things. If this is too high (e.g. 500ms) then - even two workers/tabs can easily run into locking errors. - */ - const waitTime = 100; - while(!flagAsyncShutdown){ - try { - if('timed-out'===Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, waitTime - )){ - await releaseImplicitLocks(); - continue; - } - const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); - Atomics.store(state.sabOPView, state.opIds.whichOp, 0); - const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); - const args = state.s11n.deserialize( - true /* clear s11n to keep the caller from confusing this with - an exception string written by the upcoming - operation */ - ) || []; - //warn("waitLoop() whichOp =",opId, hnd, args); - if(hnd.f) await hnd.f(...args); - else error("Missing callback for opId",opId); - }catch(e){ - error('in waitLoop():',e); - } - } -}; - -navigator.storage.getDirectory().then(function(d){ - const wMsg = (type)=>postMessage({type}); - state.rootDir = d; - self.onmessage = function({data}){ - switch(data.type){ - case 'opfs-async-init':{ - /* Receive shared state from synchronous partner */ - const opt = data.args; - state.littleEndian = opt.littleEndian; - state.asyncS11nExceptions = opt.asyncS11nExceptions; - state.verbose = opt.verbose ?? 1; - state.fileBufferSize = opt.fileBufferSize; - state.sabS11nOffset = opt.sabS11nOffset; - state.sabS11nSize = opt.sabS11nSize; - state.sabOP = opt.sabOP; - state.sabOPView = new Int32Array(state.sabOP); - state.sabIO = opt.sabIO; - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.opIds = opt.opIds; - state.sq3Codes = opt.sq3Codes; - state.opfsFlags = opt.opfsFlags; - Object.keys(vfsAsyncImpls).forEach((k)=>{ - if(!Number.isFinite(state.opIds[k])){ - toss("Maintenance required: missing state.opIds[",k,"]"); - } - }); - initS11n(); - metrics.reset(); - log("init state",state); - wMsg('opfs-async-inited'); - waitLoop(); - break; - } - case 'opfs-async-restart': - if(flagAsyncShutdown){ - warn("Restarting after opfs-async-shutdown. Might or might not work."); - flagAsyncShutdown = false; - waitLoop(); + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; } - break; - case 'opfs-async-metrics': - metrics.dump(); - break; + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; + + return state.s11n; + }/*initS11n()*/; + + const waitLoop = async function f(){ + const opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } + /** + waitTime is how long (ms) to wait for each Atomics.wait(). + We need to wake up periodically to give the thread a chance + to do other things. If this is too high (e.g. 500ms) then + even two workers/tabs can easily run into locking errors. + */ + const waitTime = 100; + while(!flagAsyncShutdown){ + try { + if('timed-out'===Atomics.wait( + state.sabOPView, state.opIds.whichOp, 0, waitTime + )){ + await releaseImplicitLocks(); + continue; + } + const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); + Atomics.store(state.sabOPView, state.opIds.whichOp, 0); + const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + const args = state.s11n.deserialize( + true /* clear s11n to keep the caller from confusing this with + an exception string written by the upcoming + operation */ + ) || []; + //warn("waitLoop() whichOp =",opId, hnd, args); + if(hnd.f) await hnd.f(...args); + else error("Missing callback for opId",opId); + }catch(e){ + error('in waitLoop():',e); + } } }; - wMsg('opfs-async-loaded'); -}).catch((e)=>error("error initializing OPFS asyncer:",e)); + + navigator.storage.getDirectory().then(function(d){ + state.rootDir = d; + self.onmessage = function({data}){ + switch(data.type){ + case 'opfs-async-init':{ + /* Receive shared state from synchronous partner */ + const opt = data.args; + state.littleEndian = opt.littleEndian; + state.asyncS11nExceptions = opt.asyncS11nExceptions; + state.verbose = opt.verbose ?? 1; + state.fileBufferSize = opt.fileBufferSize; + state.sabS11nOffset = opt.sabS11nOffset; + state.sabS11nSize = opt.sabS11nSize; + state.sabOP = opt.sabOP; + state.sabOPView = new Int32Array(state.sabOP); + state.sabIO = opt.sabIO; + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.opIds = opt.opIds; + state.sq3Codes = opt.sq3Codes; + state.opfsFlags = opt.opfsFlags; + Object.keys(vfsAsyncImpls).forEach((k)=>{ + if(!Number.isFinite(state.opIds[k])){ + toss("Maintenance required: missing state.opIds[",k,"]"); + } + }); + initS11n(); + metrics.reset(); + log("init state",state); + wPost('opfs-async-inited'); + waitLoop(); + break; + } + case 'opfs-async-restart': + if(flagAsyncShutdown){ + warn("Restarting after opfs-async-shutdown. Might or might not work."); + flagAsyncShutdown = false; + waitLoop(); + } + break; + case 'opfs-async-metrics': + metrics.dump(); + break; + } + }; + wPost('opfs-async-loaded'); + }).catch((e)=>error("error initializing OPFS asyncer:",e)); +}/*installAsyncProxy()*/; +if(!self.SharedArrayBuffer){ + wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", + "The server must emit the COOP/COEP response headers to enable that."); +}else if(!self.Atomics){ + wPost('opfs-unavailable', "Missing Atomics API.", + "The server must emit the COOP/COEP response headers to enable that."); +}else if(!self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + wPost('opfs-unavailable',"Missing required OPFS APIs."); +}else{ + installAsyncProxy(self); +} diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 5bb5959396..2fc6752694 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1784,13 +1784,12 @@ self.sqlite3InitModule = sqlite3InitModule; .t({ name: 'OPFS sanity checks', test: async function(sqlite3){ - const opfs = sqlite3.opfs; const filename = 'sqlite3-tester1.db'; const pVfs = capi.sqlite3_vfs_find('opfs'); T.assert(pVfs); const unlink = (fn=filename)=>wasm.sqlite3_wasm_vfs_unlink(pVfs,fn); unlink(); - let db = new opfs.OpfsDb(filename); + let db = new sqlite3.oo1.OpfsDb(filename); try { db.exec([ 'create table p(a);', @@ -1798,7 +1797,7 @@ self.sqlite3InitModule = sqlite3InitModule; ]); T.assert(3 === db.selectValue('select count(*) from p')); db.close(); - db = new opfs.OpfsDb(filename); + db = new sqlite3.oo1.OpfsDb(filename); db.exec('insert into p(a) values(4),(5),(6)'); T.assert(6 === db.selectValue('select count(*) from p')); }finally{ @@ -1806,8 +1805,9 @@ self.sqlite3InitModule = sqlite3InitModule; unlink(); } - if(1){ + if(sqlite3.opfs){ // Sanity-test sqlite3_wasm_vfs_create_file()... + const opfs = sqlite3.opfs; const fSize = 1379; let sh; try{ @@ -1824,20 +1824,20 @@ self.sqlite3InitModule = sqlite3InitModule; if(sh) await sh.close(); unlink(); } - } - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } } }/*OPFS sanity checks*/) ;/* end OPFS tests */ diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index e19f6a8da6..d5aa51dc93 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -33,9 +33,10 @@ with unlock-asap=0-1.

    Achtung: if it does not start to do anything within a couple of - seconds, check the dev console: Chrome often fails with "cannot allocate - WasmMemory" at startup. Closing and re-opening the tab usually resolves - it. + seconds, check the dev console: Chrome sometimes fails to load + the wasm module due to "cannot allocate WasmMemory." Closing and + re-opening the tab usually resolves it, but sometimes restarting + the browser is required.

    diff --git a/manifest b/manifest index 01aaf2a316..4dc42b8b8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexplicit\swarning\sabout\sthe\scurrent\sAPI-instability\sof\sthe\ssqlite3.opfs\snamespace,\swhich\smay\sneed\sto\sbe\seliminated\sbased\son\sre-thinking\sof\show\sthe\sOPFS\ssqlite3_vfs\sis\sregistered.\sComment\schanges\sonly\s-\sno\scode. -D 2022-11-29T02:23:12.943 +C Internal\srestructuring\sof\sthe\sOPFS\ssqlite3_vfs\sin\sorder\sto\sfacilitate\scertain\sexperimentation\sand\simprove\serror\sreporting/hints\sif\sit\scannot\sbe\sactivated.\sDeprecate\sthe\sname\ssqlite3.opfs.OpfsDb,\spreferring\ssqlite3.oo1.OpfsDb\sfor\sconsistency\swith\sJsStorageDb\sand\sany\sfuture\sDB\ssubclasses. +D 2022-11-29T05:25:08.036 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,11 +505,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js 3cdae7e98c500f89f9468a260e2a0e1b528c845a107bf72d368e5222769214d3 +F ext/wasm/api/sqlite3-api-opfs.js 583650ffdc1452496df6b9459d018fa2aede221ae6ea0cbbbe83bd2e1bdba966 F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 798383f6b46fd5dac122d6e35962d25b10401ddb825b5c66df1d21e6b1d8aacc +F ext/wasm/api/sqlite3-opfs-async-proxy.js b5dd7eda8e74e07453457925a0dd793d7785da720954e0e37e847c5c6e4d9526 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,8 +554,8 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js 3b91f192c159088004fba6fe3441edea58421a8b88bccf3dd20978a077648d19 -F ext/wasm/tests/opfs/concurrency/index.html e8fec75ea6eddc600c8a382da7ea2579feece2263a2fb4417f2cf3e9d451744c +F ext/wasm/tester1.c-pp.js a4b6a165aafcd3b86118efaec6b47c70fbb6c64b5ab86d21ca8c250d42617dfa +F ext/wasm/tests/opfs/concurrency/index.html 2b1cda51d6c786102875a28eba22f0da3eecb732a5e677b0d1ecdb53546d1a62 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46cdd3637d6a206ad2bcf8653cc6f2c7a886a16cc7685c45967938609941a755 -R 3870d04bfd54da096e986662fe29b1c8 +P 0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 +R 115b7898d7b2ce79a9261f36a9b959d1 U stephan -Z ad4a8c5f45a34a10f588c0c6dc455846 +Z f4ff31d5e2499971cf67cb62dbdd0ac3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 361de4178e..fb6fdac183 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 \ No newline at end of file +0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 \ No newline at end of file From ceedef888f38d172c203b1c7bf5265f4ee608640 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Nov 2022 06:09:32 +0000 Subject: [PATCH 099/282] Minor internal cleanups and docs in the OPFS sqlite3_vfs. FossilOrigin-Name: 61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 --- ext/wasm/api/sqlite3-api-opfs.js | 4 +++- ext/wasm/api/sqlite3-opfs-async-proxy.js | 27 ++++++++++++++++++------ manifest | 14 ++++++------ manifest.uuid | 2 +- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index cef15305d4..de7500afa3 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -290,7 +290,9 @@ const installOpfsVfs = function callee(options){ The size of the block in our SAB for serializing arguments and result values. Needs to be large enough to hold serialized values of any of the proxied APIs. Filenames are the largest - part but are limited to opfsVfs.$mxPathname bytes. + part but are limited to opfsVfs.$mxPathname bytes. We also + store exceptions there, so it needs to be long enough to hold + a reasonably long exception string. */ state.sabS11nSize = opfsVfs.$mxPathname * 2; /** diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 1ba6e9bdbb..a80eeb2903 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -239,15 +239,26 @@ const installAsyncProxy = function(self){ between locking-related failures and other types, noting that we cannot currently do so because createSyncAccessHandle() does not define its exceptions in the required level of detail. + + 2022-11-29: according to: + + https://github.com/whatwg/fs/pull/21 + + NoModificationAllowedError will be the standard exception thrown + when acquisition of a sync access handle fails due to a locking + error. As of this writing, that error type is not visible in the + dev console in Chrome v109, nor is it documented in MDN, but an + error with that "name" property is being thrown from the OPFS + layer. */ class GetSyncHandleError extends Error { constructor(errorObject, ...msg){ - super(); - this.error = errorObject; - this.message = [ - ...msg, ': Original exception ['+errorObject.name+']:', + super([ + ...msg, ': '+errorObject.name+':', errorObject.message - ].join(' '); + ].join(' '), { + cause: errorObject + }); this.name = 'GetSyncHandleError'; } }; @@ -259,8 +270,12 @@ const installAsyncProxy = function(self){ distinguish that from other errors. This approach is highly questionable. + + Note that even if we return SQLITE_IOERR_LOCK from here, + it bubbles up to the client as a plain I/O error. */ - return (e instanceof GetSyncHandleError) + return (e instanceof GetSyncHandleError + && e.cause.name==='NoModificationAllowedError') ? state.sq3Codes.SQLITE_IOERR_LOCK : rc; }else{ diff --git a/manifest b/manifest index 4dc42b8b8c..51b58eba3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Internal\srestructuring\sof\sthe\sOPFS\ssqlite3_vfs\sin\sorder\sto\sfacilitate\scertain\sexperimentation\sand\simprove\serror\sreporting/hints\sif\sit\scannot\sbe\sactivated.\sDeprecate\sthe\sname\ssqlite3.opfs.OpfsDb,\spreferring\ssqlite3.oo1.OpfsDb\sfor\sconsistency\swith\sJsStorageDb\sand\sany\sfuture\sDB\ssubclasses. -D 2022-11-29T05:25:08.036 +C Minor\sinternal\scleanups\sand\sdocs\sin\sthe\sOPFS\ssqlite3_vfs. +D 2022-11-29T06:09:32.898 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,11 +505,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js 583650ffdc1452496df6b9459d018fa2aede221ae6ea0cbbbe83bd2e1bdba966 +F ext/wasm/api/sqlite3-api-opfs.js ee97811004fd0da9311527293b519543fbb0c6b0c9beaa9b6704400bfd74d150 F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js b5dd7eda8e74e07453457925a0dd793d7785da720954e0e37e847c5c6e4d9526 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 4b8973711ae4825bb047da932b9046b4e8f678fcea759158caebc5821a655ac7 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cb2fd14179397051a25d066256a553fc198656d5668c7010c016f2b8f495bf4 -R 115b7898d7b2ce79a9261f36a9b959d1 +P 0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 +R e8239b21ab627d1eba84cc5be45ed7a1 U stephan -Z f4ff31d5e2499971cf67cb62dbdd0ac3 +Z 1147c5417905a9e8323d8798a4fd57dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fb6fdac183..85d7059802 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 \ No newline at end of file +61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 \ No newline at end of file From a302e6847643c24d331e1f1be5275bf8ec38be46 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Nov 2022 06:56:09 +0000 Subject: [PATCH 100/282] Rename one instance of opfs.OpfsDb to oo1.OpfsDb, as per [0c5c51f4fb04]. FossilOrigin-Name: 75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 --- ext/wasm/tests/opfs/concurrency/worker.js | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 95fefa1954..5d28bedee0 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -46,7 +46,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.opfs.OpfsDb({ + db = new sqlite3.oo1.OpfsDb({ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c' }); diff --git a/manifest b/manifest index 51b58eba3a..cef62cf1b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\scleanups\sand\sdocs\sin\sthe\sOPFS\ssqlite3_vfs. -D 2022-11-29T06:09:32.898 +C Rename\sone\sinstance\sof\sopfs.OpfsDb\sto\soo1.OpfsDb,\sas\sper\s[0c5c51f4fb04]. +D 2022-11-29T06:56:09.516 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -557,7 +557,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js a4b6a165aafcd3b86118efaec6b47c70fbb6c64b5ab86d21ca8c250d42617dfa F ext/wasm/tests/opfs/concurrency/index.html 2b1cda51d6c786102875a28eba22f0da3eecb732a5e677b0d1ecdb53546d1a62 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a -F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450 +F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c5c51f4fb04a4b90c50ec9704cfea9a3fb7d7d0ee55c1b0d4476129188217a6 -R e8239b21ab627d1eba84cc5be45ed7a1 +P 61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 +R ccbd533ba6a1af3ea32f1c34f9412b94 U stephan -Z 1147c5417905a9e8323d8798a4fd57dd +Z 5d4b5205e72d1fad1817c77ba7b7ffb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 85d7059802..08f00fa593 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 \ No newline at end of file +75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 \ No newline at end of file From 3c3896c9d68f1003edbd02f0657e875ae00dff5e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Nov 2022 11:28:45 +0000 Subject: [PATCH 101/282] Fix legacy build system main.mk to include basexx.c in testfixture builds. FossilOrigin-Name: 8c9200b7e156206c4270ff60631c57d5898e2769f7f68294ce652bfec4cfa7c2 --- main.mk | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/main.mk b/main.mk index 76c62fcecd..ccd5b6f5b0 100644 --- a/main.mk +++ b/main.mk @@ -363,6 +363,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/misc/basexx.c \ $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/cksumvfs.c \ $(TOP)/ext/misc/closure.c \ diff --git a/manifest b/manifest index cef62cf1b4..becd51a6d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\sone\sinstance\sof\sopfs.OpfsDb\sto\soo1.OpfsDb,\sas\sper\s[0c5c51f4fb04]. -D 2022-11-29T06:56:09.516 +C Fix\slegacy\sbuild\ssystem\smain.mk\sto\sinclude\sbasexx.c\sin\stestfixture\sbuilds. +D 2022-11-29T11:28:45.200 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -563,7 +563,7 @@ F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c7974 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk fd90f1bd90bd4070c9af7e9c8396bd0cf4208a0186c8d18fa6e2609dff447d5d +F main.mk 5ce166d63fb99df9821f262fb1777fd5eb7b6c9396e867a6b0b2761f35f8238a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 61799b05ff232c2ac349169c27bfe7f8d9277366093b0c9dd2739828993b3066 -R ccbd533ba6a1af3ea32f1c34f9412b94 -U stephan -Z 5d4b5205e72d1fad1817c77ba7b7ffb7 +P 75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 +R 5b4d2911b428490bdd01585b55a3d462 +U dan +Z d91547f2ddaedf2a7b8258684ff0e02a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 08f00fa593..989d35069a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 \ No newline at end of file +8c9200b7e156206c4270ff60631c57d5898e2769f7f68294ce652bfec4cfa7c2 \ No newline at end of file From 7741f3457a3c3e6785d987fe1be46351759851a4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 29 Nov 2022 17:52:04 +0000 Subject: [PATCH 102/282] Change the handling of hwtime.h to make it easier to compile performance measurement builds that make use of hwtime.h. This should not affect productions builds. FossilOrigin-Name: f64a224244743ab121371abd516fccbfc93c110e0952211764bd1b217e792c1b --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/os_common.h | 6 ------ src/sqliteInt.h | 4 ++++ src/util.c | 7 +++++++ src/vdbe.c | 10 ---------- 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index becd51a6d3..c06c8f707c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\slegacy\sbuild\ssystem\smain.mk\sto\sinclude\sbasexx.c\sin\stestfixture\sbuilds. -D 2022-11-29T11:28:45.200 +C Change\sthe\shandling\sof\shwtime.h\sto\smake\sit\seasier\sto\scompile\sperformance\nmeasurement\sbuilds\sthat\smake\suse\sof\shwtime.h.\s\sThis\sshould\snot\saffect\nproductions\sbuilds. +D 2022-11-29T17:52:04.105 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/mutex_w32.c caa50e1c0258ac4443f52e00fe8aaea73b6d0728bd8856bedfff822cae4185 F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 F src/os.c 81c9c1c52eab711e27e33fd51fe5788488d3a02bc1a71439857abbee5d0d2c97 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 -F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 +F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 0e59600d25b72034c7666b8b7dcc527f039b5d9c16f24a7eca4c08c66f63c364 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c @@ -649,7 +649,7 @@ F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063 F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f +F src/sqliteInt.h a8d507577ca5e6581abb65c30497f7b8cfc7c9feb0d768a081c1e4a6adfcd061 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -714,9 +714,9 @@ F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c 0be191521ff6d2805995f4910f0b6231b42843678b2efdc1abecaf39929a673f +F src/util.c 4e42c338d982e3f139004467bf0f2e9d679d519c7c75718fcf4ff1811401b03c F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c c2d6d0c0c343d8ebffef996c73cbb69e337225f757fea7fe5c0e3ea14662adec +F src/vdbe.c 00648bd76fb2145c2d890312112bda446569da64ca70aaf1f2282cc6c2b22d14 F src/vdbe.h 58675f47dcf3105bab182c3ad3726efd60ffd003e954386904ac9107d0d2b743 F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f F src/vdbeapi.c 1e8713d0b653acb43cd1bdf579c40e005c4844ea90f414f065946a83db3c27fb @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75b04c9b302ec66749d8f072a49e61de9640b868bec99eadeb5b4bdef354c336 -R 5b4d2911b428490bdd01585b55a3d462 -U dan -Z d91547f2ddaedf2a7b8258684ff0e02a +P 8c9200b7e156206c4270ff60631c57d5898e2769f7f68294ce652bfec4cfa7c2 +R 427322317ebe04d9616d118c250b6e39 +U drh +Z a5571007571b0b63420d1f8b08046ddf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 989d35069a..94da90e067 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c9200b7e156206c4270ff60631c57d5898e2769f7f68294ce652bfec4cfa7c2 \ No newline at end of file +f64a224244743ab121371abd516fccbfc93c110e0952211764bd1b217e792c1b \ No newline at end of file diff --git a/src/os_common.h b/src/os_common.h index 1ed4d7a8e1..5b532af0ac 100644 --- a/src/os_common.h +++ b/src/os_common.h @@ -35,12 +35,6 @@ */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -#include "hwtime.h" - static sqlite_uint64 g_start; static sqlite_uint64 g_elapsed; #define TIMER_START g_start=sqlite3Hwtime() diff --git a/src/sqliteInt.h b/src/sqliteInt.h index db65082bbc..7d7e5b56f0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -5535,4 +5535,8 @@ const char **sqlite3CompileOptions(int *pnOpt); int sqlite3KvvfsInit(void); #endif +#if defined(VDBE_PROFILE) || defined(SQLITE_PERFORMANCE_TRACE) +sqlite3_uint64 sqlite3Hwtime(void); +#endif + #endif /* SQLITEINT_H */ diff --git a/src/util.c b/src/util.c index 32e9c27786..7a58fc8764 100644 --- a/src/util.c +++ b/src/util.c @@ -1713,3 +1713,10 @@ int sqlite3VListNameToNum(VList *pIn, const char *zName, int nName){ }while( i Date: Tue, 29 Nov 2022 18:00:01 +0000 Subject: [PATCH 103/282] OPFS speedtest1: hide a currently-broken/to-fix WASMFS-build link. FossilOrigin-Name: cde95d382f8debcbca27c4aaf08470ffda35ab10d723a887786669367590ad3c --- ext/wasm/speedtest1-worker.html | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index 5638ce3b50..9fcd29ea72 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -40,7 +40,7 @@ → speedtest1 - → speedtest1-wasmfs → Date: Tue, 29 Nov 2022 18:28:40 +0000 Subject: [PATCH 104/282] sqlite3.oo1.OpfsDb: default to journal_mode=persist, as current tests show it to be marginally faster than truncate/delete in Chrome v109. Also increase default busy_timeout from 2 seconds to 3, admittedly on a whim. FossilOrigin-Name: d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 --- ext/wasm/api/sqlite3-api-opfs.js | 10 ++++++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index de7500afa3..3099cb486d 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -1216,12 +1216,14 @@ const installOpfsVfs = function callee(options){ sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, [ - /* Truncate journal mode is faster than delete or wal for - this vfs, per speedtest1. */ - "pragma journal_mode=truncate;", + /* Truncate journal mode is faster than delete for + this vfs, per speedtest1. That gap seems to have closed with + Chome version 108 or 109, but "persist" is very roughly 5-6% + faster than truncate in initial tests. */ + "pragma journal_mode=persist;", /* Set a default busy-timeout handler to help OPFS dbs deal with multi-tab/multi-worker contention. */ - "pragma busy_timeout=2000;", + "pragma busy_timeout=3000;", /* This vfs benefits hugely from cache on moderate/large speedtest1 --size 50 and --size 100 workloads. We currently diff --git a/manifest b/manifest index 0721c80593..e7c5ce5e81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sspeedtest1:\shide\sa\scurrently-broken/to-fix\sWASMFS-build\slink. -D 2022-11-29T18:00:01.024 +C sqlite3.oo1.OpfsDb:\sdefault\sto\sjournal_mode=persist,\sas\scurrent\stests\sshow\sit\sto\sbe\smarginally\sfaster\sthan\struncate/delete\sin\sChrome\sv109.\sAlso\sincrease\sdefault\sbusy_timeout\sfrom\s2\sseconds\sto\s3,\sadmittedly\son\sa\swhim. +D 2022-11-29T18:28:40.756 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,7 +505,7 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js ee97811004fd0da9311527293b519543fbb0c6b0c9beaa9b6704400bfd74d150 +F ext/wasm/api/sqlite3-api-opfs.js 2f99c9b2c3c0f53bf40153de20db350378deed825ec0d5cf8f449e9e0081f08b F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f64a224244743ab121371abd516fccbfc93c110e0952211764bd1b217e792c1b -R 9e4ddb7294c8c5e99856ae19fd524910 +P cde95d382f8debcbca27c4aaf08470ffda35ab10d723a887786669367590ad3c +R f012421262a97adb4d00809196a366a2 U stephan -Z 303f08b9a1a9cd719a67e29d0dc3ab85 +Z e12ff5fd260034aa46c1df81ecc7ab20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5ae68dd9ad..43e23def0d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cde95d382f8debcbca27c4aaf08470ffda35ab10d723a887786669367590ad3c \ No newline at end of file +d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 \ No newline at end of file From d0945f4638f342daf4c84ab5a03449137435dd36 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 03:08:50 +0000 Subject: [PATCH 105/282] Doc and logging text tweaks in the OPFS async proxy and test app. FossilOrigin-Name: 7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 13 +++++++------ ext/wasm/tests/opfs/concurrency/index.html | 4 +--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index a80eeb2903..da75f139c9 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -287,11 +287,12 @@ const installAsyncProxy = function(self){ handle object (which must be a valid handle object, as created by xOpen()), lazily opening it if needed. - In order to help alleviate cross-tab contention for a dabase, - if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to 3 times. If acquisition - still fails at that point it will give up and propagate the - exception. + In order to help alleviate cross-tab contention for a dabase, if + an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to some fixed number of + times. If acquisition still fails at that point it will give up + and propagate the exception. Client-level code will see that as + an I/O error. */ const getSyncHandle = async (fh,opName)=>{ if(!fh.syncHandle){ @@ -323,7 +324,7 @@ const installAsyncProxy = function(self){ 'in',performance.now() - t,'ms'); if(!fh.xLock){ __implicitLocks.add(fh.fid); - log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); + log("Acquired implicit lock for",opName+"()",fh.fid,fh.filenameAbs); } } return fh.syncHandle; diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index d5aa51dc93..9a826c6e13 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -18,9 +18,7 @@

    OPFS concurrency tester using multiple independent Workers. - Disclaimer: concurrency in OPFS is currently a pain point - and timing/concurrency mitigation in this environment is - highly unpredictable! + Disclaimer: concurrency in OPFS is currently a pain point!

    URL flags: pass a number of workers using diff --git a/manifest b/manifest index e7c5ce5e81..ecfa59693e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C sqlite3.oo1.OpfsDb:\sdefault\sto\sjournal_mode=persist,\sas\scurrent\stests\sshow\sit\sto\sbe\smarginally\sfaster\sthan\struncate/delete\sin\sChrome\sv109.\sAlso\sincrease\sdefault\sbusy_timeout\sfrom\s2\sseconds\sto\s3,\sadmittedly\son\sa\swhim. -D 2022-11-29T18:28:40.756 +C Doc\sand\slogging\stext\stweaks\sin\sthe\sOPFS\sasync\sproxy\sand\stest\sapp. +D 2022-11-30T03:08:50.289 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -509,7 +509,7 @@ F ext/wasm/api/sqlite3-api-opfs.js 2f99c9b2c3c0f53bf40153de20db350378deed825ec0d F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 4b8973711ae4825bb047da932b9046b4e8f678fcea759158caebc5821a655ac7 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 315582a4fe3b87ceaff69ad4a67d68a350b8014ac94d6a584d09ea922453c983 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js a4b6a165aafcd3b86118efaec6b47c70fbb6c64b5ab86d21ca8c250d42617dfa -F ext/wasm/tests/opfs/concurrency/index.html 2b1cda51d6c786102875a28eba22f0da3eecb732a5e677b0d1ecdb53546d1a62 +F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cde95d382f8debcbca27c4aaf08470ffda35ab10d723a887786669367590ad3c -R f012421262a97adb4d00809196a366a2 +P d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 +R 05044488394ac3d7b358270198570ce0 U stephan -Z e12ff5fd260034aa46c1df81ecc7ab20 +Z cb806f606aa315e69c2e3fceeb852a45 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 43e23def0d..12e29fe3c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 \ No newline at end of file +7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b \ No newline at end of file From ad4f7828153e6b80c0fceabb1a9ece702172b836 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 05:27:36 +0000 Subject: [PATCH 106/282] Refactor a significant chunk of the OPFS sqlite3_vfs init code into sqlite3.VfsHelper, and internal-use-only API encapsulating code relevant to creating new VFSes in JS. Intended to assist in pending experimentation with an alternative OPFS VFS. FossilOrigin-Name: e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 --- ext/wasm/GNUmakefile | 1 + ext/wasm/api/README.md | 27 ++-- ext/wasm/api/extern-post-js.js | 12 +- ext/wasm/api/sqlite3-api-cleanup.js | 2 - ext/wasm/api/sqlite3-api-opfs.js | 108 ++------------ ext/wasm/api/sqlite3-api-prologue.js | 17 ++- ext/wasm/api/sqlite3-vfs-helper.js | 209 +++++++++++++++++++++++++++ ext/wasm/tester1.c-pp.js | 3 + manifest | 25 ++-- manifest.uuid | 2 +- 10 files changed, 280 insertions(+), 126 deletions(-) create mode 100644 ext/wasm/api/sqlite3-vfs-helper.js diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 38c1cdaca8..3c6b3c9a15 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -287,6 +287,7 @@ sqlite3-api.jses += $(dir.common)/whwasmutil.js sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js sqlite3-api.jses += $(sqlite3-api-build-version.js) +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index d33c4a5831..37d986584e 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -23,13 +23,13 @@ The overall idea is that the following files get concatenated together, in the listed order, the resulting file is loaded by a browser client: -- `sqlite3-api-prologue.js`\ +- **`sqlite3-api-prologue.js`**\ Contains the initial bootstrap setup of the sqlite3 API objects. This is exposed as a function, rather than objects, so that the next step can pass in a config object which abstracts away parts of the WASM environment, to facilitate plugging it in to arbitrary WASM toolchains. -- `../common/whwasmutil.js`\ +- **`../common/whwasmutil.js`**\ A semi-third-party collection of JS/WASM utility code intended to replace much of the Emscripten glue. The sqlite3 APIs internally use these APIs instead of their Emscripten counterparts, in order to be @@ -38,47 +38,52 @@ browser client: toolchains. It is "semi-third-party" in that it was created in order to support this tree but is standalone and maintained together with... -- `../jaccwabyt/jaccwabyt.js`\ +- **`../jaccwabyt/jaccwabyt.js`**\ Another semi-third-party API which creates bindings between JS and C structs, such that changes to the struct state from either JS or C are visible to the other end of the connection. This is also an independent spinoff project, conceived for the sqlite3 project but maintained separately. -- `sqlite3-api-glue.js`\ +- **`sqlite3-api-glue.js`**\ Invokes functionality exposed by the previous two files to flesh out low-level parts of `sqlite3-api-prologue.js`. Most of these pieces related to the `sqlite3.capi.wasm` object. -- `sqlite3-api-build-version.js`\ +- **`sqlite3-api-build-version.js`**\ Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. -- `sqlite3-api-oo1.js`\ +- **`sqlite3-vfs-helper.js`**\ + This internal-use-only file installs `sqlite3.VfsHelper` for use by + `sqlite3-api-*.js` files which create `sqlite3_vfs` implemenations. + `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after + the library is finished initializing. +- **`sqlite3-api-oo1.js`**\ Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. That said, it is not a "required component" and can be elided from builds which do not want it. -- `sqlite3-api-worker1.js`\ +- **`sqlite3-api-worker1.js`**\ A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - `sqlite3-worker1.js`\ + - **`sqlite3-worker1.js`**\ Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module and runs the Worker #1 API which is implemented in `sqlite3-api-worker1.js`. - - `sqlite3-worker1-promiser.js`\ + - **`sqlite3-worker1-promiser.js`**\ Is likewise not part of the amalgamated sources and provides a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- `sqlite3-api-opfs.js`\ +- **`sqlite3-api-opfs.js`**\ is an sqlite3 VFS implementation which supports Google Chrome's Origin-Private FileSystem (OPFS) as a storage layer to provide persistent storage for database files in a browser. It requires... - - `sqlite3-opfs-async-proxy.js`\ + - **`sqlite3-opfs-async-proxy.js`**\ is the asynchronous backend part of the OPFS proxy. It speaks directly to the (async) OPFS API and channels those results back to its synchronous counterpart. This file, because it must be diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index cace6ed51c..2258697944 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -60,7 +60,7 @@ const toExportForES6 = initModuleState.sqlite3Dir = li.join('/') + '/'; } - self.sqlite3InitModule = (...args)=>{ + self.sqlite3InitModule = function ff(...args){ //console.warn("Using replaced sqlite3InitModule()",self.location); return originalInit(...args).then((EmscriptenModule)=>{ if(self.window!==self && @@ -76,10 +76,12 @@ const toExportForES6 = Emscripten details. */ return EmscriptenModule; } - EmscriptenModule.sqlite3.scriptInfo = initModuleState; - //console.warn("sqlite3.scriptInfo =",EmscriptenModule.sqlite3.scriptInfo); - const f = EmscriptenModule.sqlite3.asyncPostInit; - delete EmscriptenModule.sqlite3.asyncPostInit; + const s = EmscriptenModule.sqlite3; + s.scriptInfo = initModuleState; + //console.warn("sqlite3.scriptInfo =",s.scriptInfo); + if(ff.__isUnderTest) s.__isUnderTest = true; + const f = s.asyncPostInit; + delete s.asyncPostInit; return f(); }).catch((e)=>{ console.error("Exception loading sqlite3 module:",e); diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index bef4d91d70..0ec0fbfbe3 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -58,8 +58,6 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build self.S = sqlite3; } - /* Clean up temporary references to our APIs... */ - delete sqlite3.util /* arguable, but these are (currently) internal-use APIs */; Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to pass the sqlite3 object off to the client. */; }else{ diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 3099cb486d..5eb3b22b37 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -129,7 +129,7 @@ const installOpfsVfs = function callee(options){ const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); const error = (...args)=>logImpl(0, ...args); - const toss = function(...args){throw new Error(args.join(' '))}; + const toss = sqlite3.util.toss; const capi = sqlite3.capi; const wasm = sqlite3.wasm; const sqlite3_vfs = capi.sqlite3_vfs; @@ -191,6 +191,8 @@ const installOpfsVfs = function callee(options){ s.count = s.time = 0; } }/*metrics*/; + const opfsVfs = new sqlite3_vfs(); + const opfsIoMethods = new sqlite3_io_methods(); const promiseReject = function(err){ opfsVfs.dispose(); return promiseReject_(err); @@ -213,8 +215,6 @@ const installOpfsVfs = function callee(options){ ? new sqlite3_vfs(pDVfs) : null /* dVfs will be null when sqlite3 is built with SQLITE_OS_OTHER. */; - const opfsVfs = new sqlite3_vfs(); - const opfsIoMethods = new sqlite3_io_methods(); opfsVfs.$iVersion = 2/*yes, two*/; opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; opfsVfs.$mxPathname = 1024/*sure, why not?*/; @@ -633,77 +633,6 @@ const installOpfsVfs = function callee(options){ */ const __openFiles = Object.create(null); - /** - Installs a StructBinder-bound function pointer member of the - given name and function in the given StructType target object. - It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws - on the slightest hint of error (e.g. tgt is-not-a StructType, - name does not map to a struct-bound member, etc.). - - Returns a proxy for this function which is bound to tgt and takes - 2 args (name,func). That function returns the same thing, - permitting calls to be chained. - - If called with only 1 arg, it has no side effects but returns a - func with the same signature as described above. - */ - const installMethod = function callee(tgt, name, func){ - if(!(tgt instanceof sqlite3.StructBinder.StructType)){ - toss("Usage error: target object is-not-a StructType."); - } - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - callee.removeFuncList = function(){ - if(this.ondispose.__removeFuncList){ - this.ondispose.__removeFuncList.forEach( - (v,ndx)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - } - /* else it's a descriptive label for the next number in - the list. */ - } - ); - delete this.ondispose.__removeFuncList; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name); - if(sigN.length<2){ - toss("Member",name," is not a function pointer. Signature =",sigN); - } - const memKey = tgt.memberKey(name); - const fProxy = 0 - /** This middle-man proxy is only for use during development, to - confirm that we always pass the proper number of - arguments. We know that the C-level code will always use the - correct argument count. */ - ? callee.argcProxy(func, sigN) - : func; - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose) tgt.ondispose = []; - if(!tgt.ondispose.__removeFuncList){ - tgt.ondispose.push('ondispose.__removeFuncList handler', - callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; - } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; - const opTimer = Object.create(null); opTimer.op = undefined; opTimer.start = undefined; @@ -729,7 +658,11 @@ const installOpfsVfs = function callee(options){ success) release a sync-handle for it, but doing so would involve an inherent race condition. For the time being, pending a better solution, we simply report whether the - given pFile instance has a lock. + given pFile is open. + + FIXME: we need to ask the async half whether a lock is + held, as it's possible (since long after this method was + implemented) that we do not hold a lock on an OPFS file. */ const f = __openFiles[pFile]; wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32'); @@ -948,14 +881,6 @@ const installOpfsVfs = function callee(options){ }; } - /* Install the vfs/io_methods into their C-level shared instances... */ - for(let k of Object.keys(ioSyncWrappers)){ - installMethod(opfsIoMethods, k, ioSyncWrappers[k]); - } - for(let k of Object.keys(vfsSyncWrappers)){ - installMethod(opfsVfs, k, vfsSyncWrappers[k]); - } - /** Expects an OPFS file path. It gets resolved, such that ".." components are properly expanded, and returned. If the 2nd arg @@ -1095,8 +1020,9 @@ const installOpfsVfs = function callee(options){ Irrevocably deletes _all_ files in the current origin's OPFS. Obviously, this must be used with great caution. It may throw an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which it will throw - are not documented (so we cannot tell you what they are). + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). */ opfsUtil.rmfr = async function(){ const dir = opfsUtil.rootDirectory, opt = {recurse: true}; @@ -1320,14 +1246,10 @@ const installOpfsVfs = function callee(options){ and has finished initializing, so the real work can begin...*/ try { - const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, 0); - if(rc){ - toss("sqlite3_vfs_register(OPFS) failed with rc",rc); - } - if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){ - toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); - } - capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); + sqlite3.VfsHelper.installVfs({ + io: {struct: opfsIoMethods, methods: ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: vfsSyncWrappers} + }); state.sabOPView = new Int32Array(state.sabOP); state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 726f47e18d..6d39581965 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -612,8 +612,11 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( isBindableTypedArray, isInt32, isSQLableTypedArray, isTypedArray, typedArrayToString, - isUIThread: ()=>'undefined'===typeof WorkerGlobalScope, + isUIThread: ()=>(self.window===self && !!self.document), + // is this true for ESM?: 'undefined'===typeof WorkerGlobalScope isSharedTypedArray, + toss: function(...args){throw new Error(args.join(' '))}, + toss3, typedArrayPart }; @@ -1460,7 +1463,17 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( //let p = lip.shift(); //while(lip.length) p = p.then(lip.shift()); //return p.then(()=>sqlite3); - return Promise.all(lip).then(()=>sqlite3); + return Promise.all(lip).then(()=>{ + if(!sqlite3.__isUnderTest){ + /* Delete references to internal-only APIs which are used by + some initializers. Retain them when running in test mode + so that we can add tests for them. */ + delete sqlite3.util; + delete sqlite3.VfsHelper; + delete sqlite3.StructBinder; + } + return sqlite3; + }); }, /** scriptInfo ideally gets injected into this object by the diff --git a/ext/wasm/api/sqlite3-vfs-helper.js b/ext/wasm/api/sqlite3-vfs-helper.js new file mode 100644 index 0000000000..9a15dd85fd --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-helper.js @@ -0,0 +1,209 @@ +/* +** 2022-11-30 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +*/ + +/** + This file installs sqlite.VfsHelper, an object which exists + to assist in the creation of JavaScript implementations of + sqlite3_vfs. It is NOT part of the public API, and is an + internal implemenation detail for use in this project's + own development of VFSes. It may be exposed to clients + at some point, provided there is value in doing so. +*/ +'use strict'; +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss; + const vh = Object.create(null); + + /** + Does nothing more than holds a permanent reference to each + argument. This is useful in some cases to ensure that, e.g., a + custom sqlite3_io_methods instance does not get + garbage-collected. + + Returns this object. + */ + vh.holdReference = function(...args){ + for(const v of args) this.refs.add(v); + return vh; + }.bind({refs: new Set}); + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error, e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc. + + If applyArgcCheck is true then each method gets wrapped in a + proxy which asserts that it is passed the expected number of + arguments, throwing if the argument count does not match + expectations. That is only recommended for dev-time usage for + sanity checking. Once a VFS implementation is known to be + working, it is a given that the C API will never call it with the + wrong argument count. + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + + If tgt.ondispose is set before this is called then it _must_ + be an array, to which this function will append entries. + */ + vh.installMethod = function callee(tgt, name, func, + applyArgcCheck=callee.installMethodArgcCheck){ + if(!(tgt instanceof sqlite3.StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + /* An ondispose() callback for use with + sqlite3.StructBinder-created types. */ + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + const fProxy = applyArgcCheck + /** This middle-man proxy is only for use during development, to + confirm that we always pass the proper number of + arguments. We know that the C-level code will always use the + correct argument count. */ + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + }/*installMethod*/; + vh.installMethod.installMethodArgcCheck = false; + + /** + Installs methods into the given StructType-type object. Each + entry in the given methods object must map to a known member of + the given StructType, else an exception will be triggered. + See installMethod() for more details, including the semantics + of the 3rd argument. + + On success, passes its first argument to holdRefence() and + returns this object. Throws on error. + */ + vh.installMethods = function(structType, methods, + applyArgcCheck=vh.installMethod.installMethodArgcCheck){ + for(const k of Object.keys(methods)){ + vh.installMethod(structType, k, methods[k], applyArgcCheck); + } + return vh.holdReference(structType); + }; + + /** + Uses sqlite3_vfs_register() to register the + sqlite3.capi.sqlite3_vfs-type vfs, which must have already been + filled out properly. If the 2nd argument is truthy, the VFS is + registered as the default VFS, else it is not. + + On success, passes its first argument to this.holdReference() and + returns this object. Throws on error. + */ + vh.registerVfs = function(vfs, asDefault=false){ + if(!(vfs instanceof sqlite3.capi.sqlite3_vfs)){ + toss("Expecting a sqlite3_vfs-type argument."); + } + const rc = capi.sqlite3_vfs_register(vfs.pointer, asDefault ? 1 : 0); + if(rc){ + toss("sqlite3_vfs_register(",vfs,") failed with rc",rc); + } + if(vfs.pointer !== capi.sqlite3_vfs_find(vfs.$zName)){ + toss("BUG: sqlite3_vfs_find(vfs.$zName) failed for just-installed VFS", + vfs); + } + return vh.holdReference(vfs); + }; + + /** + A wrapper for installMethods() or registerVfs() to reduce + installation of a VFS and/or its I/O methods to a single + call. + + Accepts an object which contains the properties "io" and/or + "vfs", each of which is itself an object with following properties: + + - `struct`: an sqlite3.StructType-type struct. This must be a + populated (except for the methods) object of type + sqlite3_io_methods (for the "io" entry) or sqlite3_vfs (for the + "vfs" entry). + + - `methods`: an object mapping sqlite3_io_methods method names + (e.g. 'xClose') to JS implementations of those methods. + + For each of those object, this function passes its (`struct`, + `methods`, (optional) `applyArgcCheck`) properties to + this.installMethods(). + + If the `vfs` entry is set, its `struct` property is passed + to this.registerVfs(). The `vfs` entry may optionally have + an `asDefault` property, which gets passed as the 2nd + argument to registerVfs(). + + On success returns this object. Throws on error. + */ + vh.installVfs = function(opt){ + let count = 0; + for(const key of ['io','vfs']){ + const o = opt[key]; + if(o){ + ++count; + this.installMethods(o.struct, o.methods, !!o.applyArgcCheck); + if('vfs'===key) this.registerVfs(o.struct, !!o.asDefault); + } + } + if(!count) toss("Misue: installVfs() options object requires at least", + "one of 'io' or 'vfs' properties."); + return this; + }; + + sqlite3.VfsHelper = vh; +}/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 2fc6752694..0c32c1b071 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1867,6 +1867,9 @@ self.sqlite3InitModule = sqlite3InitModule; } importScripts(sqlite3Js); } + self.sqlite3InitModule.__isUnderTest = + true /* disables certain API-internal cleanup so that we can + test internal APIs from here */; self.sqlite3InitModule({ print: log, printErr: error diff --git a/manifest b/manifest index ecfa59693e..a615450fff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Doc\sand\slogging\stext\stweaks\sin\sthe\sOPFS\sasync\sproxy\sand\stest\sapp. -D 2022-11-30T03:08:50.289 +C Refactor\sa\ssignificant\schunk\sof\sthe\sOPFS\ssqlite3_vfs\sinit\scode\sinto\ssqlite3.VfsHelper,\sand\sinternal-use-only\sAPI\sencapsulating\scode\srelevant\sto\screating\snew\sVFSes\sin\sJS.\sIntended\sto\sassist\sin\spending\sexperimentation\swith\san\salternative\sOPFS\sVFS. +D 2022-11-30T05:27:36.071 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,25 +491,26 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile bc1696a1189f4c571b3d878b8f3a67c1f4b52c222f52d356027a5a0c707337c7 +F ext/wasm/GNUmakefile a1a94ff422be90c06263055aec4c569fe3e4ffb3e8cc7f954fd2739aefa52292 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 29276a845e57004e82efba61fa5866fd05f9137380a1dc26dc4c6d65264cd81c -F ext/wasm/api/extern-post-js.js 59e52f579cd3a332d73dae94c91b9579daafb10dd6ada03803f1afa6bdad7689 +F ext/wasm/api/README.md 44d1ea63a990678e373f17ff4b08c2f8599e5a5754b7394e76f7e52164f307c0 +F ext/wasm/api/extern-post-js.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f -F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 +F ext/wasm/api/sqlite3-api-cleanup.js 6a22a3287b34a2b4b0f8127614242e8307cce31ac060eb705a89a6d04dcb1028 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js 2f99c9b2c3c0f53bf40153de20db350378deed825ec0d5cf8f449e9e0081f08b -F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd +F ext/wasm/api/sqlite3-api-opfs.js e36eec4535dab2da186c1fc743b8fb3a37b21a7c61425546d748a3038c25efab +F ext/wasm/api/sqlite3-api-prologue.js 09fa4bf1e29596e599ead649003e01691419ffcb12d0f617152d577741973227 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 315582a4fe3b87ceaff69ad4a67d68a350b8014ac94d6a584d09ea922453c983 +F ext/wasm/api/sqlite3-vfs-helper.js 0b1de47e4407a414db1ee0ca3f9916ae74a9655fe1b1145d4a08e2b52f1ee25f F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js a4b6a165aafcd3b86118efaec6b47c70fbb6c64b5ab86d21ca8c250d42617dfa +F ext/wasm/tester1.c-pp.js 5820d578b67835aaf651230663dd207237071ee1f16f11a9c5168500aae04a50 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2064,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d0c8fa30a31c691bc1be5e98d806eeb1e23a8fc6cd54d87e5c1b720aa936e707 -R 05044488394ac3d7b358270198570ce0 +P 7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b +R 51f96a0acaea8419465c60238f6c368c U stephan -Z cb806f606aa315e69c2e3fceeb852a45 +Z aa663f409b5ddaba03a8fcd513053db4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 12e29fe3c0..4b90be6d7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b \ No newline at end of file +e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 \ No newline at end of file From 36a0163e40eb7097e562dfc6d12af516847de2f3 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 07:17:29 +0000 Subject: [PATCH 107/282] Rename some OPFS JS files. Prevent JS bindings of sqlite3_uri_...() from performing JS-to-C-string argument conversion on their first argument, as doing so is specifically illegal. FossilOrigin-Name: 79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb --- ext/wasm/GNUmakefile | 4 +-- ext/wasm/api/README.md | 12 +++---- ext/wasm/api/sqlite3-api-glue.js | 1 + ext/wasm/api/sqlite3-api-prologue.js | 33 ++++++++++++------- ext/wasm/api/sqlite3-opfs-async-proxy.js | 9 +++-- ext/wasm/api/sqlite3-vfs-helper.js | 28 +++++++++++----- ...qlite3-api-opfs.js => sqlite3-vfs-opfs.js} | 2 +- manifest | 24 +++++++------- manifest.uuid | 2 +- 9 files changed, 68 insertions(+), 47 deletions(-) rename ext/wasm/api/{sqlite3-api-opfs.js => sqlite3-vfs-opfs.js} (99%) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 3c6b3c9a15..eb23dc28bb 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -287,10 +287,10 @@ sqlite3-api.jses += $(dir.common)/whwasmutil.js sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js sqlite3-api.jses += $(sqlite3-api-build-version.js) -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # "External" API files which are part of our distribution diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 37d986584e..3a14229679 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -52,11 +52,6 @@ browser client: Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. -- **`sqlite3-vfs-helper.js`**\ - This internal-use-only file installs `sqlite3.VfsHelper` for use by - `sqlite3-api-*.js` files which create `sqlite3_vfs` implemenations. - `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after - the library is finished initializing. - **`sqlite3-api-oo1.js`**\ Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other @@ -79,7 +74,12 @@ browser client: a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- **`sqlite3-api-opfs.js`**\ +- **`sqlite3-vfs-helper.js`**\ + This internal-use-only file installs `sqlite3.VfsHelper` for use by + `sqlite3-*.js` files which create `sqlite3_vfs` implemenations. + `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after + the library is finished initializing. +- **`sqlite3-vfs-opfs.js`**\ is an sqlite3 VFS implementation which supports Google Chrome's Origin-Private FileSystem (OPFS) as a storage layer to provide persistent storage for database files in a browser. It requires... diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 86aa1d1813..6c1fcae820 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -69,6 +69,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ const aPtr = wasm.xWrap.argAdapter('*'); wasm.xWrap.argAdapter('sqlite3*', aPtr) + ('sqlite3_filename', aPtr) ('sqlite3_stmt*', aPtr) ('sqlite3_context*', aPtr) ('sqlite3_value*', aPtr) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 6d39581965..59cdab9295 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -913,9 +913,12 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_strlike", "int", "string","string","int"], ["sqlite3_trace_v2", "int", "sqlite3*", "int", "*", "*"], ["sqlite3_total_changes", "int", "sqlite3*"], - ["sqlite3_uri_boolean", "int", "string", "string", "int"], - ["sqlite3_uri_key", "string", "string", "int"], - ["sqlite3_uri_parameter", "string", "string", "string"], + /* Note sqlite3_uri_...() has very specific requirements + for their first C-string arguments, so we cannot perform + any type conversion on those. */ + ["sqlite3_uri_boolean", "int", "sqlite3_filename", "string", "int"], + ["sqlite3_uri_key", "string", "sqlite3_filename", "int"], + ["sqlite3_uri_parameter", "string", "sqlite3_filename", "string"], ["sqlite3_user_data","void*", "sqlite3_context*"], ["sqlite3_value_blob", "*", "sqlite3_value*"], ["sqlite3_value_bytes","int", "sqlite3_value*"], @@ -949,7 +952,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_realloc64", "*","*", "i64"], ["sqlite3_result_int64",undefined, "*", "i64"], ["sqlite3_total_changes64", "i64", ["sqlite3*"]], - ["sqlite3_uri_int64", "i64", ["string", "string", "i64"]], + ["sqlite3_uri_int64", "i64", ["sqlite3_filename", "string", "i64"]], ["sqlite3_value_int64","i64", "sqlite3_value*"], ]; @@ -1450,9 +1453,6 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( let lip = sqlite3ApiBootstrap.initializersAsync; delete sqlite3ApiBootstrap.initializersAsync; if(!lip || !lip.length) return Promise.resolve(sqlite3); - // Is it okay to resolve these in parallel or do we need them - // to resolve in order? We currently only have 1, so it - // makes no difference. lip = lip.map((f)=>{ const p = (f instanceof Promise) ? f : f(sqlite3); return p.catch((e)=>{ @@ -1460,10 +1460,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( throw e; }); }); - //let p = lip.shift(); - //while(lip.length) p = p.then(lip.shift()); - //return p.then(()=>sqlite3); - return Promise.all(lip).then(()=>{ + const postInit = ()=>{ if(!sqlite3.__isUnderTest){ /* Delete references to internal-only APIs which are used by some initializers. Retain them when running in test mode @@ -1473,7 +1470,19 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( delete sqlite3.StructBinder; } return sqlite3; - }); + }; + if(1){ + /* Run all initializers in sequence. The advantage is that it + allows us to have post-init cleanup defined outside of this + routine at the end of the list and have it run at a + well-defined time. */ + let p = lip.shift(); + while(lip.length) p = p.then(lip.shift()); + return p.then(postInit); + }else{ + /* Run them in an arbitrary order. */ + return Promise.all(lip).then(postInit); + } }, /** scriptInfo ideally gets injected into this object by the diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index da75f139c9..8bf34cc784 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -592,13 +592,12 @@ const installAsyncProxy = function(self){ (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite - does but it leads to immediate contention on journal files. */ + does but it leads to immediate contention on journal files. + Update: this approach reportedly only works for DELETE journal + mode. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ /* sqlite does not lock these files, so go ahead and grab an OPFS - lock. - - https://www.sqlite.org/uri.html - */ + lock. */ fh.xLock = "xOpen"/* Truthy value to keep entry from getting flagged as auto-locked. String value so that we can easily distinguish is later diff --git a/ext/wasm/api/sqlite3-vfs-helper.js b/ext/wasm/api/sqlite3-vfs-helper.js index 9a15dd85fd..f9d3c18c7b 100644 --- a/ext/wasm/api/sqlite3-vfs-helper.js +++ b/ext/wasm/api/sqlite3-vfs-helper.js @@ -183,25 +183,37 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `methods`, (optional) `applyArgcCheck`) properties to this.installMethods(). - If the `vfs` entry is set, its `struct` property is passed - to this.registerVfs(). The `vfs` entry may optionally have - an `asDefault` property, which gets passed as the 2nd - argument to registerVfs(). + If the `vfs` entry is set then: + + - Its `struct` property is passed to this.registerVfs(). The + `vfs` entry may optionally have an `asDefault` property, which + gets passed as the 2nd argument to registerVfs(). + + - If `struct.$zName` is falsy and the entry has a string-type + `name` property, `struct.$zName` is set to the C-string form of + that `name` value before registerVfs() is called. On success returns this object. Throws on error. */ vh.installVfs = function(opt){ let count = 0; - for(const key of ['io','vfs']){ + const propList = ['io','vfs']; + for(const key of propList){ const o = opt[key]; if(o){ ++count; this.installMethods(o.struct, o.methods, !!o.applyArgcCheck); - if('vfs'===key) this.registerVfs(o.struct, !!o.asDefault); + if('vfs'===key){ + if(!o.struct.$zName && 'string'===typeof o.name){ + o.struct.$zName = wasm.allocCString(o.name); + /* Note that we leak that C-string. */ + } + this.registerVfs(o.struct, !!o.asDefault); + } } } - if(!count) toss("Misue: installVfs() options object requires at least", - "one of 'io' or 'vfs' properties."); + if(!count) toss("Misuse: installVfs() options object requires at least", + "one of:", propList); return this; }; diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-vfs-opfs.js similarity index 99% rename from ext/wasm/api/sqlite3-api-opfs.js rename to ext/wasm/api/sqlite3-vfs-opfs.js index 5eb3b22b37..7c7eed7efa 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.js @@ -1266,7 +1266,7 @@ const installOpfsVfs = function callee(options){ opfsUtil.rootDirectory = d; log("End of OPFS sqlite3_vfs setup.", opfsVfs); promiseResolve(sqlite3); - }); + }).catch(promiseReject); }else{ promiseResolve(sqlite3); } diff --git a/manifest b/manifest index a615450fff..6bce7f15cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sa\ssignificant\schunk\sof\sthe\sOPFS\ssqlite3_vfs\sinit\scode\sinto\ssqlite3.VfsHelper,\sand\sinternal-use-only\sAPI\sencapsulating\scode\srelevant\sto\screating\snew\sVFSes\sin\sJS.\sIntended\sto\sassist\sin\spending\sexperimentation\swith\san\salternative\sOPFS\sVFS. -D 2022-11-30T05:27:36.071 +C Rename\ssome\sOPFS\sJS\sfiles.\sPrevent\sJS\sbindings\sof\ssqlite3_uri_...()\sfrom\sperforming\sJS-to-C-string\sargument\sconversion\son\stheir\sfirst\sargument,\sas\sdoing\sso\sis\sspecifically\sillegal. +D 2022-11-30T07:17:29.596 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,26 +491,26 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile a1a94ff422be90c06263055aec4c569fe3e4ffb3e8cc7f954fd2739aefa52292 +F ext/wasm/GNUmakefile c6b88ce5f735e29bef2b5da2e095848a7919100521a45e645cbf2456d759d5dd F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 44d1ea63a990678e373f17ff4b08c2f8599e5a5754b7394e76f7e52164f307c0 +F ext/wasm/api/README.md 44a05899d607a792370260a7c9193c9c111a7df06bc3ad1823c92a159526dbf3 F ext/wasm/api/extern-post-js.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 6a22a3287b34a2b4b0f8127614242e8307cce31ac060eb705a89a6d04dcb1028 -F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 +F ext/wasm/api/sqlite3-api-glue.js 03c40b65530d67bb2748b7380aea5fd1534500f812b76a6b401066dcd7fc4116 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-opfs.js e36eec4535dab2da186c1fc743b8fb3a37b21a7c61425546d748a3038c25efab -F ext/wasm/api/sqlite3-api-prologue.js 09fa4bf1e29596e599ead649003e01691419ffcb12d0f617152d577741973227 +F ext/wasm/api/sqlite3-api-prologue.js e1db3935e1deb1340c1dc0c0e4730b2b88254d616841ebd5bc6bb1b90b32657f F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 315582a4fe3b87ceaff69ad4a67d68a350b8014ac94d6a584d09ea922453c983 -F ext/wasm/api/sqlite3-vfs-helper.js 0b1de47e4407a414db1ee0ca3f9916ae74a9655fe1b1145d4a08e2b52f1ee25f +F ext/wasm/api/sqlite3-opfs-async-proxy.js efb0a7142c64c6a0f4cfbb588e6ea5baac9941364dfb0d40de2a21af1815bed3 +F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 +F ext/wasm/api/sqlite3-vfs-opfs.js cb38965f0f0a52d22bddd3a40312dbdb7826bc90f1a5ce9692b7986cd05bdf43 w ext/wasm/api/sqlite3-api-opfs.js F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7ce8608e221924d2c7067687eb6eef0f3cab181d5b4132e55a67d8514b6ce94b -R 51f96a0acaea8419465c60238f6c368c +P e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 +R e53006d34fb1afaea0f4ce19d105453b U stephan -Z aa663f409b5ddaba03a8fcd513053db4 +Z 3ad9d27606a854f1ad4ff4e5ecee27f6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4b90be6d7e..d65c6da223 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 \ No newline at end of file +79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb \ No newline at end of file From e7f5118bd347e83348417c91eda187d865938e4e Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 07:48:35 +0000 Subject: [PATCH 108/282] OPFS VFS: remove an invalid TODO and fix a property name typo which caused xCheckReservedLock() to always report false. FossilOrigin-Name: 3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 --- ext/wasm/api/sqlite3-vfs-opfs.js | 12 ++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.js b/ext/wasm/api/sqlite3-vfs-opfs.js index 7c7eed7efa..bd258759c4 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.js @@ -659,13 +659,9 @@ const installOpfsVfs = function callee(options){ involve an inherent race condition. For the time being, pending a better solution, we simply report whether the given pFile is open. - - FIXME: we need to ask the async half whether a lock is - held, as it's possible (since long after this method was - implemented) that we do not hold a lock on an OPFS file. */ const f = __openFiles[pFile]; - wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32'); + wasm.setMemValue(pOut, f.lockType ? 1 : 0, 'i32'); return 0; }, xClose: function(pFile){ @@ -706,7 +702,11 @@ const installOpfsVfs = function callee(options){ mTimeStart('xLock'); const f = __openFiles[pFile]; let rc = 0; - if( capi.SQLITE_LOCK_NONE === f.lockType ) { + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( !f.lockType ) { rc = opRun('xLock', pFile, lockType); if( 0===rc ) f.lockType = lockType; }else{ diff --git a/manifest b/manifest index 6bce7f15cc..8f95e10ebd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\ssome\sOPFS\sJS\sfiles.\sPrevent\sJS\sbindings\sof\ssqlite3_uri_...()\sfrom\sperforming\sJS-to-C-string\sargument\sconversion\son\stheir\sfirst\sargument,\sas\sdoing\sso\sis\sspecifically\sillegal. -D 2022-11-30T07:17:29.596 +C OPFS\sVFS:\sremove\san\sinvalid\sTODO\sand\sfix\sa\sproperty\sname\stypo\swhich\scaused\sxCheckReservedLock()\sto\salways\sreport\sfalse. +D 2022-11-30T07:48:35.434 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1 F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js efb0a7142c64c6a0f4cfbb588e6ea5baac9941364dfb0d40de2a21af1815bed3 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.js cb38965f0f0a52d22bddd3a40312dbdb7826bc90f1a5ce9692b7986cd05bdf43 w ext/wasm/api/sqlite3-api-opfs.js +F ext/wasm/api/sqlite3-vfs-opfs.js 6848abfd6dcc9e2e5ed99193929c735984c1784186d6130839ea5f5bcc7fd7df F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e25d7b080a807e35b32cb885ea75b384130e5c6e936dfef783c5b45d9bfe77d8 -R e53006d34fb1afaea0f4ce19d105453b +P 79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb +R d6a46579cca1740ff67e566c55a1e4a7 U stephan -Z 3ad9d27606a854f1ad4ff4e5ecee27f6 +Z c6b01228f796eacb2c5dcc52cd57eff3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d65c6da223..3a518c054a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb \ No newline at end of file +3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 \ No newline at end of file From c5141c9efa564dc6e6d39685b23741d6632dae5a Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 08:37:17 +0000 Subject: [PATCH 109/282] Simplify how the OPFS VFS async proxy copies initial state sent to it from the synchronous side of the connection. Make the lock-wait time a multiple of the wait-loop timeout interval. FossilOrigin-Name: eddafafffa634a42ceeed70aa3fc58be130527612157a4bf4ff9e65c7f6dc26c --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 23 +++-------------------- ext/wasm/api/sqlite3-vfs-opfs.js | 10 ++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 21 insertions(+), 28 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 8bf34cc784..d8234d5092 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -298,7 +298,7 @@ const installAsyncProxy = function(self){ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, msBase = 300; + const maxTries = 6, msBase = state.asyncIdleWaitTime * 3; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -316,7 +316,6 @@ const installAsyncProxy = function(self){ } warn("Error getting sync handle for",opName+"(). Waiting",ms, "ms and trying again.",fh.filenameAbs,e); - //await releaseImplicitLocks(); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } @@ -835,17 +834,10 @@ const installAsyncProxy = function(self){ o.key = k; o.f = vi; } - /** - waitTime is how long (ms) to wait for each Atomics.wait(). - We need to wake up periodically to give the thread a chance - to do other things. If this is too high (e.g. 500ms) then - even two workers/tabs can easily run into locking errors. - */ - const waitTime = 100; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, waitTime + state.sabOPView, state.opIds.whichOp, 0, state.asyncIdleWaitTime )){ await releaseImplicitLocks(); continue; @@ -874,20 +866,11 @@ const installAsyncProxy = function(self){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ const opt = data.args; - state.littleEndian = opt.littleEndian; - state.asyncS11nExceptions = opt.asyncS11nExceptions; + for(const k in opt) state[k] = opt[k]; state.verbose = opt.verbose ?? 1; - state.fileBufferSize = opt.fileBufferSize; - state.sabS11nOffset = opt.sabS11nOffset; - state.sabS11nSize = opt.sabS11nSize; - state.sabOP = opt.sabOP; state.sabOPView = new Int32Array(state.sabOP); - state.sabIO = opt.sabIO; state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.opIds = opt.opIds; - state.sq3Codes = opt.sq3Codes; - state.opfsFlags = opt.opfsFlags; Object.keys(vfsAsyncImpls).forEach((k)=>{ if(!Number.isFinite(state.opIds[k])){ toss("Maintenance required: missing state.opIds[",k,"]"); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.js b/ext/wasm/api/sqlite3-vfs-opfs.js index bd258759c4..7b0a0b3228 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.js @@ -268,6 +268,16 @@ const installOpfsVfs = function callee(options){ // Int16Array uses the platform's endianness. return new Int16Array(buffer)[0] === 256; })(); + /** + asyncIdleWaitTime is how long (ms) to wait, in the async proxy, + for each Atomics.wait() when waiting on inbound VFS API calls. + We need to wake up periodically to give the thread a chance to + do other things. If this is too high (e.g. 500ms) then even two + workers/tabs can easily run into locking errors. Some multiple + of this value is also used for determining how long to wait on + lock contention to free up. + */ + state.asyncIdleWaitTime = 100; /** Whether the async counterpart should log exceptions to the serialization channel. That produces a great deal of diff --git a/manifest b/manifest index 8f95e10ebd..fe90f1f943 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sVFS:\sremove\san\sinvalid\sTODO\sand\sfix\sa\sproperty\sname\stypo\swhich\scaused\sxCheckReservedLock()\sto\salways\sreport\sfalse. -D 2022-11-30T07:48:35.434 +C Simplify\show\sthe\sOPFS\sVFS\sasync\sproxy\scopies\sinitial\sstate\ssent\sto\sit\sfrom\sthe\ssynchronous\sside\sof\sthe\sconnection.\sMake\sthe\slock-wait\stime\sa\smultiple\sof\sthe\swait-loop\stimeout\sinterval. +D 2022-11-30T08:37:17.860 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -508,9 +508,9 @@ F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f F ext/wasm/api/sqlite3-api-prologue.js e1db3935e1deb1340c1dc0c0e4730b2b88254d616841ebd5bc6bb1b90b32657f F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js efb0a7142c64c6a0f4cfbb588e6ea5baac9941364dfb0d40de2a21af1815bed3 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.js 6848abfd6dcc9e2e5ed99193929c735984c1784186d6130839ea5f5bcc7fd7df +F ext/wasm/api/sqlite3-vfs-opfs.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 79832808de2cbdba140ed9e0558f1502b51d131ab4315265315922cda7b748cb -R d6a46579cca1740ff67e566c55a1e4a7 +P 3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 +R e3e2fc506276773f57deed8b914e8dd1 U stephan -Z c6b01228f796eacb2c5dcc52cd57eff3 +Z 98f64cec6c8eaf42e6c02ddc6003f146 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a518c054a..20dcd52ad4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 \ No newline at end of file +eddafafffa634a42ceeed70aa3fc58be130527612157a4bf4ff9e65c7f6dc26c \ No newline at end of file From 8ae954557738b29ebaaa6b1eb954c4e9a703de36 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 11:50:16 +0000 Subject: [PATCH 110/282] Install sqlite3_malloc/sqlite3_free() as the JS-side WASM allocator (as opposed to replacing C-level's malloc()/free() with them). All tests work and this eliminates the potential for allocator discrepancies when using the (de)serialize APIs. FossilOrigin-Name: 95c78f6b46e0d8efa4313061f47677479f48610b7a7261dc8d0fb1859aca2ad9 --- ext/wasm/api/sqlite3-api-cleanup.js | 4 +--- ext/wasm/api/sqlite3-api-prologue.js | 16 ++++++++++------ ext/wasm/tester1.c-pp.js | 4 ++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index 0ec0fbfbe3..30cd64b053 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -22,12 +22,10 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build */ const SABC = Object.assign( Object.create(null), { - Module: Module /* ==> Currently needs to be exposed here for - test code. NOT part of the public API. */, exports: Module['asm'], memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */ }, - self.sqlite3ApiConfig || Object.create(null) + self.sqlite3ApiConfig || {} ); /** diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 59cdab9295..31cd8aa539 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -65,11 +65,15 @@ - `allocExportName`: the name of the function, in `exports`, of the `malloc(3)`-compatible routine for the WASM environment. Defaults - to `"malloc"`. + to `"sqlite3_malloc"`. Beware that using any allocator other than + sqlite3_malloc() may require care in certain client-side code + regarding which allocator is uses. Notably, sqlite3_deserialize() + and sqlite3_serialize() can only safely use memory from different + allocators under very specific conditions. - `deallocExportName`: the name of the function, in `exports`, of the `free(3)`-compatible routine for the WASM - environment. Defaults to `"free"`. + environment. Defaults to `"sqlite3_free"`. - `wasmfsOpfsDir`[^1]: if the environment supports persistent storage using OPFS-over-WASMFS , this directory names the "mount @@ -104,8 +108,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( } return !!self.BigInt64Array; })(), - allocExportName: 'malloc', - deallocExportName: 'free', + allocExportName: 'sqlite3_malloc', + deallocExportName: 'sqlite3_free', wasmfsOpfsDir: '/opfs' }, apiConfig || {}); @@ -727,8 +731,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return pRet; }; - const keyAlloc = config.allocExportName || 'malloc', - keyDealloc = config.deallocExportName || 'free'; + const keyAlloc = config.allocExportName, + keyDealloc = config.deallocExportName; for(const key of [keyAlloc, keyDealloc]){ const f = wasm.exports[key]; if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 0c32c1b071..08f5480b54 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -336,6 +336,10 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////// T.g('Basic sanity checks') + .t("JS wasm-side allocator === sqlite3_malloc()", function(sqlite3){ + T.assert(wasm.alloc.impl === wasm.exports.sqlite3_malloc) + .assert(wasm.dealloc === wasm.exports.sqlite3_free); + }) .t('Namespace object checks', function(sqlite3){ const wasmCtypes = wasm.ctype; T.assert(wasmCtypes.structs[0].name==='sqlite3_vfs'). diff --git a/manifest b/manifest index fe90f1f943..557b4aef57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\show\sthe\sOPFS\sVFS\sasync\sproxy\scopies\sinitial\sstate\ssent\sto\sit\sfrom\sthe\ssynchronous\sside\sof\sthe\sconnection.\sMake\sthe\slock-wait\stime\sa\smultiple\sof\sthe\swait-loop\stimeout\sinterval. -D 2022-11-30T08:37:17.860 +C Install\ssqlite3_malloc/sqlite3_free()\sas\sthe\sJS-side\sWASM\sallocator\s(as\sopposed\sto\sreplacing\sC-level's\smalloc()/free()\swith\sthem).\sAll\stests\swork\sand\sthis\seliminates\sthe\spotential\sfor\sallocator\sdiscrepancies\swhen\susing\sthe\s(de)serialize\sAPIs. +D 2022-11-30T11:50:16.116 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,10 +502,10 @@ F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f -F ext/wasm/api/sqlite3-api-cleanup.js 6a22a3287b34a2b4b0f8127614242e8307cce31ac060eb705a89a6d04dcb1028 +F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js 03c40b65530d67bb2748b7380aea5fd1534500f812b76a6b401066dcd7fc4116 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-prologue.js e1db3935e1deb1340c1dc0c0e4730b2b88254d616841ebd5bc6bb1b90b32657f +F ext/wasm/api/sqlite3-api-prologue.js e63bcd1d1942d3313bd11e76ac1fccdd3e34ba54a48b8c8296db8dd892705dbc F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js 5820d578b67835aaf651230663dd207237071ee1f16f11a9c5168500aae04a50 +F ext/wasm/tester1.c-pp.js c39594bb1a0272a08a1544277a29c1ed920b4c4877611591e7c3185744ae2431 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3b037caa2fa07b6c44c485574e9e5dc71f4a8e82bc902c1321bb0b918b139c74 -R e3e2fc506276773f57deed8b914e8dd1 +P eddafafffa634a42ceeed70aa3fc58be130527612157a4bf4ff9e65c7f6dc26c +R 40e47e8cbba6f9c6283963002c627e97 U stephan -Z 98f64cec6c8eaf42e6c02ddc6003f146 +Z 339b33401805070aa414e2061b0a89bf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 20dcd52ad4..01d15d8764 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eddafafffa634a42ceeed70aa3fc58be130527612157a4bf4ff9e65c7f6dc26c \ No newline at end of file +95c78f6b46e0d8efa4313061f47677479f48610b7a7261dc8d0fb1859aca2ad9 \ No newline at end of file From a42281a312459d266d4a2189c43b80e83d2ab8e7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 30 Nov 2022 13:44:31 +0000 Subject: [PATCH 111/282] Always use nanosleep() (instead of usleep() or sleep) if the _POSIX_C_SOURCE macro says it should be available. FossilOrigin-Name: 6620c57b9d3eae7226a412318b43393196df069b5b90aae0cf1743fdd2d102dd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 557b4aef57..0f77704114 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Install\ssqlite3_malloc/sqlite3_free()\sas\sthe\sJS-side\sWASM\sallocator\s(as\sopposed\sto\sreplacing\sC-level's\smalloc()/free()\swith\sthem).\sAll\stests\swork\sand\sthis\seliminates\sthe\spotential\sfor\sallocator\sdiscrepancies\swhen\susing\sthe\s(de)serialize\sAPIs. -D 2022-11-30T11:50:16.116 +C Always\suse\snanosleep()\s(instead\sof\susleep()\sor\ssleep)\sif\sthe\s\n_POSIX_C_SOURCE\smacro\ssays\sit\sshould\sbe\savailable. +D 2022-11-30T13:44:31.719 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -629,7 +629,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 0e59600d25b72034c7666b8b7dcc527f039b5d9c16f24a7eca4c08c66f63c364 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c +F src/os_unix.c 08191111a7040b8d5a6fff946f9fc9a11a0f83bac727c0415dfc5d030e1bc41f F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c d3122cf67f327f1e2df12d06236a3473a8099542071e257067552f42917f172d @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eddafafffa634a42ceeed70aa3fc58be130527612157a4bf4ff9e65c7f6dc26c -R 40e47e8cbba6f9c6283963002c627e97 -U stephan -Z 339b33401805070aa414e2061b0a89bf +P 95c78f6b46e0d8efa4313061f47677479f48610b7a7261dc8d0fb1859aca2ad9 +R d8bc00ef34572bdd01b9cc18f03ff989 +U drh +Z 7f2c7f128d3f16bc147b96b4f9824bd1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 01d15d8764..0e864d4c09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95c78f6b46e0d8efa4313061f47677479f48610b7a7261dc8d0fb1859aca2ad9 \ No newline at end of file +6620c57b9d3eae7226a412318b43393196df069b5b90aae0cf1743fdd2d102dd \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index c390b51888..ddb6f0c07c 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -6676,7 +6676,7 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** than the argument. */ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ -#if OS_VXWORKS +#if OS_VXWORKS || _POSIX_C_SOURCE >= 199309L struct timespec sp; sp.tv_sec = microseconds / 1000000; From ec192e0659cd0fac1a29616c81a693f696f6f24a Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 30 Nov 2022 18:21:01 +0000 Subject: [PATCH 112/282] Rename some JS files from X.js to X.c-pp.js to keep the maintainer, and downstream build customizers, aware that those files contain constructs specific to the c-pp preprocessor and will not run as-is in JS. FossilOrigin-Name: 2eade7c7b17a186735c72974c11a34798a08364861d0f307e897ba765c0a93c7 --- ext/wasm/GNUmakefile | 12 +++--- ext/wasm/api/README.md | 40 +++++++++++++------ ...tern-post-js.js => extern-post-js.c-pp.js} | 0 ext/wasm/api/post-js-header.js | 3 +- ext/wasm/api/{pre-js.js => pre-js.c-pp.js} | 0 ...3-vfs-opfs.js => sqlite3-vfs-opfs.c-pp.js} | 0 manifest | 24 +++++------ manifest.uuid | 2 +- 8 files changed, 48 insertions(+), 33 deletions(-) rename ext/wasm/api/{extern-post-js.js => extern-post-js.c-pp.js} (100%) rename ext/wasm/api/{pre-js.js => pre-js.c-pp.js} (100%) rename ext/wasm/api/{sqlite3-vfs-opfs.js => sqlite3-vfs-opfs.c-pp.js} (100%) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index eb23dc28bb..31431352ea 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -279,8 +279,8 @@ sqlite3-license-version-header.js := $(dir.api)/sqlite3-license-version-header.j # sqlite3-api-build-version.js = generated JS file which populates the # sqlite3.version object using $(bin.version-info). sqlite3-api-build-version.js := $(dir.tmp)/sqlite3-api-build-version.js -# sqlite3-api.jses = the list of JS files which make up $(sqlite3-api.js), in -# the order they need to be assembled. +# sqlite3-api.jses = the list of JS files which make up +# $(sqlite3-api.js.in), in the order they need to be assembled. sqlite3-api.jses := $(sqlite3-license-version.js) sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js sqlite3-api.jses += $(dir.common)/whwasmutil.js @@ -290,7 +290,7 @@ sqlite3-api.jses += $(sqlite3-api-build-version.js) sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # "External" API files which are part of our distribution @@ -346,12 +346,12 @@ $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) # --post-js and --pre-js are emcc flags we use to append/prepend JS to # the generated emscripten module file. The following rules generate # various versions of those files for the vanilla and ESM builds. -pre-js.js.in := $(dir.api)/pre-js.js +pre-js.js.in := $(dir.api)/pre-js.c-pp.js pre-js.js.esm := $(dir.tmp)/pre-js.esm.js pre-js.js.vanilla := $(dir.tmp)/pre-js.vanilla.js $(eval $(call C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.vanilla),$(c-pp.D.vanilla))) $(eval $(call C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.esm),$(c-pp.D.esm))) -post-js.js.in := $(dir.tmp)/post-js.js +post-js.js.in := $(dir.tmp)/post-js.c-pp.js post-js.js.vanilla := $(dir.tmp)/post-js.vanilla.js post-js.js.esm := $(dir.tmp)/post-js.esm.js post-jses.js := \ @@ -371,7 +371,7 @@ $(eval $(call C-PP.FILTER,$(post-js.js.in),$(post-js.js.esm),$(c-pp.D.esm))) # extern-post-js* and extern-pre-js* are files for use with # Emscripten's --extern-pre-js and --extern-post-js flags. These # rules make different copies for the vanilla and ESM builds. -extern-post-js.js.in := $(dir.api)/extern-post-js.js +extern-post-js.js.in := $(dir.api)/extern-post-js.c-pp.js extern-post-js.js.vanilla := $(dir.tmp)/extern-post-js.vanilla.js extern-post-js.js.esm := $(dir.tmp)/extern-post-js.esm.js $(eval $(call C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.vanilla),$(c-pp.D.vanilla))) diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 3a14229679..f9955a8957 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -17,7 +17,8 @@ multiple JS files because: to be loaded as JS Workers. Note that the structure described here is the current state of things, -not necessarily the "final" state. +as of this writing, but is not set in stone forever and may change +at any time. The overall idea is that the following files get concatenated together, in the listed order, the resulting file is loaded by a @@ -45,10 +46,13 @@ browser client: independent spinoff project, conceived for the sqlite3 project but maintained separately. - **`sqlite3-api-glue.js`**\ - Invokes functionality exposed by the previous two files to - flesh out low-level parts of `sqlite3-api-prologue.js`. Most of - these pieces related to the `sqlite3.capi.wasm` object. -- **`sqlite3-api-build-version.js`**\ + Invokes functionality exposed by the previous two files to flesh out + low-level parts of `sqlite3-api-prologue.js`. Most of these pieces + related to populating the `sqlite3.capi.wasm` object. This file + also deletes most global-scope symbols the above files create, + effectively moving them into the scope being used for initializing + the API. +- **`/sqlite3-api-build-version.js`**\ Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. @@ -76,10 +80,10 @@ browser client: in a Worker thread. - **`sqlite3-vfs-helper.js`**\ This internal-use-only file installs `sqlite3.VfsHelper` for use by - `sqlite3-*.js` files which create `sqlite3_vfs` implemenations. + `sqlite3-*.js` files which create `sqlite3_vfs` implementations. `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after the library is finished initializing. -- **`sqlite3-vfs-opfs.js`**\ +- **`sqlite3-vfs-opfs.c-pp.js`**\ is an sqlite3 VFS implementation which supports Google Chrome's Origin-Private FileSystem (OPFS) as a storage layer to provide persistent storage for database files in a browser. It requires... @@ -100,6 +104,13 @@ browser client: symbol installed. When adapting the API for non-Emscripten toolchains, this "should" be the only file where changes are needed. + +**Files with the extension `.c-pp.js`** are intended [to be processed +with `c-pp`](#c-pp), noting that such preprocessing may be applied +after all of the relevant files are concatenated. That extension is +used primarily to keep the code maintainers cognisant of the fact that +those files contain constructs which will not run as-is in JavaScript. + The build process glues those files together, resulting in `sqlite3-api.js`, which is everything except for the `post-js-*.js` files, and `sqlite3.js`, which is the Emscripten-generated amalgamated @@ -119,7 +130,7 @@ into the build-generated `sqlite3.js` along with `sqlite3-api.js`. Emscripten-specific header for Emscripten's `--extern-pre-js` flag. As of this writing, that file is only used for experimentation purposes and holds no code relevant to the production deliverables. -- `pre-js.js`\ +- `pre-js.c-pp.js`\ Emscripten-specific header for Emscripten's `--pre-js` flag. This file is intended as a place to override certain Emscripten behavior before it starts up, but corner-case Emscripten bugs keep that from @@ -130,17 +141,20 @@ into the build-generated `sqlite3.js` along with `sqlite3-api.js`. - `post-js-footer.js`\ Emscripten-specific footer for the `--post-js` input. This closes off the lexical scope opened by `post-js-header.js`. -- `extern-post-js.js`\ +- `extern-post-js.c-pp.js`\ Emscripten-specific header for Emscripten's `--extern-post-js` flag. This file overwrites the Emscripten-installed `sqlite3InitModule()` function with one which, after the module is loaded, also initializes the asynchronous parts of the sqlite3 module. For example, the OPFS VFS support. -## Preprocessing of Source Files + +Preprocessing of Source Files +------------------------------------------------------------------------ Certain files in the build require preprocessing to filter in/out parts which differ between vanilla JS builds and ES6 Module -(a.k.a. esm) builds. The preprocessor itself is in -[](/file/ext/wasm/c-pp.c) and the associates flags and rules are in -[](/file/ext/wasm/GNUmakefile). +(a.k.a. esm) builds. The preprocessor application itself is in +[`c-pp.c`](/file/ext/wasm/c-pp.c) and the complete technical details +of such preprocessing are maintained in +[`GNUMakefile`](/file/ext/wasm/GNUmakefile). diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.c-pp.js similarity index 100% rename from ext/wasm/api/extern-post-js.js rename to ext/wasm/api/extern-post-js.c-pp.js diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index 82a80e5a17..0e27e1fd94 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -19,7 +19,8 @@ Module.postRun.push(function(Module/*the Emscripten-style module object*/){ - sqlite3-api-glue.js => glues previous parts together - sqlite3-api-oo.js => SQLite3 OO API #1 - sqlite3-api-worker1.js => Worker-based API - - sqlite3-api-opfs.js => OPFS VFS + - sqlite3-vfs-helper.js => Internal-use utilities for... + - sqlite3-vfs-opfs.js => OPFS VFS - sqlite3-api-cleanup.js => final API cleanup - post-js-footer.js => closes this postRun() function */ diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.c-pp.js similarity index 100% rename from ext/wasm/api/pre-js.js rename to ext/wasm/api/pre-js.c-pp.js diff --git a/ext/wasm/api/sqlite3-vfs-opfs.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js similarity index 100% rename from ext/wasm/api/sqlite3-vfs-opfs.js rename to ext/wasm/api/sqlite3-vfs-opfs.c-pp.js diff --git a/manifest b/manifest index 0f77704114..39ae01b92a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\suse\snanosleep()\s(instead\sof\susleep()\sor\ssleep)\sif\sthe\s\n_POSIX_C_SOURCE\smacro\ssays\sit\sshould\sbe\savailable. -D 2022-11-30T13:44:31.719 +C Rename\ssome\sJS\sfiles\sfrom\sX.js\sto\sX.c-pp.js\sto\skeep\sthe\smaintainer,\sand\sdownstream\sbuild\scustomizers,\saware\sthat\sthose\sfiles\scontain\sconstructs\sspecific\sto\sthe\sc-pp\spreprocessor\sand\swill\snot\srun\sas-is\sin\sJS. +D 2022-11-30T18:21:01.774 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,17 +491,17 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile c6b88ce5f735e29bef2b5da2e095848a7919100521a45e645cbf2456d759d5dd +F ext/wasm/GNUmakefile 85fb066f3ad17f9252deba2a3625736fc9e4172ab1494a73f74d65e21cf93387 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 44a05899d607a792370260a7c9193c9c111a7df06bc3ad1823c92a159526dbf3 -F ext/wasm/api/extern-post-js.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d +F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a +F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d w ext/wasm/api/extern-post-js.js F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 -F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b38718168bbde8fdb2a439b8 -F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f +F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 +F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f w ext/wasm/api/pre-js.js F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js 03c40b65530d67bb2748b7380aea5fd1534500f812b76a6b401066dcd7fc4116 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 @@ -510,7 +510,7 @@ F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1 F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 w ext/wasm/api/sqlite3-vfs-opfs.js F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 95c78f6b46e0d8efa4313061f47677479f48610b7a7261dc8d0fb1859aca2ad9 -R d8bc00ef34572bdd01b9cc18f03ff989 -U drh -Z 7f2c7f128d3f16bc147b96b4f9824bd1 +P 6620c57b9d3eae7226a412318b43393196df069b5b90aae0cf1743fdd2d102dd +R f73408d2fb9e184edb2374ed7b5424d1 +U stephan +Z a35fbe212dde08194786732b15211922 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0e864d4c09..019a43910e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6620c57b9d3eae7226a412318b43393196df069b5b90aae0cf1743fdd2d102dd \ No newline at end of file +2eade7c7b17a186735c72974c11a34798a08364861d0f307e897ba765c0a93c7 \ No newline at end of file From 2a757658a042586820c0f292d4292e052202fd3d Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 30 Nov 2022 19:11:31 +0000 Subject: [PATCH 113/282] Attempt to rationalize the bits associated with ".wheretrace". Provide a decoder key in sqliteInt.h for what each bit is intended to do. FossilOrigin-Name: 8ec361695a107a94f2cf6a7fe509656a99d85d49bd7c74133c69903f059a2675 --- manifest | 24 ++++++++--------- manifest.uuid | 2 +- src/sqliteInt.h | 30 +++++++++++++++++++++ src/where.c | 72 ++++++++++++++++++++++++------------------------- src/wherecode.c | 28 ++++++++++--------- 5 files changed, 94 insertions(+), 62 deletions(-) diff --git a/manifest b/manifest index 39ae01b92a..172dd44886 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\ssome\sJS\sfiles\sfrom\sX.js\sto\sX.c-pp.js\sto\skeep\sthe\smaintainer,\sand\sdownstream\sbuild\scustomizers,\saware\sthat\sthose\sfiles\scontain\sconstructs\sspecific\sto\sthe\sc-pp\spreprocessor\sand\swill\snot\srun\sas-is\sin\sJS. -D 2022-11-30T18:21:01.774 +C Attempt\sto\srationalize\sthe\sbits\sassociated\swith\s".wheretrace".\s\sProvide\na\sdecoder\skey\sin\ssqliteInt.h\sfor\swhat\seach\sbit\sis\sintended\sto\sdo. +D 2022-11-30T19:11:31.801 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -497,11 +497,11 @@ F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a -F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d w ext/wasm/api/extern-post-js.js +F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 -F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f w ext/wasm/api/pre-js.js +F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js 03c40b65530d67bb2748b7380aea5fd1534500f812b76a6b401066dcd7fc4116 F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 @@ -510,7 +510,7 @@ F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1 F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 w ext/wasm/api/sqlite3-vfs-opfs.js +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -650,7 +650,7 @@ F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063 F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h a8d507577ca5e6581abb65c30497f7b8cfc7c9feb0d768a081c1e4a6adfcd061 +F src/sqliteInt.h 4ddd98e423276714479f9f22dbbda050e8ef99aa97e7e26bf0bdf58acef0ca42 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -732,9 +732,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 81422870d17fdf3880fdb9d3b3965e1aa5efb7a72c6933cec3f1bc8081c463db +F src/where.c bf470b5d1ba03af8d558a0c98cc1fa97b330e03a198a7af61895e5a2e8d93f20 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 +F src/wherecode.c ee52c2781c36004d23c85bf111063b78fc16e5e1b6a0d424326af8bf90babb0b F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6620c57b9d3eae7226a412318b43393196df069b5b90aae0cf1743fdd2d102dd -R f73408d2fb9e184edb2374ed7b5424d1 -U stephan -Z a35fbe212dde08194786732b15211922 +P 2eade7c7b17a186735c72974c11a34798a08364861d0f307e897ba765c0a93c7 +R 017a8b97f0f146b3506618eca7867206 +U drh +Z b8ffdae4ce7e302410b9546e9e5b50f2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 019a43910e..55309e681a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2eade7c7b17a186735c72974c11a34798a08364861d0f307e897ba765c0a93c7 \ No newline at end of file +8ec361695a107a94f2cf6a7fe509656a99d85d49bd7c74133c69903f059a2675 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7d7e5b56f0..4711e4f094 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1073,6 +1073,36 @@ extern u32 sqlite3WhereTrace; # define WHERETRACE(K,X) #endif +/* +** Bits for the sqlite3WhereTrace mask: +** +** (---any--) Top-level block structure +** 0x-------F High-level debug messages +** 0x----FFF- More detail +** 0xFFFF---- Low-level debug messages +** +** 0x00000001 Code generation +** 0x00000002 Solver +** 0x00000004 Solver costs +** 0x00000008 WhereLoop inserts +** +** 0x00000010 Display sqlite3_index_info xBestIndex calls +** 0x00000020 Range an equality scan metrics +** 0x00000040 IN operator decisions +** 0x00000080 WhereLoop cost adjustements +** 0x00000100 +** 0x00000200 Covering index decisions +** 0x00000400 OR optimization +** 0x00000800 Index scanner +** 0x00001000 More details associated with code generation +** 0x00002000 +** 0x00004000 Show all WHERE terms at key points +** 0x00008000 Show the full SELECT statement at key places +** +** 0x00010000 Show more detail when printing WHERE terms +** 0x00020000 Show WHERE terms returned from whereScanNext() +*/ + /* ** An instance of the following structure is used to store the busy-handler diff --git a/src/where.c b/src/where.c index 589300c6ba..c9eeabe8b6 100644 --- a/src/where.c +++ b/src/where.c @@ -705,7 +705,7 @@ static void translateColumnToCopy( #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; for(i=0; inConstraint; i++){ sqlite3DebugPrintf( " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", @@ -725,7 +725,7 @@ static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ } static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; for(i=0; inConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, @@ -1742,7 +1742,7 @@ static int whereRangeSkipScanEst( int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); pLoop->nOut -= nAdjust; *pbDone = 1; - WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", nLower, nUpper, nAdjust*-1, pLoop->nOut)); } @@ -1920,7 +1920,7 @@ static int whereRangeScanEst( if( nNewnOut>nOut ){ - WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n", pLoop->nOut, nOut)); } #endif @@ -2018,7 +2018,7 @@ static int whereEqualScanEst( pBuilder->nRecValid = nEq; whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", + WHERETRACE(0x20,("equality scan regions %s(%d): %d\n", p->zName, nEq-1, (int)a[1])); *pnRow = a[1]; @@ -2068,7 +2068,7 @@ static int whereInScanEst( if( rc==SQLITE_OK ){ if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); + WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); return rc; @@ -2177,7 +2177,7 @@ void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm); } sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){ int i; for(i=0; inLTerm; i++){ sqlite3WhereTermPrint(p->aLTerm[i], i); @@ -3055,7 +3055,7 @@ static int whereLoopAddBtreeIndex( && pNew->nOut+10 > pProbe->aiRowLogEst[0] ){ #if WHERETRACE_ENABLED /* 0x01 */ - if( sqlite3WhereTrace & 0x01 ){ + if( sqlite3WhereTrace & 0x20 ){ sqlite3DebugPrintf( "STAT4 determines term has low selectivity:\n"); sqlite3WhereTermPrint(pTerm, 999); @@ -3611,7 +3611,7 @@ static int whereLoopAddBtree( if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); if( isCov==0 ){ - WHERETRACE(0xff, + WHERETRACE(0x200, ("-> %s is not a covering index" " according to whereIsCoveringIndex()\n", pProbe->zName)); assert( m!=0 ); @@ -3619,18 +3619,18 @@ static int whereLoopAddBtree( m = 0; pNew->wsFlags |= isCov; if( isCov & WHERE_IDX_ONLY ){ - WHERETRACE(0xff, + WHERETRACE(0x200, ("-> %s is a covering expression index" " according to whereIsCoveringIndex()\n", pProbe->zName)); }else{ assert( isCov==WHERE_EXPRIDX ); - WHERETRACE(0xff, + WHERETRACE(0x200, ("-> %s might be a covering expression index" " according to whereIsCoveringIndex()\n", pProbe->zName)); } } }else if( m==0 ){ - WHERETRACE(0xff, + WHERETRACE(0x200, ("-> %s a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; @@ -3807,7 +3807,7 @@ static int whereLoopAddVirtualOne( ** that the particular combination of parameters provided is unusable. ** Make no entries in the loop table. */ - WHERETRACE(0xffff, (" ^^^^--- non-viable plan rejected!\n")); + WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n")); return SQLITE_OK; } return rc; @@ -3918,7 +3918,7 @@ static int whereLoopAddVirtualOne( sqlite3_free(pNew->u.vtab.idxStr); pNew->u.vtab.needFree = 0; } - WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", + WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", *pbIn, (sqlite3_uint64)mPrereq, (sqlite3_uint64)(pNew->prereq & ~mPrereq))); @@ -4110,7 +4110,7 @@ static int whereLoopAddVirtual( /* First call xBestIndex() with all constraints usable. */ WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); - WHERETRACE(0x40, (" VirtualOne: all usable\n")); + WHERETRACE(0x800, (" VirtualOne: all usable\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry ); @@ -4135,7 +4135,7 @@ static int whereLoopAddVirtual( /* If the plan produced by the earlier call uses an IN(...) term, call ** xBestIndex again, this time with IN(...) terms disabled. */ if( bIn ){ - WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); + WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0); assert( bIn==0 ); @@ -4161,7 +4161,7 @@ static int whereLoopAddVirtual( mPrev = mNext; if( mNext==ALLBITS ) break; if( mNext==mBest || mNext==mBestNoIn ) continue; - WHERETRACE(0x40, (" VirtualOne: mPrev=%04llx mNext=%04llx\n", + WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n", (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext)); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0); @@ -4175,7 +4175,7 @@ static int whereLoopAddVirtual( ** that requires no source tables at all (i.e. one guaranteed to be ** usable), make a call here with all source tables disabled */ if( rc==SQLITE_OK && seenZero==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled\n")); + WHERETRACE(0x800, (" VirtualOne: all disabled\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0); if( bIn==0 ) seenZeroNoIN = 1; @@ -4185,7 +4185,7 @@ static int whereLoopAddVirtual( ** that requires no source tables at all and does not use an IN(...) ** operator, make a final call to obtain one here. */ if( rc==SQLITE_OK && seenZeroNoIN==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); + WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0); } @@ -4241,7 +4241,7 @@ static int whereLoopAddOr( sSubBuild = *pBuilder; sSubBuild.pOrSet = &sCur; - WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm)); for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; @@ -4258,9 +4258,9 @@ static int whereLoopAddOr( } sCur.n = 0; #ifdef WHERETRACE_ENABLED - WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n", (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); - if( sqlite3WhereTrace & 0x400 ){ + if( sqlite3WhereTrace & 0x20000 ){ sqlite3WhereClausePrint(sSubBuild.pWC); } #endif @@ -4322,7 +4322,7 @@ static int whereLoopAddOr( pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } - WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm)); } } return rc; @@ -5337,7 +5337,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->cId = '0'; #endif #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("whereShortCut() used to compute solution\n"); } #endif @@ -5467,7 +5467,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( } } if( pTerm drop loop %c not used\n", pLoop->cId)); + WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId)); notReady &= ~pLoop->maskSelf; for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ @@ -5527,7 +5527,7 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( testcase( pItem->fg.jointype & JT_LEFT ); pLoop->wsFlags |= WHERE_BLOOMFILTER; pLoop->wsFlags &= ~WHERE_IDX_ONLY; - WHERETRACE(0xffff, ( + WHERETRACE(0xffffffff, ( "-> use Bloom-filter on loop %c because there are ~%.1e " "lookups into %s which has only ~%.1e rows\n", pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName, @@ -5894,13 +5894,13 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0xffff ){ + if( sqlite3WhereTrace & 0xffffffff ){ sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); if( wctrlFlags & WHERE_USE_LIMIT ){ sqlite3DebugPrintf(", limit: %d", iAuxArg); } sqlite3DebugPrintf(")\n"); - if( sqlite3WhereTrace & 0x100 ){ + if( sqlite3WhereTrace & 0x8000 ){ Select sSelect; memset(&sSelect, 0, sizeof(sSelect)); sSelect.selFlags = SF_WhereBegin; @@ -5910,10 +5910,10 @@ WhereInfo *sqlite3WhereBegin( sSelect.pEList = pResultSet; sqlite3TreeViewSelect(0, &sSelect, 0); } - } - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ - sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); - sqlite3WhereClausePrint(sWLB.pWC); + if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */ + sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); + } } #endif @@ -5929,7 +5929,7 @@ WhereInfo *sqlite3WhereBegin( ** loops will be built using the revised truthProb values. */ if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); - WHERETRACE(0xffff, + WHERETRACE(0xffffffff, ("**** Redo all loop computations due to" " TERM_HIGHTRUTH changes ****\n")); while( pWInfo->pLoops ){ @@ -6015,11 +6015,11 @@ WhereInfo *sqlite3WhereBegin( } #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */ sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n"); sqlite3WhereClausePrint(sWLB.pWC); } - WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n")); #endif pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; diff --git a/src/wherecode.c b/src/wherecode.c index e36d1c9964..c731cc82f4 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -359,7 +359,7 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ pTerm->wtFlags |= TERM_CODED; } #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ sqlite3DebugPrintf("DISABLE-"); sqlite3WhereTermPrint(pTerm, (int)(pTerm - (pTerm->pWC->a))); } @@ -1346,13 +1346,15 @@ Bitmask sqlite3WhereCodeOneLoopStart( pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); -#if WHERETRACE_ENABLED /* 0x20800 */ - if( sqlite3WhereTrace & 0x800 ){ +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", iLevel, pWInfo->nLevel, (u64)notReady, pLevel->iFrom); - sqlite3WhereLoopPrint(pLoop, pWC); + if( sqlite3WhereTrace & 0x1000 ){ + sqlite3WhereLoopPrint(pLoop, pWC); + } } - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ if( iLevel==0 ){ sqlite3DebugPrintf("WHERE clause being coded:\n"); sqlite3TreeViewExpr(0, pWInfo->pWhere, 0); @@ -2276,7 +2278,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( } /* Loop through table entries that match term pOrTerm. */ ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1)); - WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + WHERETRACE(0xffffffff, ("Subplan for OR-clause:\n")); pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, 0, WHERE_OR_SUBCLAUSE, iCovCur); assert( pSubWInfo || pParse->nErr ); @@ -2513,12 +2515,12 @@ Bitmask sqlite3WhereCodeOneLoopStart( } #endif } -#ifdef WHERETRACE_ENABLED /* 0xffff */ +#ifdef WHERETRACE_ENABLED /* 0xffffffff */ if( sqlite3WhereTrace ){ VdbeNoopComment((v, "WhereTerm[%d] (%p) priority=%d", pWC->nTerm-j, pTerm, iLoop)); } - if( sqlite3WhereTrace & 0x800 ){ + if( sqlite3WhereTrace & 0x4000 ){ sqlite3DebugPrintf("Coding auxiliary constraint:\n"); sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); } @@ -2547,8 +2549,8 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( pTerm->leftCursor!=iCur ) continue; if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ) continue; pE = pTerm->pExpr; -#ifdef WHERETRACE_ENABLED /* 0x800 */ - if( sqlite3WhereTrace & 0x800 ){ +#ifdef WHERETRACE_ENABLED /* 0x4001 */ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ sqlite3DebugPrintf("Coding transitive constraint:\n"); sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); } @@ -2663,13 +2665,13 @@ Bitmask sqlite3WhereCodeOneLoopStart( } } -#if WHERETRACE_ENABLED /* 0x20800 */ - if( sqlite3WhereTrace & 0x20000 ){ +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x4000 ){ sqlite3DebugPrintf("All WHERE-clause terms after coding level %d:\n", iLevel); sqlite3WhereClausePrint(pWC); } - if( sqlite3WhereTrace & 0x800 ){ + if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("End Coding level %d: notReady=%llx\n", iLevel, (u64)pLevel->notReady); } From 75fba2fd28bd14c9b3780171063fdb0ff69dec89 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 30 Nov 2022 20:22:31 +0000 Subject: [PATCH 114/282] Fix an over-zealous assert() reported by Yong Heng. FossilOrigin-Name: 6ee61f8cede4998f0c838d6506b058c6b09f34b3d7f30ed296100785c93f8d00 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 172dd44886..1410a73de2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\srationalize\sthe\sbits\sassociated\swith\s".wheretrace".\s\sProvide\na\sdecoder\skey\sin\ssqliteInt.h\sfor\swhat\seach\sbit\sis\sintended\sto\sdo. -D 2022-11-30T19:11:31.801 +C Fix\san\sover-zealous\sassert()\sreported\sby\sYong\sHeng. +D 2022-11-30T20:22:31.967 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -721,7 +721,7 @@ F src/vdbe.c 00648bd76fb2145c2d890312112bda446569da64ca70aaf1f2282cc6c2b22d14 F src/vdbe.h 58675f47dcf3105bab182c3ad3726efd60ffd003e954386904ac9107d0d2b743 F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f F src/vdbeapi.c 1e8713d0b653acb43cd1bdf579c40e005c4844ea90f414f065946a83db3c27fb -F src/vdbeaux.c 87684b89877eae0c58c78b340bb5356aa1c8fb1dd650b29410c8b745aeeb20b5 +F src/vdbeaux.c 8ebe337e82d99cf3b01cd4fd67103a5b0054d53fee8456b90dbeba46ebf97ceb F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2eade7c7b17a186735c72974c11a34798a08364861d0f307e897ba765c0a93c7 -R 017a8b97f0f146b3506618eca7867206 +P 8ec361695a107a94f2cf6a7fe509656a99d85d49bd7c74133c69903f059a2675 +R 2a3a42c549d849993e0ed51d258499d0 U drh -Z b8ffdae4ce7e302410b9546e9e5b50f2 +Z 19ee33e552ce9650229a03299d6e09f2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 55309e681a..c50bf0179d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ec361695a107a94f2cf6a7fe509656a99d85d49bd7c74133c69903f059a2675 \ No newline at end of file +6ee61f8cede4998f0c838d6506b058c6b09f34b3d7f30ed296100785c93f8d00 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 2e5e769d74..fd196f37bc 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1457,7 +1457,7 @@ void sqlite3VdbeAppendP4(Vdbe *p, void *pP4, int n){ if( p->db->mallocFailed ){ freeP4(p->db, n, pP4); }else{ - assert( pP4!=0 ); + assert( pP4!=0 || n==P4_DYNAMIC ); assert( p->nOp>0 ); pOp = &p->aOp[p->nOp-1]; assert( pOp->p4type==P4_NOTUSED ); From 85ec20ac66d0a5fe8d6ae4769216bd06338d9a9f Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 30 Nov 2022 21:18:23 +0000 Subject: [PATCH 115/282] Add a testcase() macro to verify that the case of a NOT NULL error message hitting the string length limit. FossilOrigin-Name: 91f50964c10fb12d889bda7d597d8edf475d97d2d8b534b4400e0fed1d753c6a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/insert.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 1410a73de2..b6571cb9ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sover-zealous\sassert()\sreported\sby\sYong\sHeng. -D 2022-11-30T20:22:31.967 +C Add\sa\stestcase()\smacro\sto\sverify\sthat\sthe\scase\sof\sa\sNOT\sNULL\serror\smessage\nhitting\sthe\sstring\slength\slimit. +D 2022-11-30T21:18:23.655 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -604,7 +604,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 90a32bc7faa755cd5292ade21d2b3c6edba8fd1d70754a364caccabfde2c3bb2 +F src/insert.c 1b11a2e33ee52db93c02fddac67e39d00161d61b69fac2675b82f2aa68c1b61c F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 25663175950c5c4404b9377840b7b4c6fe5c53b415caf43634c62f442c02a9a7 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8ec361695a107a94f2cf6a7fe509656a99d85d49bd7c74133c69903f059a2675 -R 2a3a42c549d849993e0ed51d258499d0 +P 6ee61f8cede4998f0c838d6506b058c6b09f34b3d7f30ed296100785c93f8d00 +R ca2f65a200c7997e977e891739f47e7d U drh -Z 19ee33e552ce9650229a03299d6e09f2 +Z 02ee85efaa582407b1d21e8a02ad86a6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c50bf0179d..46eb4d34cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ee61f8cede4998f0c838d6506b058c6b09f34b3d7f30ed296100785c93f8d00 \ No newline at end of file +91f50964c10fb12d889bda7d597d8edf475d97d2d8b534b4400e0fed1d753c6a \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 1f6cdffcc3..65d11bee54 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1793,6 +1793,7 @@ void sqlite3GenerateConstraintChecks( case OE_Fail: { char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, pCol->zCnName); + testcase( zMsg==0 && db->mallocFailed==0 ); sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, iReg); sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); From 2b2199570da5d8e3e6a5cf1caae97a66b705124f Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 1 Dec 2022 03:55:28 +0000 Subject: [PATCH 116/282] Expand "sqlite3_vfs*" JS-to-WASM function argument conversions to accept VFS names (JS strings) and capi.sqlite3_vfs instances. Implement sqlite3_js_vfs_create_file() to facilitate creation of file-upload features which store the file in VFS-specific storage (where possible, e.g. "unix" and "opfs" VFSes). Correct an argument type check in the SQLite3Error and WasmAllocError constructors. FossilOrigin-Name: e1009b16d351b23676ad7bffab0c91b373a92132eb855c9af61991b50cd237ed --- ext/wasm/api/sqlite3-api-glue.js | 27 +++++- ext/wasm/api/sqlite3-api-prologue.js | 77 +++++++++++++++- ext/wasm/common/whwasmutil.js | 35 +++++++- ext/wasm/tester1.c-pp.js | 128 ++++++++++++++++++--------- manifest | 20 ++--- manifest.uuid | 2 +- 6 files changed, 225 insertions(+), 64 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 6c1fcae820..aa0d48af56 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -73,8 +73,27 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ('sqlite3_stmt*', aPtr) ('sqlite3_context*', aPtr) ('sqlite3_value*', aPtr) - ('sqlite3_vfs*', aPtr) - ('void*', aPtr); + ('void*', aPtr) + /** + `sqlite3_vfs*`: + + - v is-a string: use the result of sqlite3_vfs_find(v) but + throw if it returns 0. + - v is-a capi.sqlite3_vfs: use v.pointer. + - Else return the same as the `'*'` argument conversion. + */ + ('sqlite3_vfs*', (v)=>{ + if('string'===typeof v){ + const x = capi.sqlite3_vfs_find(v); + /* A NULL sqlite3_vfs pointer will be treated as the default + VFS in many contexts. We specifically do not want that + behavior here. */ + if(!x) sqlite3.SQLite3Error.toss("Unknown sqlite3_vfs name:",v); + return x; + }else if(v instanceof sqlite3.capi.sqlite3_vfs) v = v.pointer; + return aPtr(v); + }); + wasm.xWrap.resultAdapter('sqlite3*', aPtr) ('sqlite3_context*', aPtr) ('sqlite3_stmt*', aPtr) @@ -588,8 +607,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'version' ]){ for(const e of Object.entries(wasm.ctype[t])){ - // ^^^ [k,v] there triggers a buggy code transormation via one - // of the Emscripten-driven optimizers. + // ^^^ [k,v] there triggers a buggy code transformation via + // one of the Emscripten-driven optimizers. capi[e[0]] = e[1]; } } diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 31cd8aa539..0321472423 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -185,7 +185,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( constructor(...args){ if(1===args.length && __isInt(args[0])){ super(__rcStr(args[0])); - }else if(2===args.length && 'object'===typeof args){ + }else if(2===args.length && 'object'===typeof args[1]){ if(__isInt(args[0])) super(__rcStr(args[0]), args[1]); else super(...args); }else{ @@ -354,7 +354,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( message. */ constructor(...args){ - if(2===args.length && 'object'===typeof args){ + if(2===args.length && 'object'===typeof args[1]){ super(...args); }else if(args.length){ super(args.join(' ')); @@ -748,7 +748,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** Reports info about compile-time options using - sqlite_compileoption_get() and sqlite3_compileoption_used(). It + sqlite3_compileoption_get() and sqlite3_compileoption_used(). It has several distinct uses: If optName is an array then it is expected to be a list of @@ -973,7 +973,6 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_wasm_vfs_unlink", "int", "sqlite3_vfs*","string"] ]; - /** sqlite3.wasm.pstack (pseudo-stack) holds a special-case stack-style allocator intended only for use with _small_ data of @@ -1327,6 +1326,76 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( "bytes for sqlite3_aggregate_context()") : 0); }; + + /** + Creates a file using the storage appropriate for the given + sqlite3_vfs. The first argument may be a VFS name (JS string + only, NOT a WASM C-string), WASM-managed `sqlite3_vfs*`, or + a capi.sqlite3_vfs instance. Pass 0 (a NULL pointer) to use the + default VFS. If passed a string which does not resolve using + sqlite3_vfs_find(), an exception is thrown. (Note that a WASM + C-string is not accepted because it is impossible to + distinguish from a C-level `sqlite3_vfs*`.) + + The second argument, the filename, must be a JS or WASM C-string. + + The 3rd may either be falsy, a valid WASM memory pointer, or a + Uint8Array. The 4th must be the length, in bytes, of the data + array to copy. If the 3rd argument is a Uint8Array and the 4th is + not a positive integer then the 4th defaults to the array's + byteLength value. + + If data is falsy then a file is created with dataLen bytes filled + with uninitialized data (whatever truncate() leaves there). If + data is not falsy then a file is created or truncated and it is + filled with the first dataLen bytes of the data source. + + Throws if any arguments are invalid or if creating or writing to + the file fails. + + Note that most VFSes do _not_ automatically create directory + parts of filenames, nor do all VFSes have a concept of + directories. If the given filename is not valid for the given + VFS, an exception will be thrown. This function exists primarily + to assist in implementing file-upload capability, with the caveat + that clients must have some idea of the VFS into which they want + to upload and that VFS must support the operation. + + VFS-specific notes: + + - "memdb" and "kvvfs": results are undefined. + + - "unix" and related: will use the WASM build's equivalent of the + POSIX I/O APIs. + + - "opfs": if available, uses OPFS storage and _does_ create + directory parts of the filename. + */ + capi.sqlite3_js_vfs_create_file = function(vfs, filename, data, dataLen){ + let pData; + if(!data) pData = 0; + else if(wasm.isPtr(data)){ + pData = data; + }else if(data instanceof Uint8Array){ + pData = wasm.allocFromTypedArray(data); + if(arguments.length<4 || !util.isInt32(dataLen) || dataLen<0){ + dataLen = data.byteLength; + } + }else{ + SQLite3Error.toss("Invalid 3rd argument type for sqlite3_js_vfs_create_file()."); + } + if(!util.isInt32(dataLen) || dataLen<0){ + wasm.dealloc(pData); + SQLite3Error.toss("Invalid 4th argument for sqlite3_js_vfs_create_file()."); + } + try{ + const rc = wasm.sqlite3_wasm_vfs_create_file(vfs, filename, pData, dataLen); + if(rc) SQLite3Error.toss("Creation of file failed with sqlite3 result code", + capi.sqlite3_js_rc_str(rc)); + }finally{ + wasm.dealloc(pData); + } + }; if( util.isUIThread() ){ /* Features specific to the main window thread... */ diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 7e5e7981f7..0190433773 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -1313,7 +1313,7 @@ self.WhWasmUtilInstaller = function(target){ const __xResultAdapterCheck = (t)=>xcv.result[t] || toss("Result adapter not found:",t); - + cache.xWrap.convertArg = (t,v)=>__xArgAdapterCheck(t)(v); cache.xWrap.convertResult = (t,v)=>(null===t ? v : (t ? __xResultAdapterCheck(t)(v) : undefined)); @@ -1442,7 +1442,7 @@ self.WhWasmUtilInstaller = function(target){ exception. Clients may map their own result and argument adapters using - xWrap.resultAdapter() and xWrap.argAdaptor(), noting that not all + xWrap.resultAdapter() and xWrap.argAdapter(), noting that not all type conversions are valid for both arguments _and_ result types as they often have different memory ownership requirements. @@ -1497,7 +1497,7 @@ self.WhWasmUtilInstaller = function(target){ }; }/*xWrap()*/; - /** Internal impl for xWrap.resultAdapter() and argAdaptor(). */ + /** Internal impl for xWrap.resultAdapter() and argAdapter(). */ const __xAdapter = function(func, argc, typeName, adapter, modeName, xcvPart){ if('string'===typeof typeName){ if(1===argc) return xcvPart[typeName]; @@ -1575,7 +1575,7 @@ self.WhWasmUtilInstaller = function(target){ */ target.xWrap.argAdapter = function f(typeName, adapter){ return __xAdapter(f, arguments.length, typeName, adapter, - 'argAdaptor()', xcv.arg); + 'argAdapter()', xcv.arg); }; /** @@ -1601,6 +1601,33 @@ self.WhWasmUtilInstaller = function(target){ return target.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]); }; + /** + This function is ONLY exposed in the public API to facilitate + testing. It should not be used in application-level code, only + in test code. + + Expects to be given (typeName, value) and returns a conversion + of that value as has been registered using argAdapter(). + It throws if no adapter is found. + + ACHTUNG: the adapter may require that a scopedAllocPush() is + active and it may allocate memory within that scope. + */ + target.xWrap.testConvertArg = cache.xWrap.convertArg; + /** + This function is ONLY exposed in the public API to facilitate + testing. It should not be used in application-level code, only + in test code. + + Expects to be given (typeName, value) and returns a conversion + of that value as has been registered using resultAdapter(). + It throws if no adapter is found. + + ACHTUNG: the adapter may allocate memory which the caller may need + to know how to free. + */ + target.xWrap.testConvertResult = cache.xWrap.convertResult; + return target; }; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 08f5480b54..1278e772f9 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1236,8 +1236,19 @@ self.sqlite3InitModule = sqlite3InitModule; //log("vfsList =",vfsList); for(const v of vfsList){ T.assert('string' === typeof v) - .assert(capi.sqlite3_vfs_find(v) > 0); + .assert(wasm.isPtr( + capi.sqlite3_vfs_find(v) + )); } + // While we have vfsList handy, let's verify... + wasm.scopedAllocCall(()=>{ + const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v); + T.assert(wasm.isPtr(vfsArg(vfsList[0]))); + const pVfs = capi.sqlite3_vfs_find(vfsList[0]); + const vfs = new capi.sqlite3_vfs(pVfs); + T.assert(wasm.isPtr(vfsArg(vfs))); + vfs.dispose(); + }); } /** Trivia: the magic db name ":memory:" does not actually use the @@ -1784,14 +1795,15 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('OPFS (Worker thread only and only in supported browsers)', - (sqlite3)=>{return !!sqlite3.opfs}) + (sqlite3)=>!!sqlite3.opfs) .t({ - name: 'OPFS sanity checks', + name: 'OPFS db sanity checks', test: async function(sqlite3){ - const filename = 'sqlite3-tester1.db'; - const pVfs = capi.sqlite3_vfs_find('opfs'); + const filename = this.opfsDbFile = 'sqlite3-tester1.db'; + const pVfs = this.opfsVfs = capi.sqlite3_vfs_find('opfs'); T.assert(pVfs); - const unlink = (fn=filename)=>wasm.sqlite3_wasm_vfs_unlink(pVfs,fn); + const unlink = this.opfsUnlink = + (fn=filename)=>{wasm.sqlite3_wasm_vfs_unlink(pVfs,fn)}; unlink(); let db = new sqlite3.oo1.OpfsDb(filename); try { @@ -1808,42 +1820,76 @@ self.sqlite3InitModule = sqlite3InitModule; db.close(); unlink(); } - - if(sqlite3.opfs){ - // Sanity-test sqlite3_wasm_vfs_create_file()... - const opfs = sqlite3.opfs; - const fSize = 1379; - let sh; - try{ - T.assert(!(await opfs.entryExists(filename))); - let rc = wasm.sqlite3_wasm_vfs_create_file( - pVfs, filename, null, fSize - ); - T.assert(0===rc) - .assert(await opfs.entryExists(filename)); - const fh = await opfs.rootDirectory.getFileHandle(filename); - sh = await fh.createSyncAccessHandle(); - T.assert(fSize === await sh.getSize()); - }finally{ - if(sh) await sh.close(); - unlink(); - } - - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); - } } - }/*OPFS sanity checks*/) + }/*OPFS db sanity checks*/) + .t({ + name: 'OPFS utility APIs and sqlite3_js_vfs_create_file()', + test: async function(sqlite3){ + const filename = this.opfsDbFile; + const pVfs = this.opfsVfs; + const unlink = this.opfsUnlink; + T.assert(filename && pVfs && !!unlink); + unlink(); + // Sanity-test sqlite3_js_vfs_create_file()... + /************************************************************** + ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended + for client-side use. It is only for this project's own + internal use. Its APIs are subject to change or removal at + any time. + ***************************************************************/ + const opfs = sqlite3.opfs; + const fSize = 1379; + let sh; + try{ + T.assert(!(await opfs.entryExists(filename))); + capi.sqlite3_js_vfs_create_file( + pVfs, filename, null, fSize + ); + T.assert(await opfs.entryExists(filename)); + let fh = await opfs.rootDirectory.getFileHandle(filename); + sh = await fh.createSyncAccessHandle(); + T.assert(fSize === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + T.assert(!(await opfs.entryExists(filename))); + + const ba = new Uint8Array([1,2,3,4,5]); + capi.sqlite3_js_vfs_create_file( + "opfs", filename, ba + ); + T.assert(await opfs.entryExists(filename)); + fh = await opfs.rootDirectory.getFileHandle(filename); + sh = await fh.createSyncAccessHandle(); + T.assert(ba.byteLength === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + + T.mustThrowMatching(()=>{ + capi.sqlite3_js_vfs_create_file( + "no-such-vfs", filename, ba + ); + }, "Unknown sqlite3_vfs name: no-such-vfs"); + }finally{ + if(sh) await sh.close(); + unlink(); + } + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } + }/*OPFS util sanity checks*/) ;/* end OPFS tests */ //////////////////////////////////////////////////////////////////////// diff --git a/manifest b/manifest index b6571cb9ec..06b75100f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stestcase()\smacro\sto\sverify\sthat\sthe\scase\sof\sa\sNOT\sNULL\serror\smessage\nhitting\sthe\sstring\slength\slimit. -D 2022-11-30T21:18:23.655 +C Expand\s"sqlite3_vfs*"\sJS-to-WASM\sfunction\sargument\sconversions\sto\saccept\sVFS\snames\s(JS\sstrings)\sand\scapi.sqlite3_vfs\sinstances.\sImplement\ssqlite3_js_vfs_create_file()\sto\sfacilitate\screation\sof\sfile-upload\sfeatures\swhich\sstore\sthe\sfile\sin\sVFS-specific\sstorage\s(where\spossible,\se.g.\s"unix"\sand\s"opfs"\sVFSes).\sCorrect\san\sargument\stype\scheck\sin\sthe\sSQLite3Error\sand\sWasmAllocError\sconstructors. +D 2022-12-01T03:55:28.175 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,9 +503,9 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js 03c40b65530d67bb2748b7380aea5fd1534500f812b76a6b401066dcd7fc4116 +F ext/wasm/api/sqlite3-api-glue.js 5468ba750e081c55ff3c572e3ae47dd415be73907ce40117fb121f37f4d0649e F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-prologue.js e63bcd1d1942d3313bd11e76ac1fccdd3e34ba54a48b8c8296db8dd892705dbc +F ext/wasm/api/sqlite3-api-prologue.js 04f789bab878ea67be8b18dd4292011bbfeb78ebad79eb79c591136332bd469a F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 @@ -521,7 +521,7 @@ F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c07 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/common/testing.css 35889709547d89a6109ff83b25c11bbc91d8dd43aab8722e428655ca98880a06 -F ext/wasm/common/whwasmutil.js 16a592d5c304a2d268ca1c28e08a5b029a2f3cbe10af78dbc3456cfc9e3559d1 +F ext/wasm/common/whwasmutil.js dbe625a22bf0962cde1f958f2be604d27d2f97ee1b4e6ee0f19c6480aa36aeed F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js c39594bb1a0272a08a1544277a29c1ed920b4c4877611591e7c3185744ae2431 +F ext/wasm/tester1.c-pp.js b6a8b9d8e8b070d40c140a68072bf0aa41819fe37fb5bbb9391966a3cbc154fa F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ee61f8cede4998f0c838d6506b058c6b09f34b3d7f30ed296100785c93f8d00 -R ca2f65a200c7997e977e891739f47e7d -U drh -Z 02ee85efaa582407b1d21e8a02ad86a6 +P 91f50964c10fb12d889bda7d597d8edf475d97d2d8b534b4400e0fed1d753c6a +R e4bc31bd87c945bd19fac07fc1661145 +U stephan +Z a323b013b8116acefa1b2ce6dcb45815 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 46eb4d34cb..55e974a247 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91f50964c10fb12d889bda7d597d8edf475d97d2d8b534b4400e0fed1d753c6a \ No newline at end of file +e1009b16d351b23676ad7bffab0c91b373a92132eb855c9af61991b50cd237ed \ No newline at end of file From 9ec1a7a7a07be84a597fb6d7a455b52275f8eab2 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 1 Dec 2022 04:45:51 +0000 Subject: [PATCH 117/282] Reformulate and simplify some JS tests related to the previous checkin. FossilOrigin-Name: 9ea2d3dcf798393a7fd231e199c0e2c6302949fe2a7f2573178fb0e50c78a2f4 --- ext/wasm/api/sqlite3-api-prologue.js | 3 ++- ext/wasm/tester1.c-pp.js | 22 +++++++++------------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 0321472423..a7dfa015ba 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -963,7 +963,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** Functions which are intended solely for API-internal use by the WASM components, not client code. These get installed into - sqlite3.wasm. + sqlite3.wasm. Some of them get exposed to clients via variants + named sqlite3_js_...(). */ wasm.bindingSignatures.wasm = [ ["sqlite3_wasm_db_reset", "int", "sqlite3*"], diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 1278e772f9..b1d9cba2e9 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1232,22 +1232,18 @@ self.sqlite3InitModule = sqlite3InitModule; if(1){ const vfsList = capi.sqlite3_js_vfs_list(); T.assert(vfsList.length>1); - T.assert('string'===typeof vfsList[0]); //log("vfsList =",vfsList); - for(const v of vfsList){ - T.assert('string' === typeof v) - .assert(wasm.isPtr( - capi.sqlite3_vfs_find(v) - )); - } - // While we have vfsList handy, let's verify... wasm.scopedAllocCall(()=>{ const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v); - T.assert(wasm.isPtr(vfsArg(vfsList[0]))); - const pVfs = capi.sqlite3_vfs_find(vfsList[0]); - const vfs = new capi.sqlite3_vfs(pVfs); - T.assert(wasm.isPtr(vfsArg(vfs))); - vfs.dispose(); + for(const v of vfsList){ + T.assert('string' === typeof v); + const pVfs = capi.sqlite3_vfs_find(v); + T.assert(wasm.isPtr(pVfs)) + .assert(pVfs===vfsArg(v)); + const vfs = new capi.sqlite3_vfs(pVfs); + try { T.assert(vfsArg(vfs)===pVfs) } + finally{ vfs.dispose() } + } }); } /** diff --git a/manifest b/manifest index 06b75100f2..c351539155 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expand\s"sqlite3_vfs*"\sJS-to-WASM\sfunction\sargument\sconversions\sto\saccept\sVFS\snames\s(JS\sstrings)\sand\scapi.sqlite3_vfs\sinstances.\sImplement\ssqlite3_js_vfs_create_file()\sto\sfacilitate\screation\sof\sfile-upload\sfeatures\swhich\sstore\sthe\sfile\sin\sVFS-specific\sstorage\s(where\spossible,\se.g.\s"unix"\sand\s"opfs"\sVFSes).\sCorrect\san\sargument\stype\scheck\sin\sthe\sSQLite3Error\sand\sWasmAllocError\sconstructors. -D 2022-12-01T03:55:28.175 +C Reformulate\sand\ssimplify\ssome\sJS\stests\srelated\sto\sthe\sprevious\scheckin. +D 2022-12-01T04:45:51.580 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,7 +505,7 @@ F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a9578430388 F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js 5468ba750e081c55ff3c572e3ae47dd415be73907ce40117fb121f37f4d0649e F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-prologue.js 04f789bab878ea67be8b18dd4292011bbfeb78ebad79eb79c591136332bd469a +F ext/wasm/api/sqlite3-api-prologue.js 603d074cf7824ebfe4014a475fe59d88134b03ca75f3e00c42b8ddc8a474d57b F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js b6a8b9d8e8b070d40c140a68072bf0aa41819fe37fb5bbb9391966a3cbc154fa +F ext/wasm/tester1.c-pp.js b6cad0c527efedd373698fccb1c56fe187c008afc08be955616b099588b5505c F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 91f50964c10fb12d889bda7d597d8edf475d97d2d8b534b4400e0fed1d753c6a -R e4bc31bd87c945bd19fac07fc1661145 +P e1009b16d351b23676ad7bffab0c91b373a92132eb855c9af61991b50cd237ed +R 635926f8422902f122ea2d3b331c7a8f U stephan -Z a323b013b8116acefa1b2ce6dcb45815 +Z ff69e84e8c5af74f8df80a8186f78bea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 55e974a247..6149c38715 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1009b16d351b23676ad7bffab0c91b373a92132eb855c9af61991b50cd237ed \ No newline at end of file +9ea2d3dcf798393a7fd231e199c0e2c6302949fe2a7f2573178fb0e50c78a2f4 \ No newline at end of file From 919dbc846a097a1f044cfd2c7003c4f256ccb155 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 1 Dec 2022 15:22:03 +0000 Subject: [PATCH 118/282] sqlite3_js_create_file() now accepts an ArrayBuffer data source. Add test for OPFS-based export/re-import. The (sqlite3*) argument converter now optionally accepts sqlite3.oo1.DB instances. FossilOrigin-Name: 14a84b67fb17e16a5691ea4bf7f374123ac73a361a5d3d0efca53788d2001e3a --- ext/wasm/api/sqlite3-api-glue.js | 7 +++-- ext/wasm/api/sqlite3-api-prologue.js | 40 +++++++++++++++++----------- ext/wasm/tester1.c-pp.js | 36 ++++++++++++++++++++++++- manifest | 16 +++++------ manifest.uuid | 2 +- 5 files changed, 74 insertions(+), 27 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index aa0d48af56..e60baeb7f3 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -68,12 +68,15 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `sqlite3_vfs*` via capi.sqlite3_vfs.pointer. */ const aPtr = wasm.xWrap.argAdapter('*'); - wasm.xWrap.argAdapter('sqlite3*', aPtr) - ('sqlite3_filename', aPtr) + wasm.xWrap.argAdapter('sqlite3_filename', aPtr) ('sqlite3_stmt*', aPtr) ('sqlite3_context*', aPtr) ('sqlite3_value*', aPtr) ('void*', aPtr) + ('sqlite3*', (v)=>{ + if(sqlite3.oo1 && v instanceof sqlite3.oo1.DB) v = v.pointer; + return aPtr(v); + }) /** `sqlite3_vfs*`: diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index a7dfa015ba..1bc1c1559f 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1258,7 +1258,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** A convenience wrapper around sqlite3_serialize() which serializes - the given `sqlite3*` pointer to a Uint8Array. + the given `sqlite3*` pointer to a Uint8Array. The first argument + may be either an `sqlite3*` or an sqlite3.oo1.DB instance. On success it returns a Uint8Array. If the schema is empty, an empty array is returned. @@ -1269,6 +1270,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( On error it throws with a description of the problem. */ capi.sqlite3_js_db_export = function(pDb, schema=0){ + pDb = wasm.xWrap.testConvertArg('sqlite3*', pDb); if(!pDb) toss3('Invalid sqlite3* argument.'); if(!wasm.bigIntEnabled) toss3('BigInt64 support is not enabled.'); const scope = wasm.scopedAllocPush(); @@ -1340,11 +1342,11 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( The second argument, the filename, must be a JS or WASM C-string. - The 3rd may either be falsy, a valid WASM memory pointer, or a - Uint8Array. The 4th must be the length, in bytes, of the data - array to copy. If the 3rd argument is a Uint8Array and the 4th is - not a positive integer then the 4th defaults to the array's - byteLength value. + The 3rd may either be falsy, a valid WASM memory pointer, an + ArrayBuffer, or a Uint8Array. The 4th must be the length, in + bytes, of the data array to copy. If the 3rd argument is an + ArrayBuffer or Uint8Array and the 4th is not a positive integer + then the 4th defaults to the array's byteLength value. If data is falsy then a file is created with dataLen bytes filled with uninitialized data (whatever truncate() leaves there). If @@ -1364,7 +1366,9 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( VFS-specific notes: - - "memdb" and "kvvfs": results are undefined. + - "memdb": results are undefined. + + - "kvvfs": results are undefined. - "unix" and related: will use the WASM build's equivalent of the POSIX I/O APIs. @@ -1374,16 +1378,22 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( */ capi.sqlite3_js_vfs_create_file = function(vfs, filename, data, dataLen){ let pData; - if(!data) pData = 0; - else if(wasm.isPtr(data)){ - pData = data; - }else if(data instanceof Uint8Array){ - pData = wasm.allocFromTypedArray(data); - if(arguments.length<4 || !util.isInt32(dataLen) || dataLen<0){ - dataLen = data.byteLength; + if(data){ + if(wasm.isPtr(data)){ + pData = data; + }else if(data instanceof ArrayBuffer){ + data = new Uint8Array(data); + } + if(data instanceof Uint8Array){ + pData = wasm.allocFromTypedArray(data); + if(arguments.length<4 || !util.isInt32(dataLen) || dataLen<0){ + dataLen = data.byteLength; + } + }else{ + SQLite3Error.toss("Invalid 3rd argument type for sqlite3_js_vfs_create_file()."); } }else{ - SQLite3Error.toss("Invalid 3rd argument type for sqlite3_js_vfs_create_file()."); + pData = 0; } if(!util.isInt32(dataLen) || dataLen<0){ wasm.dealloc(pData); diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index b1d9cba2e9..9a65c8b232 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1160,7 +1160,8 @@ self.sqlite3InitModule = sqlite3InitModule; .mustThrowMatching(()=>db.pointer=1, /read-only/) .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) .assert('main'===db.dbName(0)) - .assert('string' === typeof db.dbVfsName()); + .assert('string' === typeof db.dbVfsName()) + .assert(db.pointer === wasm.xWrap.testConvertArg('sqlite3*',db)); // Custom db error message handling via sqlite3_prepare_v2/v3() let rc = capi.sqlite3_prepare_v3(db.pointer, {/*invalid*/}, -1, 0, null, null); T.assert(capi.SQLITE_MISUSE === rc) @@ -1782,6 +1783,17 @@ self.sqlite3InitModule = sqlite3InitModule; db = new JDb(filename); db.exec('insert into kvvfs(a) values(4),(5),(6)'); T.assert(6 === db.selectValue('select count(*) from kvvfs')); + + // Check import/export of db... + if(0){ + // does not yet work with kvvfs for unknown reasons... + const exp = capi.sqlite3_js_db_export(db); + db.close(); + unlink(); + capi.sqlite3_js_vfs_create_file("kvvfs", filename, exp); + db = new JDb(filename); + T.assert(6 === db.selectValue('select count(*) from kvvfs')); + } }finally{ db.close(); unlink(); @@ -1812,12 +1824,31 @@ self.sqlite3InitModule = sqlite3InitModule; db = new sqlite3.oo1.OpfsDb(filename); db.exec('insert into p(a) values(4),(5),(6)'); T.assert(6 === db.selectValue('select count(*) from p')); + this.opfsDbExport = capi.sqlite3_js_db_export(db); + T.assert(this.opfsDbExport instanceof Uint8Array) + .assert(this.opfsDbExport.byteLength>0 + && 0===this.opfsDbExport.byteLength % 512); }finally{ db.close(); unlink(); } } }/*OPFS db sanity checks*/) + .t({ + name: 'OPFS export/import', + test: async function(sqlite3){ + let db; + try { + const exp = this.opfsDbExport; + delete this.opfsDbExport; + capi.sqlite3_js_vfs_create_file("opfs", this.opfsDbFile, exp); + const db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')); + }finally{ + if(db) db.close(); + } + } + }/*OPFS export/import*/) .t({ name: 'OPFS utility APIs and sqlite3_js_vfs_create_file()', test: async function(sqlite3){ @@ -1825,6 +1856,9 @@ self.sqlite3InitModule = sqlite3InitModule; const pVfs = this.opfsVfs; const unlink = this.opfsUnlink; T.assert(filename && pVfs && !!unlink); + delete this.opfsDbFile; + delete this.opfsVfs; + delete this.opfsUnlink; unlink(); // Sanity-test sqlite3_js_vfs_create_file()... /************************************************************** diff --git a/manifest b/manifest index c351539155..687edef534 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reformulate\sand\ssimplify\ssome\sJS\stests\srelated\sto\sthe\sprevious\scheckin. -D 2022-12-01T04:45:51.580 +C sqlite3_js_create_file()\snow\saccepts\san\sArrayBuffer\sdata\ssource.\sAdd\stest\sfor\sOPFS-based\sexport/re-import.\sThe\s(sqlite3*)\sargument\sconverter\snow\soptionally\saccepts\ssqlite3.oo1.DB\sinstances. +D 2022-12-01T15:22:03.961 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,9 +503,9 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js 5468ba750e081c55ff3c572e3ae47dd415be73907ce40117fb121f37f4d0649e +F ext/wasm/api/sqlite3-api-glue.js b528207ba43f7740d1ade623f3f6b08a49f44ce7e9126915b78e1818c2466d8e F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 -F ext/wasm/api/sqlite3-api-prologue.js 603d074cf7824ebfe4014a475fe59d88134b03ca75f3e00c42b8ddc8a474d57b +F ext/wasm/api/sqlite3-api-prologue.js f8330a837cf008cae65083c356ebddb6a00054e94a2d2927f2828ad83d3112ce F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js b6cad0c527efedd373698fccb1c56fe187c008afc08be955616b099588b5505c +F ext/wasm/tester1.c-pp.js 9ae36e9b6055975348d7893f2d89ba7119f54e7e78f3d1c7007522d12c444165 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1009b16d351b23676ad7bffab0c91b373a92132eb855c9af61991b50cd237ed -R 635926f8422902f122ea2d3b331c7a8f +P 9ea2d3dcf798393a7fd231e199c0e2c6302949fe2a7f2573178fb0e50c78a2f4 +R 03a40b38b5a233d1d5761f06f6f34dbb U stephan -Z ff69e84e8c5af74f8df80a8186f78bea +Z 343e90b1bb6c198c8661d1f234560b23 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6149c38715..411cc90ecc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ea2d3dcf798393a7fd231e199c0e2c6302949fe2a7f2573178fb0e50c78a2f4 \ No newline at end of file +14a84b67fb17e16a5691ea4bf7f374123ac73a361a5d3d0efca53788d2001e3a \ No newline at end of file From 31892c28206a4516a077b9ab233379ab23ab9652 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 03:37:49 +0000 Subject: [PATCH 119/282] Remove extraneous/unused sqlite3.oo1.version object. Add httpd makefile target. FossilOrigin-Name: 8e4d30ac033a6d9019a7eeedfe788dc0120f565cef2ae8f09d2bf32eb94d8a33 --- ext/wasm/GNUmakefile | 4 ++++ ext/wasm/api/sqlite3-api-oo1.js | 4 ---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 31431352ea..8238dc895e 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -854,3 +854,7 @@ update-docs: endif # end /wasm docs ######################################################################## + +# Run local web server for the test/demo pages. +httpd: + althttpd -max-age 1 -enable-sab -page index.html diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 9544a96c6f..45c2ba913b 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -1759,10 +1759,6 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** The OO API's public namespace. */ sqlite3.oo1 = { - version: { - lib: capi.sqlite3_libversion(), - ooApi: "0.1" - }, DB, Stmt }/*oo1 object*/; diff --git a/manifest b/manifest index 687edef534..3fd905ca0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C sqlite3_js_create_file()\snow\saccepts\san\sArrayBuffer\sdata\ssource.\sAdd\stest\sfor\sOPFS-based\sexport/re-import.\sThe\s(sqlite3*)\sargument\sconverter\snow\soptionally\saccepts\ssqlite3.oo1.DB\sinstances. -D 2022-12-01T15:22:03.961 +C Remove\sextraneous/unused\ssqlite3.oo1.version\sobject.\sAdd\shttpd\smakefile\starget. +D 2022-12-02T03:37:49.317 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 85fb066f3ad17f9252deba2a3625736fc9e4172ab1494a73f74d65e21cf93387 +F ext/wasm/GNUmakefile 6b90a38f9d7b64e3cf1c494fea1d29ac175ab66eed816cdc6bb3164d94f16aed F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -504,7 +504,7 @@ F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js b528207ba43f7740d1ade623f3f6b08a49f44ce7e9126915b78e1818c2466d8e -F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8 +F ext/wasm/api/sqlite3-api-oo1.js c8b6c9ccb64cf93ca990ac689e98963735110aec21f98e04b55018f8e67b8147 F ext/wasm/api/sqlite3-api-prologue.js f8330a837cf008cae65083c356ebddb6a00054e94a2d2927f2828ad83d3112ce F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ea2d3dcf798393a7fd231e199c0e2c6302949fe2a7f2573178fb0e50c78a2f4 -R 03a40b38b5a233d1d5761f06f6f34dbb +P 14a84b67fb17e16a5691ea4bf7f374123ac73a361a5d3d0efca53788d2001e3a +R 731cae2fa3f5f500915aa9b0ca4c85de U stephan -Z 343e90b1bb6c198c8661d1f234560b23 +Z 40b7eebb887f929404345b830b46e1fc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 411cc90ecc..e3f1787a58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14a84b67fb17e16a5691ea4bf7f374123ac73a361a5d3d0efca53788d2001e3a \ No newline at end of file +8e4d30ac033a6d9019a7eeedfe788dc0120f565cef2ae8f09d2bf32eb94d8a33 \ No newline at end of file From cc70dfe679b61346d0cece11a1e70f97b14c1524 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 07:14:56 +0000 Subject: [PATCH 120/282] Expand JS tests for db export/import and document reason it cannot currently work with kvvfs. Fix a minor JS build dependencies bug. Update page title with PASS/FAIL prefix for tester1.js to improve overview when launching multiple test tabs. Add ability of tester1 should-run-test predicates to report why a given test is disabled. FossilOrigin-Name: 75f610d3a4cf3d972220f9abc27cdf5990451e3835ceb9cf66973934004dfc5c --- ext/wasm/GNUmakefile | 2 +- ext/wasm/api/sqlite3-api-prologue.js | 10 +- ext/wasm/api/sqlite3-wasm.c | 26 +++- ext/wasm/tester1-worker.html | 15 ++- ext/wasm/tester1.c-pp.js | 170 +++++++++++++++++---------- manifest | 20 ++-- manifest.uuid | 2 +- 7 files changed, 160 insertions(+), 85 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 8238dc895e..08c325a541 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -569,7 +569,7 @@ sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c # instead of building a shared copy of sqlite3-wasm.o. $(eval $(call call-make-pre-js,sqlite3,vanilla)) $(eval $(call call-make-pre-js,sqlite3,esm)) -$(sqlite3.js) $(sqlite3.mjs): $(MAKEFILE) $(sqlite3.wasm.obj) \ +$(sqlite3.js) $(sqlite3.mjs): $(MAKEFILE) $(sqlite3-wasm.c) \ $(EXPORTED_FUNCTIONS.api) $(sqlite3.js): $(pre-post-sqlite3.deps.vanilla) $(sqlite3.mjs): $(pre-post-sqlite3.deps.esm) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 1bc1c1559f..a99065663d 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1368,13 +1368,15 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( - "memdb": results are undefined. - - "kvvfs": results are undefined. + - "kvvfs": will fail with an I/O error due to strict internal + requirments of that VFS's xTruncate(). - "unix" and related: will use the WASM build's equivalent of the - POSIX I/O APIs. + POSIX I/O APIs. This will work so long as neither a specific + VFS nor the WASM environment imposes requirements which break it. - - "opfs": if available, uses OPFS storage and _does_ create - directory parts of the filename. + - "opfs": uses OPFS storage and creates directory parts of the + filename. */ capi.sqlite3_js_vfs_create_file = function(vfs, filename, data, dataLen){ let pData; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 67e97c4a03..6f214df347 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -950,14 +950,18 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** for use by the sqlite project's own JS/WASM bindings. ** ** Creates a new file using the I/O API of the given VFS, containing -** the given number of bytes of the given data. If the file exists, -** it is truncated to the given length and populated with the given +** the given number of bytes of the given data. If the file exists, it +** is truncated to the given length and populated with the given ** data. ** ** This function exists so that we can implement the equivalent of ** Emscripten's FS.createDataFile() in a VFS-agnostic way. This ** functionality is intended for use in uploading database files. ** +** Note that not all VFSes support this operation because they impose +** specific requirements on truncate and write sizes. e.g. kvvfs does +** not work with this. +** ** If pVfs is NULL, sqlite3_vfs_find(0) is used. ** ** If zFile is NULL, pVfs is NULL (and sqlite3_vfs_find(0) returns @@ -1004,11 +1008,17 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, ** it may have a buffer limit related to sqlite3's pager size, we ** conservatively write in 512-byte blocks (smallest page ** size). */; - + //fprintf(stderr, "pVfs=%p, zFilename=%s, nData=%d\n", pVfs, zFilename, nData); if( !pVfs ) pVfs = sqlite3_vfs_find(0); if( !pVfs || !zFilename || nData<0 ) return SQLITE_MISUSE; pVfs->xAccess(pVfs, zFilename, SQLITE_ACCESS_EXISTS, &fileExisted); rc = sqlite3OsOpenMalloc(pVfs, zFilename, &pFile, openFlags, &flagsOut); +#if 0 +# define RC fprintf(stderr,"create_file(%s,%s) @%d rc=%d\n", pVfs->zName, zFilename, __LINE__, rc); +#else +# define RC +#endif + RC; if(rc) return rc; pIo = pFile->pMethods; if( pIo->xLock ) { @@ -1019,19 +1029,25 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, ** xFileSize(), xTruncate(), and the like, and release the lock ** only if it was unlocked when the op was started. */ rc = pIo->xLock(pFile, SQLITE_LOCK_EXCLUSIVE); + RC; doUnlock = 0==rc; } - if( 0==rc) rc = pIo->xTruncate(pFile, nData); + if( 0==rc ){ + rc = pIo->xTruncate(pFile, nData); + RC; + } if( 0==rc && 0!=pData && nData>0 ){ while( 0==rc && nData>0 ){ const int n = nData>=blockSize ? blockSize : nData; rc = pIo->xWrite(pFile, pPos, n, (sqlite3_int64)(pPos - pData)); + RC; nData -= n; pPos += n; } if( 0==rc && nData>0 ){ assert( nDataxWrite(pFile, pPos, nData, (sqlite3_int64)(pPos - pData)); + RC; } } if( pIo->xUnlock && doUnlock!=0 ) pIo->xUnlock(pFile, SQLITE_LOCK_NONE); @@ -1039,6 +1055,8 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, if( rc!=0 && 0==fileExisted ){ pVfs->xDelete(pVfs, zFilename, 1); } + RC; +#undef RC return rc; } diff --git a/ext/wasm/tester1-worker.html b/ext/wasm/tester1-worker.html index 33ffa64186..ee03874df3 100644 --- a/ext/wasm/tester1-worker.html +++ b/ext/wasm/tester1-worker.html @@ -62,11 +62,16 @@ case 'error': logHtml('error', ...data.payload.args); break; - case 'test-result': - document.querySelector('#color-target').classList.add( - data.payload.pass ? 'tests-pass' : 'tests-fail' - ); - break; + case 'test-result':{ + document.querySelector('#color-target').classList.add( + data.payload.pass ? 'tests-pass' : 'tests-fail' + ); + const e = document.querySelector('title'); + e.innerText = ( + data.payload.pass ? 'PASS' : 'FAIL' + ) + ': ' + e.innerText; + break; + } default: logHtml('error',"Unhandled message:",data.type); }; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 9a65c8b232..1db8499ef6 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -118,8 +118,10 @@ self.sqlite3InitModule = sqlite3InitModule; } const reportFinalTestStatus = function(pass){ if(isUIThread()){ - const e = document.querySelector('#color-target'); + let e = document.querySelector('#color-target'); e.classList.add(pass ? 'tests-pass' : 'tests-fail'); + e = document.querySelector('title'); + e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText; }else{ postMessage({type:'test-result', payload:{pass}}); } @@ -246,10 +248,13 @@ self.sqlite3InitModule = sqlite3InitModule; log(TestUtil.separator); logClass('group-start',"Group #"+this.number+':',this.name); const indent = ' '; - if(this.predicate && !this.predicate(sqlite3)){ - logClass('warning',indent, - "SKIPPING group because predicate says to."); - return; + if(this.predicate){ + const p = this.predicate(sqlite3); + if(!p || 'string'===typeof p){ + logClass('warning',indent, + "SKIPPING group:", p ? p : "predicate says to" ); + return; + } } const assertCount = TestUtil.counter; const groupState = Object.create(null); @@ -259,24 +264,27 @@ self.sqlite3InitModule = sqlite3InitModule; ++i; const n = this.number+"."+i; log(indent, n+":", t.name); - if(t.predicate && !t.predicate(sqlite3)){ - logClass('warning', indent, indent, - 'SKIPPING because predicate says to'); - skipped.push( n+': '+t.name ); - }else{ - const tc = TestUtil.counter, now = performance.now(); - await t.test.call(groupState, sqlite3); - const then = performance.now(); - runtime += then - now; - logClass('faded',indent, indent, - TestUtil.counter - tc, 'assertion(s) in', - roundMs(then-now),'ms'); + if(t.predicate){ + const p = t.predicate(sqlite3); + if(!p || 'string'===typeof p){ + logClass('warning',indent, + "SKIPPING:", p ? p : "predicate says to" ); + skipped.push( n+': '+t.name ); + continue; + } } + const tc = TestUtil.counter, now = performance.now(); + await t.test.call(groupState, sqlite3); + const then = performance.now(); + runtime += then - now; + logClass('faded',indent, indent, + TestUtil.counter - tc, 'assertion(s) in', + roundMs(then-now),'ms'); } logClass('green', "Group #"+this.number+":",(TestUtil.counter - assertCount), "assertion(s) in",roundMs(runtime),"ms"); - if(skipped.length){ + if(0 && skipped.length){ logClass('warning',"SKIPPED test(s) in group",this.number+":",skipped); } } @@ -1380,13 +1388,38 @@ self.sqlite3InitModule = sqlite3InitModule; }) //////////////////////////////////////////////////////////////////////// - .t('sqlite3_js_db_export()', function(){ - const db = this.db; - const xp = capi.sqlite3_js_db_export(db.pointer); - T.assert(xp instanceof Uint8Array) - .assert(xp.byteLength>0) - .assert(0 === xp.byteLength % 512); + .t({ + name: 'sqlite3_js_db_export()', + predicate: ()=>true, + test: function(sqlite3){ + const db = this.db; + const xp = capi.sqlite3_js_db_export(db.pointer); + T.assert(xp instanceof Uint8Array) + .assert(xp.byteLength>0) + .assert(0 === xp.byteLength % 512); + this.dbExport = xp; + } }/*sqlite3_js_db_export()*/) + .t({ + name: 'sqlite3_js_vfs_create_file() with db in default VFS', + predicate: ()=>true, + test: function(sqlite3){ + const db = this.db; + const pVfs = capi.sqlite3_js_db_vfs(db); + const filename = "sqlite3_js_vfs_create_file().db"; + capi.sqlite3_js_vfs_create_file(pVfs, filename, this.dbExport); + delete this.dbExport; + const db2 = new sqlite3.oo1.DB(filename,'r'); + try { + const sql = "select count(*) from t"; + const n = db.selectValue(sql); + T.assert(n>0 && db2.selectValue(sql) === n); + }finally{ + if(db2) db2.close(); + wasm.sqlite3_wasm_vfs_unlink(pVfs, filename); + } + } + }/*sqlite3_js_vfs_create_file()*/) //////////////////////////////////////////////////////////////////// .t('Scalar UDFs', function(sqlite3){ @@ -1757,53 +1790,70 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('kvvfs') - .t('kvvfs sanity checks', function(sqlite3){ - if(isWorker()){ + .t({ + name: 'kvvfs is disabled in worker', + predicate: ()=>(isWorker() || "test is only valid in a Worker"), + test: function(sqlite3){ T.assert( !capi.sqlite3_vfs_find('kvvfs'), "Expecting kvvfs to be unregistered." ); - log("kvvfs is (correctly) unavailable in a Worker."); - return; } - const filename = 'session'; - const pVfs = capi.sqlite3_vfs_find('kvvfs'); - T.assert(pVfs); - const JDb = sqlite3.oo1.JsStorageDb; - const unlink = ()=>JDb.clearStorage(filename); - unlink(); - let db = new JDb(filename); - try { - db.exec([ - 'create table kvvfs(a);', - 'insert into kvvfs(a) values(1),(2),(3)' - ]); - T.assert(3 === db.selectValue('select count(*) from kvvfs')); - db.close(); - db = new JDb(filename); - db.exec('insert into kvvfs(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from kvvfs')); - - // Check import/export of db... - if(0){ - // does not yet work with kvvfs for unknown reasons... - const exp = capi.sqlite3_js_db_export(db); - db.close(); - unlink(); - capi.sqlite3_js_vfs_create_file("kvvfs", filename, exp); - db = new JDb(filename); - T.assert(6 === db.selectValue('select count(*) from kvvfs')); - } - }finally{ - db.close(); + }) + .t({ + name: 'kvvfs in main thread', + predicate: ()=>(isUIThread() ? true : "No local/sessionStorage in Worker"), + test: function(sqlite3){ + const filename = this.kvvfsDbFile = 'session'; + const pVfs = capi.sqlite3_vfs_find('kvvfs'); + T.assert(pVfs); + const JDb = this.JDb = sqlite3.oo1.JsStorageDb; + const unlink = this.kvvfsUnlink = ()=>{JDb.clearStorage(filename)}; unlink(); + let db = new JDb(filename); + try { + db.exec([ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]); + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + db.close(); + db = new JDb(filename); + db.exec('insert into kvvfs(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from kvvfs')); + }finally{ + db.close(); + } } }/*kvvfs sanity checks*/) + .t({ + name: 'kvvfs sqlite3_js_vfs_create_file()', + predicate: ()=>"kvvfs does not currently support this", + test: function(sqlite3){ + let db; + try { + db = new this.JDb(this.kvvfsDbFile); + const exp = capi.sqlite3_js_db_export(db); + db.close(); + this.kvvfsUnlink(); + capi.sqlite3_js_vfs_create_file("kvvfs", this.kvvfsDbFile, exp); + db = new this.JDb(filename); + T.assert(6 === db.selectValue('select count(*) from kvvfs')); + }finally{ + db.close(); + this.kvvfsUnlink(); + } + delete this.kvvfsDbFile; + delete this.kvvfsUnlink; + delete this.JDb; + } + }/*kvvfs sqlite3_js_vfs_create_file()*/) ;/* end kvvfs tests */ //////////////////////////////////////////////////////////////////////// - T.g('OPFS (Worker thread only and only in supported browsers)', - (sqlite3)=>!!sqlite3.opfs) + T.g('OPFS: Origin-Private File System', + (sqlite3)=>(sqlite3.opfs + ? true : "requires Worker thread in a compatible browser")) .t({ name: 'OPFS db sanity checks', test: async function(sqlite3){ diff --git a/manifest b/manifest index 3fd905ca0e..30c47b87f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sextraneous/unused\ssqlite3.oo1.version\sobject.\sAdd\shttpd\smakefile\starget. -D 2022-12-02T03:37:49.317 +C Expand\sJS\stests\sfor\sdb\sexport/import\sand\sdocument\sreason\sit\scannot\scurrently\swork\swith\skvvfs.\sFix\sa\sminor\sJS\sbuild\sdependencies\sbug.\sUpdate\spage\stitle\swith\sPASS/FAIL\sprefix\sfor\stester1.js\sto\simprove\soverview\swhen\slaunching\smultiple\stest\stabs.\sAdd\sability\sof\stester1\sshould-run-test\spredicates\sto\sreport\swhy\sa\sgiven\stest\sis\sdisabled. +D 2022-12-02T07:14:56.641 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 6b90a38f9d7b64e3cf1c494fea1d29ac175ab66eed816cdc6bb3164d94f16aed +F ext/wasm/GNUmakefile 891c5ce27966e6a87267300cafda145747bf9ad5eddacf1fbb5f1600768c8157 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -505,14 +505,14 @@ F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a9578430388 F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js b528207ba43f7740d1ade623f3f6b08a49f44ce7e9126915b78e1818c2466d8e F ext/wasm/api/sqlite3-api-oo1.js c8b6c9ccb64cf93ca990ac689e98963735110aec21f98e04b55018f8e67b8147 -F ext/wasm/api/sqlite3-api-prologue.js f8330a837cf008cae65083c356ebddb6a00054e94a2d2927f2828ad83d3112ce +F ext/wasm/api/sqlite3-api-prologue.js 42d6b316b542cf8e086f2f272460deb72dff184f1438a3377383cab99b08070b F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 +F ext/wasm/api/sqlite3-wasm.c edae35c7fe070ae7db5feaf3a7d9ba45708f6f56fdcbb3ad0fb2710e4da3bb05 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -553,9 +553,9 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac -F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 +F ext/wasm/tester1-worker.html ead6bdcc6cca221deb0dc9855a56f376351dbf2294fd7978cd1609b3a56b245b F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js 9ae36e9b6055975348d7893f2d89ba7119f54e7e78f3d1c7007522d12c444165 +F ext/wasm/tester1.c-pp.js e73a91eba4b59aaadd98f383c00a5101dbbbc52d937fff3162fc4761986f4a88 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14a84b67fb17e16a5691ea4bf7f374123ac73a361a5d3d0efca53788d2001e3a -R 731cae2fa3f5f500915aa9b0ca4c85de +P 8e4d30ac033a6d9019a7eeedfe788dc0120f565cef2ae8f09d2bf32eb94d8a33 +R 537bb49fe6ebd23b0d6c90dc722897c8 U stephan -Z 40b7eebb887f929404345b830b46e1fc +Z eb02f2c4f6198c757ec0df7e4c3e3028 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e3f1787a58..f4aeb21848 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e4d30ac033a6d9019a7eeedfe788dc0120f565cef2ae8f09d2bf32eb94d8a33 \ No newline at end of file +75f610d3a4cf3d972220f9abc27cdf5990451e3835ceb9cf66973934004dfc5c \ No newline at end of file From f3409db2d1c76cec89cd325b2baede6c582778c2 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 08:29:03 +0000 Subject: [PATCH 121/282] sqlite3-wasm.c: code legibility and coding style tweaks. Increase SQLITE_DEFAULT_PAGE_SIZE from 4k to 8k, as that improves OPFS speedtest1 performance by roughly 12%. FossilOrigin-Name: c260895faacb3458c557778630756d02a8520c0f1864bddcf86cdd27ef4a42bd --- ext/wasm/api/sqlite3-wasm.c | 165 ++++++++++++++++++------------------ manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 89 insertions(+), 90 deletions(-) diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 6f214df347..1edaec390a 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -32,21 +32,16 @@ ** the same db handle as another thread, thus multi-threading support ** is unnecessary in the library. Because the filesystems are virtual ** and local to a given wasm runtime instance, two Workers can never -** access the same db file at once, with the exception of OPFS. As of -** this writing (2022-09-30), OPFS exclusively locks a file when -** opening it, so two Workers can never open the same OPFS-backed file -** at once. That situation will change if and when lower-level locking -** features are added to OPFS (as is currently planned, per folks -** involved with its development). +** access the same db file at once, with the exception of OPFS. ** -** Summary: except for the case of future OPFS, which supports -** locking, and any similar future filesystems, threading and file -** locking support are unnecessary in the wasm build. +** Summary: except for the case of OPFS, which supports locking using +** its own API, threading and file locking support are unnecessary in +** the wasm build. */ /* ** Undefine any SQLITE_... config flags which we specifically do not -** want undefined. Please keep these alphabetized. +** want defined. Please keep these alphabetized. */ #undef SQLITE_OMIT_DESERIALIZE #undef SQLITE_OMIT_MEMORYDB @@ -69,9 +64,11 @@ */ # define SQLITE_DEFAULT_CACHE_SIZE -16384 #endif -#if 0 && !defined(SQLITE_DEFAULT_PAGE_SIZE) -/* TODO: experiment with this. */ -# define SQLITE_DEFAULT_PAGE_SIZE 8192 /*4096*/ +#if !defined(SQLITE_DEFAULT_PAGE_SIZE) +/* OPFS performance is improved with a page size of 8k instead +** of 4k. kvvfs, OTOH, likely suffers from that. Peformance +** with 16k is equivalent to 8k. */ +# define SQLITE_DEFAULT_PAGE_SIZE 8196 /*4096*/ #endif #ifndef SQLITE_DEFAULT_UNIX_VFS # define SQLITE_DEFAULT_UNIX_VFS "unix-none" @@ -363,7 +360,7 @@ const char * sqlite3_wasm_enum_json(void){ int n = 0, nChildren = 0, nStruct = 0 /* output counters for figuring out where commas go */; char * zPos = &aBuffer[1] /* skip first byte for now to help protect - ** against a small race condition */; + ** against a small race condition */; char const * const zEnd = &aBuffer[0] + sizeof(aBuffer) /* one-past-the-end */; if(aBuffer[0]) return aBuffer; /* Leave aBuffer[0] at 0 until the end to help guard against a tiny @@ -695,8 +692,8 @@ const char * sqlite3_wasm_enum_json(void){ /** Macros for emitting StructBinder description. */ #define StructBinder__(TYPE) \ n = 0; \ - outf("%s{", (nStruct++ ? ", " : "")); \ - out("\"name\": \"" # TYPE "\","); \ + outf("%s{", (nStruct++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ out(",\"members\": {"); #define StructBinder_(T) StructBinder__(T) @@ -716,78 +713,78 @@ const char * sqlite3_wasm_enum_json(void){ #define CurrentStruct sqlite3_vfs StructBinder { - M(iVersion,"i"); - M(szOsFile,"i"); - M(mxPathname,"i"); - M(pNext,"p"); - M(zName,"s"); - M(pAppData,"p"); - M(xOpen,"i(pppip)"); - M(xDelete,"i(ppi)"); - M(xAccess,"i(ppip)"); - M(xFullPathname,"i(ppip)"); - M(xDlOpen,"p(pp)"); - M(xDlError,"p(pip)"); - M(xDlSym,"p()"); - M(xDlClose,"v(pp)"); - M(xRandomness,"i(pip)"); - M(xSleep,"i(pi)"); - M(xCurrentTime,"i(pp)"); - M(xGetLastError,"i(pip)"); - M(xCurrentTimeInt64,"i(pp)"); - M(xSetSystemCall,"i(ppp)"); - M(xGetSystemCall,"p(pp)"); - M(xNextSystemCall,"p(pp)"); + M(iVersion, "i"); + M(szOsFile, "i"); + M(mxPathname, "i"); + M(pNext, "p"); + M(zName, "s"); + M(pAppData, "p"); + M(xOpen, "i(pppip)"); + M(xDelete, "i(ppi)"); + M(xAccess, "i(ppip)"); + M(xFullPathname, "i(ppip)"); + M(xDlOpen, "p(pp)"); + M(xDlError, "p(pip)"); + M(xDlSym, "p()"); + M(xDlClose, "v(pp)"); + M(xRandomness, "i(pip)"); + M(xSleep, "i(pi)"); + M(xCurrentTime, "i(pp)"); + M(xGetLastError, "i(pip)"); + M(xCurrentTimeInt64, "i(pp)"); + M(xSetSystemCall, "i(ppp)"); + M(xGetSystemCall, "p(pp)"); + M(xNextSystemCall, "p(pp)"); } _StructBinder; #undef CurrentStruct #define CurrentStruct sqlite3_io_methods StructBinder { - M(iVersion,"i"); - M(xClose,"i(p)"); - M(xRead,"i(ppij)"); - M(xWrite,"i(ppij)"); - M(xTruncate,"i(pj)"); - M(xSync,"i(pi)"); - M(xFileSize,"i(pp)"); - M(xLock,"i(pi)"); - M(xUnlock,"i(pi)"); - M(xCheckReservedLock,"i(pp)"); - M(xFileControl,"i(pip)"); - M(xSectorSize,"i(p)"); - M(xDeviceCharacteristics,"i(p)"); - M(xShmMap,"i(piiip)"); - M(xShmLock,"i(piii)"); - M(xShmBarrier,"v(p)"); - M(xShmUnmap,"i(pi)"); - M(xFetch,"i(pjip)"); - M(xUnfetch,"i(pjp)"); + M(iVersion, "i"); + M(xClose, "i(p)"); + M(xRead, "i(ppij)"); + M(xWrite, "i(ppij)"); + M(xTruncate, "i(pj)"); + M(xSync, "i(pi)"); + M(xFileSize, "i(pp)"); + M(xLock, "i(pi)"); + M(xUnlock, "i(pi)"); + M(xCheckReservedLock, "i(pp)"); + M(xFileControl, "i(pip)"); + M(xSectorSize, "i(p)"); + M(xDeviceCharacteristics, "i(p)"); + M(xShmMap, "i(piiip)"); + M(xShmLock, "i(piii)"); + M(xShmBarrier, "v(p)"); + M(xShmUnmap, "i(pi)"); + M(xFetch, "i(pjip)"); + M(xUnfetch, "i(pjp)"); } _StructBinder; #undef CurrentStruct #define CurrentStruct sqlite3_file StructBinder { - M(pMethods,"p"); + M(pMethods, "p"); } _StructBinder; #undef CurrentStruct #define CurrentStruct sqlite3_kvvfs_methods StructBinder { - M(xRead,"i(sspi)"); - M(xWrite,"i(sss)"); - M(xDelete,"i(ss)"); - M(nKeySize,"i"); + M(xRead, "i(sspi)"); + M(xWrite, "i(sss)"); + M(xDelete, "i(ss)"); + M(nKeySize, "i"); } _StructBinder; #undef CurrentStruct #if SQLITE_WASM_TESTS #define CurrentStruct WasmTestStruct StructBinder { - M(v4,"i"); - M(cstr,"s"); - M(ppV,"p"); - M(v8,"j"); - M(xFunc,"v(p)"); + M(v4, "i"); + M(cstr, "s"); + M(ppV, "p"); + M(v8, "j"); + M(xFunc, "v(p)"); } _StructBinder; #undef CurrentStruct #endif @@ -820,7 +817,7 @@ const char * sqlite3_wasm_enum_json(void){ ** call is returned. */ SQLITE_WASM_KEEP -int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char * zName){ +int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ int rc = SQLITE_MISUSE /* ??? */; if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); if( zName && pVfs && pVfs->xDelete ){ @@ -857,12 +854,14 @@ sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ ** SQLITE_MISUSE if pDb is NULL. */ SQLITE_WASM_KEEP -int sqlite3_wasm_db_reset(sqlite3*pDb){ +int sqlite3_wasm_db_reset(sqlite3 *pDb){ int rc = SQLITE_MISUSE; if( pDb ){ rc = sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); - if( 0==rc ) rc = sqlite3_exec(pDb, "VACUUM", 0, 0, 0); - sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + if( 0==rc ){ + rc = sqlite3_exec(pDb, "VACUUM", 0, 0, 0); + sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + } } return rc; } @@ -907,7 +906,7 @@ int sqlite3_wasm_db_export_chunked( sqlite3* pDb, } for( ; 0==rc && nPospMethods->xRead(pFile, buf, nBuf, nPos); - if(SQLITE_IOERR_SHORT_READ == rc){ + if( SQLITE_IOERR_SHORT_READ == rc ){ rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/; } if( 0==rc ) rc = xCallback(buf, nBuf); @@ -935,7 +934,7 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, sqlite3_int64 *nOut, unsigned int mFlags ){ unsigned char * z; if( !pDb || !pOut ) return SQLITE_MISUSE; - if(nOut) *nOut = 0; + if( nOut ) *nOut = 0; z = sqlite3_serialize(pDb, zSchema ? zSchema : "main", nOut, mFlags); if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){ *pOut = z; @@ -958,9 +957,8 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** Emscripten's FS.createDataFile() in a VFS-agnostic way. This ** functionality is intended for use in uploading database files. ** -** Note that not all VFSes support this operation because they impose -** specific requirements on truncate and write sizes. e.g. kvvfs does -** not work with this. +** Not all VFSes support this functionality, e.g. the "kvvfs" does +** not. ** ** If pVfs is NULL, sqlite3_vfs_find(0) is used. ** @@ -973,10 +971,7 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** ** Whether or not directory components of zFilename are created ** automatically or not is unspecified: that detail is left to the -** VFS. The "opfs" VFS, for example, create them. -** -** Not all VFSes support this functionality, e.g. the "kvvfs" does -** not. +** VFS. The "opfs" VFS, for example, creates them. ** ** If an error happens while populating or truncating the file, the ** target file will be deleted (if needed) if this function created @@ -1014,7 +1009,8 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, pVfs->xAccess(pVfs, zFilename, SQLITE_ACCESS_EXISTS, &fileExisted); rc = sqlite3OsOpenMalloc(pVfs, zFilename, &pFile, openFlags, &flagsOut); #if 0 -# define RC fprintf(stderr,"create_file(%s,%s) @%d rc=%d\n", pVfs->zName, zFilename, __LINE__, rc); +# define RC fprintf(stderr,"create_file(%s,%s) @%d rc=%d\n", \ + pVfs->zName, zFilename, __LINE__, rc); #else # define RC #endif @@ -1046,11 +1042,14 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, } if( 0==rc && nData>0 ){ assert( nDataxWrite(pFile, pPos, nData, (sqlite3_int64)(pPos - pData)); + rc = pIo->xWrite(pFile, pPos, nData, + (sqlite3_int64)(pPos - pData)); RC; } } - if( pIo->xUnlock && doUnlock!=0 ) pIo->xUnlock(pFile, SQLITE_LOCK_NONE); + if( pIo->xUnlock && doUnlock!=0 ){ + pIo->xUnlock(pFile, SQLITE_LOCK_NONE); + } pIo->xClose(pFile); if( rc!=0 && 0==fileExisted ){ pVfs->xDelete(pVfs, zFilename, 1); diff --git a/manifest b/manifest index 30c47b87f6..d8a09a5f4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expand\sJS\stests\sfor\sdb\sexport/import\sand\sdocument\sreason\sit\scannot\scurrently\swork\swith\skvvfs.\sFix\sa\sminor\sJS\sbuild\sdependencies\sbug.\sUpdate\spage\stitle\swith\sPASS/FAIL\sprefix\sfor\stester1.js\sto\simprove\soverview\swhen\slaunching\smultiple\stest\stabs.\sAdd\sability\sof\stester1\sshould-run-test\spredicates\sto\sreport\swhy\sa\sgiven\stest\sis\sdisabled. -D 2022-12-02T07:14:56.641 +C sqlite3-wasm.c:\scode\slegibility\sand\scoding\sstyle\stweaks.\sIncrease\sSQLITE_DEFAULT_PAGE_SIZE\sfrom\s4k\sto\s8k,\sas\sthat\simproves\sOPFS\sspeedtest1\sperformance\sby\sroughly\s12%. +D 2022-12-02T08:29:03.564 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -512,7 +512,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e8 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c edae35c7fe070ae7db5feaf3a7d9ba45708f6f56fdcbb3ad0fb2710e4da3bb05 +F ext/wasm/api/sqlite3-wasm.c 3fb517065417bfddf9f49b73cf1f8af1d8d50fbb295c484e77359e5ed6e14a21 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8e4d30ac033a6d9019a7eeedfe788dc0120f565cef2ae8f09d2bf32eb94d8a33 -R 537bb49fe6ebd23b0d6c90dc722897c8 +P 75f610d3a4cf3d972220f9abc27cdf5990451e3835ceb9cf66973934004dfc5c +R 476dae20eba5ae2d14fd70169918f0cd U stephan -Z eb02f2c4f6198c757ec0df7e4c3e3028 +Z e6e64ad365153ec68f24a3bd7c94c825 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f4aeb21848..cdd727ce41 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75f610d3a4cf3d972220f9abc27cdf5990451e3835ceb9cf66973934004dfc5c \ No newline at end of file +c260895faacb3458c557778630756d02a8520c0f1864bddcf86cdd27ef4a42bd \ No newline at end of file From 20f0128acc16b44519933d0ab6ff9373b647227f Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 08:38:04 +0000 Subject: [PATCH 122/282] Roll back the SQLITE_DEFAULT_PAGE_SIZE part of [c260895faacb34] because kvvfs does not work at all with a page size of 8kb. FossilOrigin-Name: 7eec635562f65592121d7ff0a31a7b4ff6cdfe38e657f326302ef851b3b37c3f --- ext/wasm/api/sqlite3-wasm.c | 11 +++++++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 1edaec390a..0e8aabd1a5 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -65,10 +65,13 @@ # define SQLITE_DEFAULT_CACHE_SIZE -16384 #endif #if !defined(SQLITE_DEFAULT_PAGE_SIZE) -/* OPFS performance is improved with a page size of 8k instead -** of 4k. kvvfs, OTOH, likely suffers from that. Peformance -** with 16k is equivalent to 8k. */ -# define SQLITE_DEFAULT_PAGE_SIZE 8196 /*4096*/ +/* +** OPFS performance is improved with a page size of 8kb instead of +** 4kb. Performance with 16kb is equivalent to 8kb. +** +** However... kvvfs doesn't work at all with 8kb! +*/ +# define SQLITE_DEFAULT_PAGE_SIZE 4096 /* 8196 */ #endif #ifndef SQLITE_DEFAULT_UNIX_VFS # define SQLITE_DEFAULT_UNIX_VFS "unix-none" diff --git a/manifest b/manifest index d8a09a5f4d..9d66ae5b1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C sqlite3-wasm.c:\scode\slegibility\sand\scoding\sstyle\stweaks.\sIncrease\sSQLITE_DEFAULT_PAGE_SIZE\sfrom\s4k\sto\s8k,\sas\sthat\simproves\sOPFS\sspeedtest1\sperformance\sby\sroughly\s12%. -D 2022-12-02T08:29:03.564 +C Roll\sback\sthe\sSQLITE_DEFAULT_PAGE_SIZE\spart\sof\s[c260895faacb34]\sbecause\skvvfs\sdoes\snot\swork\sat\sall\swith\sa\spage\ssize\sof\s8kb. +D 2022-12-02T08:38:04.930 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -512,7 +512,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e8 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 3fb517065417bfddf9f49b73cf1f8af1d8d50fbb295c484e77359e5ed6e14a21 +F ext/wasm/api/sqlite3-wasm.c 4ee44cc6934a4aa904c7f70043c33a560dfa369bea10cafd505ded15aadd8692 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75f610d3a4cf3d972220f9abc27cdf5990451e3835ceb9cf66973934004dfc5c -R 476dae20eba5ae2d14fd70169918f0cd +P c260895faacb3458c557778630756d02a8520c0f1864bddcf86cdd27ef4a42bd +R 8883779dc6cb9fbc583e811dbb56f056 U stephan -Z e6e64ad365153ec68f24a3bd7c94c825 +Z eb61cb45b5ecf451015b257eeb20fb82 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cdd727ce41..84dc7436da 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c260895faacb3458c557778630756d02a8520c0f1864bddcf86cdd27ef4a42bd \ No newline at end of file +7eec635562f65592121d7ff0a31a7b4ff6cdfe38e657f326302ef851b3b37c3f \ No newline at end of file From 1078ee095980c97d7c873fca488470e43ed56797 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 08:51:22 +0000 Subject: [PATCH 123/282] Correct the problem which triggered the rollback in [7eec635562f6]: an incorrect default db page size (not a multiple of 512 bytes). FossilOrigin-Name: e06e490c240aa56b616e6f0380b5d08abf06f35f9e683e5150c5ac464eae9e55 --- ext/wasm/api/sqlite3-wasm.c | 11 +++++++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 0e8aabd1a5..9acc8020e5 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -66,12 +66,15 @@ #endif #if !defined(SQLITE_DEFAULT_PAGE_SIZE) /* -** OPFS performance is improved with a page size of 8kb instead of -** 4kb. Performance with 16kb is equivalent to 8kb. +** OPFS performance is improved by approx. 12% with a page size of 8kb +** instead of 4kb. Performance with 16kb is equivalent to 8kb. ** -** However... kvvfs doesn't work at all with 8kb! +** Performance difference of kvvfs with a page size of 8kb compared to +** 4kb, as measured by speedtest1 --size 4, is indeterminate: +** measurements are all over the place either way and not +** significantly different. */ -# define SQLITE_DEFAULT_PAGE_SIZE 4096 /* 8196 */ +# define SQLITE_DEFAULT_PAGE_SIZE 8192 #endif #ifndef SQLITE_DEFAULT_UNIX_VFS # define SQLITE_DEFAULT_UNIX_VFS "unix-none" diff --git a/manifest b/manifest index 9d66ae5b1e..797eb0bbe2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Roll\sback\sthe\sSQLITE_DEFAULT_PAGE_SIZE\spart\sof\s[c260895faacb34]\sbecause\skvvfs\sdoes\snot\swork\sat\sall\swith\sa\spage\ssize\sof\s8kb. -D 2022-12-02T08:38:04.930 +C Correct\sthe\sproblem\swhich\striggered\sthe\srollback\sin\s[7eec635562f6]:\san\sincorrect\sdefault\sdb\spage\ssize\s(not\sa\smultiple\sof\s512\sbytes). +D 2022-12-02T08:51:22.208 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -512,7 +512,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e8 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 4ee44cc6934a4aa904c7f70043c33a560dfa369bea10cafd505ded15aadd8692 +F ext/wasm/api/sqlite3-wasm.c 733bc939f93caef0df0b3ebfea14cbd528da580fdef1a35b1f69c2b3e044c7b7 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c260895faacb3458c557778630756d02a8520c0f1864bddcf86cdd27ef4a42bd -R 8883779dc6cb9fbc583e811dbb56f056 +P 7eec635562f65592121d7ff0a31a7b4ff6cdfe38e657f326302ef851b3b37c3f +R d69c10698dc7edf445a81ee298ed44aa U stephan -Z eb61cb45b5ecf451015b257eeb20fb82 +Z 4d99e2beb03c6043d74aa14f952e72dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 84dc7436da..9ef750f5f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7eec635562f65592121d7ff0a31a7b4ff6cdfe38e657f326302ef851b3b37c3f \ No newline at end of file +e06e490c240aa56b616e6f0380b5d08abf06f35f9e683e5150c5ac464eae9e55 \ No newline at end of file From d09414e6231bf27af0713c5ca38c2ef6525c9289 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 09:23:38 +0000 Subject: [PATCH 124/282] wasm: after building snapshot zip file, emit instructions for pushing it to the test server. FossilOrigin-Name: 9615c77919fee60d708d72729c741b373a89900aabc318e0ceb4154638cdf339 --- ext/wasm/dist.make | 2 ++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index 9b0267dac5..c8b9ee1fb3 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -98,6 +98,8 @@ dist: \ set +e; \ unzip -lv $$arczip || echo "Missing unzip app? Not fatal." snapshot: dist + @echo "Upload snapshot with:"; \ + echo "rsync -ve ssh $(dist-name-prefix)*.zip $(wasm-testing.dest)/snapshots/." # We need a separate `clean` rule to account for weirdness in # a sub-make, where we get a copy of the $(dist-name) dir # copied into the new $(dist-name) dir. diff --git a/manifest b/manifest index 797eb0bbe2..b028e276cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\sthe\sproblem\swhich\striggered\sthe\srollback\sin\s[7eec635562f6]:\san\sincorrect\sdefault\sdb\spage\ssize\s(not\sa\smultiple\sof\s512\sbytes). -D 2022-12-02T08:51:22.208 +C wasm:\safter\sbuilding\ssnapshot\szip\sfile,\semit\sinstructions\sfor\spushing\sit\sto\sthe\stest\sserver. +D 2022-12-02T09:23:38.378 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -531,7 +531,7 @@ F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98 F ext/wasm/demo-worker1-promiser.js b85a2bb1b918db4f09dfa24419241cb3edad7791389425c2505092e9b715017d F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa -F ext/wasm/dist.make 11b98da79385701a568a4728671821fe2524c1d1ecd05ff2e24cb3e33b2c6c4f +F ext/wasm/dist.make 994cc61822694b123d4357731072937a54153fbe5d9b12c6cb95d5562d2766dc F ext/wasm/fiddle.make 2812c44c9bafb5be9c8767963d1b9f374d77af7795fcaa06483c03e7059dea74 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7eec635562f65592121d7ff0a31a7b4ff6cdfe38e657f326302ef851b3b37c3f -R d69c10698dc7edf445a81ee298ed44aa +P e06e490c240aa56b616e6f0380b5d08abf06f35f9e683e5150c5ac464eae9e55 +R 8b2a86dd310ca7e22a548f45d379ed5f U stephan -Z 4d99e2beb03c6043d74aa14f952e72dd +Z d9c77faf9eb600fc5df42049da15712b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ef750f5f7..616b2f2fcc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e06e490c240aa56b616e6f0380b5d08abf06f35f9e683e5150c5ac464eae9e55 \ No newline at end of file +9615c77919fee60d708d72729c741b373a89900aabc318e0ceb4154638cdf339 \ No newline at end of file From 5ad4e89659a3fa056c41293f2aefe586c059e7e7 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 10:43:14 +0000 Subject: [PATCH 125/282] wasm builds: explicitly set a default stack size because emsdk 3.1.27 reduced it from 4MB to only 64kb, leading to memory corruption when kvvfs is used (it requires at least twice that for I/O). FossilOrigin-Name: 758112460fb624198a144ed7d8d2324497fa72a2eee80010bd22b621e064b389 --- ext/wasm/GNUmakefile | 6 +++++- ext/wasm/wasmfs.make | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 08c325a541..4f4f7b1b5e 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -488,7 +488,10 @@ emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) ######################################################################## emcc.jsflags += $(emcc.environment) -#emcc.jsflags += -sTOTAL_STACK=4194304 +emcc.jsflags += -sSTACK_SIZE=1MB +# ^^^ ACHTUNG: emsdk 3.1.27 reduced the default stack size from 4MB to +# a mere 64KB, which leads to silent memory corruption via the kvvfs +# VFS, which requires twice that for its xRead() and xWrite() methods. ######################################################################## # $(sqlite3.js.init-func) is the name Emscripten assigns our exported # module init/load function. This symbol name is hard-coded in @@ -670,6 +673,7 @@ speedtest1.eflags.common += -sSTRICT_JS speedtest1.eflags.common += -sMODULARIZE speedtest1.eflags.common += -Wno-limited-postlink-optimizations EXPORTED_FUNCTIONS.speedtest1 := $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) +speedtest1.eflags.common += -sSTACK_SIZE=1MB speedtest1.eflags.common += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.speedtest1) speedtest1.eflags.common += $(emcc.exportedRuntimeMethods) speedtest1.eflags.common += -sALLOW_TABLE_GROWTH diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make index e4d72059a4..c64ce071e5 100644 --- a/ext/wasm/wasmfs.make +++ b/ext/wasm/wasmfs.make @@ -44,7 +44,7 @@ sqlite3-wasmfs.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8O sqlite3-wasmfs.jsflags += -sUSE_CLOSURE_COMPILER=0 sqlite3-wasmfs.jsflags += -sIMPORTED_MEMORY #sqlite3-wasmfs.jsflags += -sINITIAL_MEMORY=13107200 -#sqlite3-wasmfs.jsflags += -sTOTAL_STACK=4194304 +sqlite3-wasmfs.jsflags += -sSTACK_SIZE=1MB sqlite3-wasmfs.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) sqlite3-wasmfs.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. #sqlite3-wasmfs.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API diff --git a/manifest b/manifest index b028e276cc..ead2dbd348 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm:\safter\sbuilding\ssnapshot\szip\sfile,\semit\sinstructions\sfor\spushing\sit\sto\sthe\stest\sserver. -D 2022-12-02T09:23:38.378 +C wasm\sbuilds:\sexplicitly\sset\sa\sdefault\sstack\ssize\sbecause\semsdk\s3.1.27\sreduced\sit\sfrom\s4MB\sto\sonly\s64kb,\sleading\sto\smemory\scorruption\swhen\skvvfs\sis\sused\s(it\srequires\sat\sleast\stwice\sthat\sfor\sI/O). +D 2022-12-02T10:43:14.520 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 891c5ce27966e6a87267300cafda145747bf9ad5eddacf1fbb5f1600768c8157 +F ext/wasm/GNUmakefile 3ccde6913ab2d1739d03a950b26ba84b2e66df35dd9dcd3d6e9609d963d12b50 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -560,7 +560,7 @@ F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 -F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d +F ext/wasm/wasmfs.make 7ab655788bf0b52dce4538acbd5b11cdbe77edd36a14af5dec6dfe1ec4ab25fc F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e06e490c240aa56b616e6f0380b5d08abf06f35f9e683e5150c5ac464eae9e55 -R 8b2a86dd310ca7e22a548f45d379ed5f +P 9615c77919fee60d708d72729c741b373a89900aabc318e0ceb4154638cdf339 +R 29a0b11b169b5fde0c1900de7b5662de U stephan -Z d9c77faf9eb600fc5df42049da15712b +Z b619ea6beb43ef409ffd5a793f810416 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 616b2f2fcc..fc54e21c9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9615c77919fee60d708d72729c741b373a89900aabc318e0ceb4154638cdf339 \ No newline at end of file +758112460fb624198a144ed7d8d2324497fa72a2eee80010bd22b621e064b389 \ No newline at end of file From 80ea1f406a593fb052694108ee0951439c662a9a Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 11:35:21 +0000 Subject: [PATCH 126/282] Minor touchups to the JS test index page and test server push rules. FossilOrigin-Name: 0881f3e92364b3dc81443220451e853c0763efaf97653a6348bc39bd9bdb23ad --- ext/wasm/GNUmakefile | 3 ++- ext/wasm/index-dist.html | 4 ++++ ext/wasm/index.html | 10 +++++++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 4f4f7b1b5e..d8ee9c758a 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -819,8 +819,9 @@ endif ######################################################################## # Push files to public wasm-testing.sqlite.org server -wasm-testing.include = *.js *.html batch-runner.list \ +wasm-testing.include = *.js *.mjs *.html \ ./tests \ + batch-runner.list \ $(dir.dout) $(dir.sql) $(dir.common) $(dir.fiddle) $(dir.jacc) wasm-testing.exclude = sql/speedtest1.sql wasm-testing.dir = /jail/sites/wasm-testing diff --git a/ext/wasm/index-dist.html b/ext/wasm/index-dist.html index 29891c91e9..1c23bd82c7 100644 --- a/ext/wasm/index-dist.html +++ b/ext/wasm/index-dist.html @@ -19,6 +19,10 @@ header { font-size: 130%; font-weight: bold; + background: #044a64; + color: white; + padding: 0.5em; + border-radius: 0.25em; } .hidden, .initially-hidden { position: absolute !important; diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 9fa5bbdf49..33a115d2e9 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -8,6 +8,14 @@ sqlite3 WASM Testing Page Index +

    sqlite3 WASM test pages
    Below is the list of test pages for the sqlite3 WASM @@ -30,7 +38,7 @@ Chrome or Chromium (v102 at least, possibly newer). OPFS support in the other major browsers is pending. Development and testing is currently done against a dev-channel release - of Chrome (v107 as of 2022-09-26). + of Chrome (v110 as of 2022-12-02).
    • diff --git a/manifest b/manifest index ead2dbd348..026dcfc9cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm\sbuilds:\sexplicitly\sset\sa\sdefault\sstack\ssize\sbecause\semsdk\s3.1.27\sreduced\sit\sfrom\s4MB\sto\sonly\s64kb,\sleading\sto\smemory\scorruption\swhen\skvvfs\sis\sused\s(it\srequires\sat\sleast\stwice\sthat\sfor\sI/O). -D 2022-12-02T10:43:14.520 +C Minor\stouchups\sto\sthe\sJS\stest\sindex\spage\sand\stest\sserver\spush\srules. +D 2022-12-02T11:35:21.811 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 3ccde6913ab2d1739d03a950b26ba84b2e66df35dd9dcd3d6e9609d963d12b50 +F ext/wasm/GNUmakefile 54ab8da16a01e78bf5767c0e7bd57af07bfeb3a71fbecd63a39b3dbeec967c4e F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -537,8 +537,8 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695 F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 -F ext/wasm/index-dist.html c4337617c4d6d4d0796827cec28ac81d128c6f911dcf888a290a32ad50890408 -F ext/wasm/index.html 5be176de5be8ae96889798f803fef4f6a2ef31cee305a0430ca4629f6ae04c27 +F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 +F ext/wasm/index.html 618ad5cbc07b55556a4e09931773761698a38323387d392046bda907f1ce4c52 F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e F ext/wasm/module-symbols.html b8eebafef8e536624bbe5f7a3da40c07a9062b843dfd3161a0bb72cbb6763dc5 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9615c77919fee60d708d72729c741b373a89900aabc318e0ceb4154638cdf339 -R 29a0b11b169b5fde0c1900de7b5662de +P 758112460fb624198a144ed7d8d2324497fa72a2eee80010bd22b621e064b389 +R 7335100c09b3966256d4f515791e8509 U stephan -Z b619ea6beb43ef409ffd5a793f810416 +Z 82fdbc046c1724ee06335f4931485b56 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc54e21c9a..b5b97e7e1c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -758112460fb624198a144ed7d8d2324497fa72a2eee80010bd22b621e064b389 \ No newline at end of file +0881f3e92364b3dc81443220451e853c0763efaf97653a6348bc39bd9bdb23ad \ No newline at end of file From 8b5d7fda5908dcf6fedd440f184d45c9eab8de87 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 2 Dec 2022 15:31:47 +0000 Subject: [PATCH 127/282] Use sqlite3_result_int64() instead of sqlite3_result_int() when returning potentially large values from the DBSTAT virtual table, to avoid integer overflows in the result. [forum:/forumpost/ada2ab044f|Forum post ada2ab044f]. FossilOrigin-Name: 5652154a8c93cf3b1ff6c2e55e94abbe995b0bb625f733461df20e006c2f13f8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/dbstat.c | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 026dcfc9cc..b6ab812953 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\stouchups\sto\sthe\sJS\stest\sindex\spage\sand\stest\sserver\spush\srules. -D 2022-12-02T11:35:21.811 +C Use\ssqlite3_result_int64()\sinstead\sof\ssqlite3_result_int()\swhen\sreturning\npotentially\slarge\svalues\sfrom\sthe\sDBSTAT\svirtual\stable,\sto\savoid\sinteger\noverflows\sin\sthe\sresult.\n[forum:/forumpost/ada2ab044f|Forum\spost\sada2ab044f]. +D 2022-12-02T15:31:47.541 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -593,7 +593,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 -F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d +F src/dbstat.c a56a7ad1163a9888d46cd5820be2e65354fb1aa04ed6909f7c3e5831e0ee2c29 F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e F src/expr.c 9e7fadc664b938c18f006be0d4f6669888f9302756ee204420c7eccaed5435a6 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 758112460fb624198a144ed7d8d2324497fa72a2eee80010bd22b621e064b389 -R 7335100c09b3966256d4f515791e8509 -U stephan -Z 82fdbc046c1724ee06335f4931485b56 +P 0881f3e92364b3dc81443220451e853c0763efaf97653a6348bc39bd9bdb23ad +R e242bef08302ff1ff3602cd93b1159c9 +U drh +Z e49db422920b594adc6e6d22717359b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b5b97e7e1c..19414a7f61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0881f3e92364b3dc81443220451e853c0763efaf97653a6348bc39bd9bdb23ad \ No newline at end of file +5652154a8c93cf3b1ff6c2e55e94abbe995b0bb625f733461df20e006c2f13f8 \ No newline at end of file diff --git a/src/dbstat.c b/src/dbstat.c index bb88a76f40..fdef913f88 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -824,16 +824,16 @@ static int statColumn( } break; case 4: /* ncell */ - sqlite3_result_int(ctx, pCsr->nCell); + sqlite3_result_int64(ctx, pCsr->nCell); break; case 5: /* payload */ - sqlite3_result_int(ctx, pCsr->nPayload); + sqlite3_result_int64(ctx, pCsr->nPayload); break; case 6: /* unused */ - sqlite3_result_int(ctx, pCsr->nUnused); + sqlite3_result_int64(ctx, pCsr->nUnused); break; case 7: /* mx_payload */ - sqlite3_result_int(ctx, pCsr->nMxPayload); + sqlite3_result_int64(ctx, pCsr->nMxPayload); break; case 8: /* pgoffset */ if( !pCsr->isAgg ){ @@ -841,7 +841,7 @@ static int statColumn( } break; case 9: /* pgsize */ - sqlite3_result_int(ctx, pCsr->szPage); + sqlite3_result_int64(ctx, pCsr->szPage); break; case 10: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); From d2603adf46aee6e5e7d3ad963e1947e69d617b58 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 2 Dec 2022 17:52:52 +0000 Subject: [PATCH 128/282] For the sqlite3_bind and sqlite3_result interfaces for UTF16 strings, round the number of bytes down to the next even number, to avoid creating a UTF16 string that is an odd number of bytes. [forum:/forumpost/411199488d065f83|Forum post 411199488d065f83]. FossilOrigin-Name: b57e3c3db00a6bc6db20c82530479f9eba7e37b731f0da6fe81682e84c7ac916 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 20 +++++++++++++------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index b6ab812953..a3d352d1d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\ssqlite3_result_int64()\sinstead\sof\ssqlite3_result_int()\swhen\sreturning\npotentially\slarge\svalues\sfrom\sthe\sDBSTAT\svirtual\stable,\sto\savoid\sinteger\noverflows\sin\sthe\sresult.\n[forum:/forumpost/ada2ab044f|Forum\spost\sada2ab044f]. -D 2022-12-02T15:31:47.541 +C For\sthe\ssqlite3_bind\sand\ssqlite3_result\sinterfaces\sfor\sUTF16\sstrings,\sround\nthe\snumber\sof\sbytes\sdown\sto\sthe\snext\seven\snumber,\sto\savoid\screating\sa\sUTF16\nstring\sthat\sis\san\sodd\snumber\sof\sbytes.\n[forum:/forumpost/411199488d065f83|Forum\spost\s411199488d065f83]. +D 2022-12-02T17:52:52.611 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -720,7 +720,7 @@ F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c 00648bd76fb2145c2d890312112bda446569da64ca70aaf1f2282cc6c2b22d14 F src/vdbe.h 58675f47dcf3105bab182c3ad3726efd60ffd003e954386904ac9107d0d2b743 F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f -F src/vdbeapi.c 1e8713d0b653acb43cd1bdf579c40e005c4844ea90f414f065946a83db3c27fb +F src/vdbeapi.c b75fe402b8c0db8839aa9968f917bae49361a364ed28069b27b19ac7761e48b0 F src/vdbeaux.c 8ebe337e82d99cf3b01cd4fd67103a5b0054d53fee8456b90dbeba46ebf97ceb F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0881f3e92364b3dc81443220451e853c0763efaf97653a6348bc39bd9bdb23ad -R e242bef08302ff1ff3602cd93b1159c9 +P 5652154a8c93cf3b1ff6c2e55e94abbe995b0bb625f733461df20e006c2f13f8 +R abeafb6b0b2c2a0d499e07055ea2ac86 U drh -Z e49db422920b594adc6e6d22717359b3 +Z 85652852cafea1303e4df55e9d9dc80a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 19414a7f61..3229673c36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5652154a8c93cf3b1ff6c2e55e94abbe995b0bb625f733461df20e006c2f13f8 \ No newline at end of file +b57e3c3db00a6bc6db20c82530479f9eba7e37b731f0da6fe81682e84c7ac916 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 04295342b9..f67ca828ca 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -505,7 +505,10 @@ void sqlite3_result_text64( ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + n &= ~(u64)1; + } if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ @@ -520,7 +523,7 @@ void sqlite3_result_text16( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16NATIVE, xDel); } void sqlite3_result_text16be( sqlite3_context *pCtx, @@ -529,7 +532,7 @@ void sqlite3_result_text16be( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16BE, xDel); } void sqlite3_result_text16le( sqlite3_context *pCtx, @@ -538,7 +541,7 @@ void sqlite3_result_text16le( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ @@ -1603,7 +1606,10 @@ int sqlite3_bind_text64( unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + nData &= ~(u16)1; + } return bindText(pStmt, i, zData, nData, xDel, enc); } #ifndef SQLITE_OMIT_UTF16 @@ -1611,10 +1617,10 @@ int sqlite3_bind_text16( sqlite3_stmt *pStmt, int i, const void *zData, - int nData, + int n, void (*xDel)(void*) ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); + return bindText(pStmt, i, zData, n & ~(u64)1, xDel, SQLITE_UTF16NATIVE); } #endif /* SQLITE_OMIT_UTF16 */ int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ From 95bc4d67bbc8abe0783605e5f98dc1689d213352 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 18:06:26 +0000 Subject: [PATCH 129/282] OPFS VFS: translate createSyncAccessHandle() exceptions which appear to be locking violations to SQLITE_BUSY. This seems to improve concurrency considerably even with a reduced retry count of 5 (was 6). FossilOrigin-Name: 0d36021d107d3afca190ad61c3380536ad0cc2d493d345d48f9f9c1191741128 --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 29 +++++++++++------------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 10 ++++++-- ext/wasm/tests/opfs/concurrency/test.js | 5 +++- manifest | 18 +++++++-------- manifest.uuid | 2 +- 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index d8234d5092..e77ff78092 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -263,21 +263,18 @@ const installAsyncProxy = function(self){ } }; GetSyncHandleError.convertRc = (e,rc)=>{ - if(0){ - /* This approach makes the very wild assumption that such a - failure _is_ a locking error. In practice that appears to be - the most common error, by far, but we cannot unambiguously - distinguish that from other errors. - - This approach is highly questionable. - - Note that even if we return SQLITE_IOERR_LOCK from here, - it bubbles up to the client as a plain I/O error. - */ - return (e instanceof GetSyncHandleError - && e.cause.name==='NoModificationAllowedError') - ? state.sq3Codes.SQLITE_IOERR_LOCK - : rc; + if(1){ + return ( + e instanceof GetSyncHandleError + && ((e.cause.name==='NoModificationAllowedError') + /* Inconsistent exception.name from Chrome/ium with the + same exception.message text: */ + || (e.cause.name==='DOMException' + && 0===e.cause.message.indexOf('Access Handles cannot'))) + ) ? ( + /*console.warn("SQLITE_BUSY",e),*/ + state.sq3Codes.SQLITE_BUSY + ) : rc; }else{ return rc; } @@ -298,7 +295,7 @@ const installAsyncProxy = function(self){ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, msBase = state.asyncIdleWaitTime * 3; + const maxTries = 5, msBase = state.asyncIdleWaitTime * 2; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 7b0a0b3228..85e65a4aee 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -277,7 +277,7 @@ const installOpfsVfs = function callee(options){ of this value is also used for determining how long to wait on lock contention to free up. */ - state.asyncIdleWaitTime = 100; + state.asyncIdleWaitTime = 150; /** Whether the async counterpart should log exceptions to the serialization channel. That produces a great deal of @@ -636,6 +636,12 @@ const installOpfsVfs = function callee(options){ a[i] = f._chars[ndx]; } return a.join(""); + /* + An alternative impl. with an unpredictable length + but much simpler: + + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) + */ }; /** @@ -1159,7 +1165,7 @@ const installOpfsVfs = function callee(options){ "pragma journal_mode=persist;", /* Set a default busy-timeout handler to help OPFS dbs deal with multi-tab/multi-worker contention. */ - "pragma busy_timeout=3000;", + "pragma busy_timeout=5000;", /* This vfs benefits hugely from cache on moderate/large speedtest1 --size 50 and --size 100 workloads. We currently diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 044d343745..b994a8e79f 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -69,6 +69,7 @@ options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; + options.noUnlink = !!urlArgsHtml.has('no-unlink'); const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); @@ -124,7 +125,9 @@ for(let i = 0; i < options.workerCount; ++i){ stdout("Launching worker..."); workers.push(new Worker( - workers.uri+'&workerId='+(i+1)+(i ? '' : '&unlink-db') + workers.uri+'&workerId='+(i+1)+( + (i || options.noUnlink) ? '' : '&unlink-db' + ) )); } // Have to delay onmessage assignment until after the loop diff --git a/manifest b/manifest index a3d352d1d1..ccaa918011 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\ssqlite3_bind\sand\ssqlite3_result\sinterfaces\sfor\sUTF16\sstrings,\sround\nthe\snumber\sof\sbytes\sdown\sto\sthe\snext\seven\snumber,\sto\savoid\screating\sa\sUTF16\nstring\sthat\sis\san\sodd\snumber\sof\sbytes.\n[forum:/forumpost/411199488d065f83|Forum\spost\s411199488d065f83]. -D 2022-12-02T17:52:52.611 +C OPFS\sVFS:\stranslate\screateSyncAccessHandle()\sexceptions\swhich\sappear\sto\sbe\slocking\sviolations\sto\sSQLITE_BUSY.\sThis\sseems\sto\simprove\sconcurrency\sconsiderably\seven\swith\sa\sreduced\sretry\scount\sof\s5\s(was\s6). +D 2022-12-02T18:06:26.286 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -508,9 +508,9 @@ F ext/wasm/api/sqlite3-api-oo1.js c8b6c9ccb64cf93ca990ac689e98963735110aec21f98e F ext/wasm/api/sqlite3-api-prologue.js 42d6b316b542cf8e086f2f272460deb72dff184f1438a3377383cab99b08070b F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 9963c78bf6e5ccb5ba28e8597851bd9d980e86803b6d341cc985e586aef10c82 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 132e27f2172411020bb3a35ce5897c6437a960b4d744e7b8bdf3c6e3e32369dd F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 654f37fd6312d3bb0d067b21ad42f9dcfd629fd34ace892e67e06143a65dc6d0 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 146fbdac3c2ffa006ad4e95430396496caca0584c70623ae64e6c452a82c0e7f F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 733bc939f93caef0df0b3ebfea14cbd528da580fdef1a35b1f69c2b3e044c7b7 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -557,7 +557,7 @@ F ext/wasm/tester1-worker.html ead6bdcc6cca221deb0dc9855a56f376351dbf2294fd7978c F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js e73a91eba4b59aaadd98f383c00a5101dbbbc52d937fff3162fc4761986f4a88 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 -F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a +F ext/wasm/tests/opfs/concurrency/test.js 9315339ed27849e65890eda924a516562936525a4f3f162fa71aeb489b9dc707 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 7ab655788bf0b52dce4538acbd5b11cdbe77edd36a14af5dec6dfe1ec4ab25fc @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5652154a8c93cf3b1ff6c2e55e94abbe995b0bb625f733461df20e006c2f13f8 -R abeafb6b0b2c2a0d499e07055ea2ac86 -U drh -Z 85652852cafea1303e4df55e9d9dc80a +P b57e3c3db00a6bc6db20c82530479f9eba7e37b731f0da6fe81682e84c7ac916 +R 8232ca838233bb862b578dd17ef993c2 +U stephan +Z a512e857bd2ba2b089c64ccfca3b82d0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3229673c36..3568e89033 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b57e3c3db00a6bc6db20c82530479f9eba7e37b731f0da6fe81682e84c7ac916 \ No newline at end of file +0d36021d107d3afca190ad61c3380536ad0cc2d493d345d48f9f9c1191741128 \ No newline at end of file From bb4e4a4840530da37c6fdaabc9769a1996f25809 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 2 Dec 2022 18:56:37 +0000 Subject: [PATCH 130/282] Minor internal tweaks to the OPFS VFS. Resolve a missing result code which lead to a null deref in xFileSize(). FossilOrigin-Name: 57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 8 ++++---- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 16 +++++++++++----- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index e77ff78092..339ec126d0 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -295,7 +295,8 @@ const installAsyncProxy = function(self){ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 5, msBase = state.asyncIdleWaitTime * 2; + const maxTries = 6, + msBase = state.asyncIdleWaitTime * 2; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -515,15 +516,14 @@ const installAsyncProxy = function(self){ xFileSize: async function(fid/*sqlite3_file pointer*/){ mTimeStart('xFileSize'); const fh = __openFiles[fid]; - let rc; + let rc = 0; wTimeStart('xFileSize'); try{ affirmLocked('xFileSize',fh); const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); state.s11n.serialize(Number(sz)); - rc = 0; }catch(e){ - state.s11n.storeException(2,e); + state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); } await releaseImplicitLock(fh); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 85e65a4aee..4dc145a614 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -284,7 +284,7 @@ const installOpfsVfs = function callee(options){ noise for seemingly innocuous things like xAccess() checks for missing files, so this option may have one of 3 values: - 0 = no exception logging + 0 = no exception logging. 1 = only log exceptions for "significant" ops like xOpen(), xRead(), and xWrite(). @@ -363,6 +363,7 @@ const installOpfsVfs = function callee(options){ [ 'SQLITE_ACCESS_EXISTS', 'SQLITE_ACCESS_READWRITE', + 'SQLITE_BUSY', 'SQLITE_ERROR', 'SQLITE_IOERR', 'SQLITE_IOERR_ACCESS', @@ -706,10 +707,15 @@ const installOpfsVfs = function callee(options){ }, xFileSize: function(pFile,pSz64){ mTimeStart('xFileSize'); - const rc = opRun('xFileSize', pFile); + let rc = opRun('xFileSize', pFile); if(0==rc){ - const sz = state.s11n.deserialize()[0]; - wasm.setMemValue(pSz64, sz, 'i64'); + try { + const sz = state.s11n.deserialize()[0]; + wasm.setMemValue(pSz64, sz, 'i64'); + }catch(e){ + error("Unexpected error reading xFileSize() result:",e); + rc = state.sq3Codes.SQLITE_IOERR; + } } mTimeEnd(); return rc; @@ -1160,7 +1166,7 @@ const installOpfsVfs = function callee(options){ [ /* Truncate journal mode is faster than delete for this vfs, per speedtest1. That gap seems to have closed with - Chome version 108 or 109, but "persist" is very roughly 5-6% + Chrome version 108 or 109, but "persist" is very roughly 5-6% faster than truncate in initial tests. */ "pragma journal_mode=persist;", /* Set a default busy-timeout handler to help OPFS dbs diff --git a/manifest b/manifest index ccaa918011..61b48477c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sVFS:\stranslate\screateSyncAccessHandle()\sexceptions\swhich\sappear\sto\sbe\slocking\sviolations\sto\sSQLITE_BUSY.\sThis\sseems\sto\simprove\sconcurrency\sconsiderably\seven\swith\sa\sreduced\sretry\scount\sof\s5\s(was\s6). -D 2022-12-02T18:06:26.286 +C Minor\sinternal\stweaks\sto\sthe\sOPFS\sVFS.\sResolve\sa\smissing\sresult\scode\swhich\slead\sto\sa\snull\sderef\sin\sxFileSize(). +D 2022-12-02T18:56:37.557 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -508,9 +508,9 @@ F ext/wasm/api/sqlite3-api-oo1.js c8b6c9ccb64cf93ca990ac689e98963735110aec21f98e F ext/wasm/api/sqlite3-api-prologue.js 42d6b316b542cf8e086f2f272460deb72dff184f1438a3377383cab99b08070b F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 132e27f2172411020bb3a35ce5897c6437a960b4d744e7b8bdf3c6e3e32369dd +F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 146fbdac3c2ffa006ad4e95430396496caca0584c70623ae64e6c452a82c0e7f +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b8889f182ba7b5c1be8bcf6f47b538519e3fff0e98d7cf8b9d1fa4a9afdfb8ce F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 733bc939f93caef0df0b3ebfea14cbd528da580fdef1a35b1f69c2b3e044c7b7 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b57e3c3db00a6bc6db20c82530479f9eba7e37b731f0da6fe81682e84c7ac916 -R 8232ca838233bb862b578dd17ef993c2 +P 0d36021d107d3afca190ad61c3380536ad0cc2d493d345d48f9f9c1191741128 +R 6eba0a795b491027bc2604925d67afb7 U stephan -Z a512e857bd2ba2b089c64ccfca3b82d0 +Z 379ecd060fe213536d8ac858396e84c5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3568e89033..2c043a6432 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d36021d107d3afca190ad61c3380536ad0cc2d493d345d48f9f9c1191741128 \ No newline at end of file +57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 \ No newline at end of file From 231ff4b027b077ce9ac0c829cd9cee4108ca9b7a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Dec 2022 20:32:22 +0000 Subject: [PATCH 131/282] Enhance the sqlite3_stmt_scanstatus() API and add sqlite3_stmt_scanstatus_v2(). For creation of easier to read query performance reports. FossilOrigin-Name: 55800833645739efeddcacef464c623931cb6aeb43f4219b4e4faf473c25c8bb --- manifest | 44 ++++++++------ manifest.uuid | 2 +- src/expr.c | 7 ++- src/select.c | 7 ++- src/shell.c.in | 127 +++++++++++++++++++++++++++++---------- src/sqlite.h.in | 12 ++++ src/sqliteInt.h | 4 +- src/test1.c | 107 ++++++++++++++++++++++++++------- src/util.c | 4 +- src/vdbe.c | 55 +++++++++++------ src/vdbe.h | 16 +++-- src/vdbeInt.h | 7 ++- src/vdbeapi.c | 83 +++++++++++++++++++++++-- src/vdbeaux.c | 55 +++++++++++------ src/wherecode.c | 2 + test/scanstatus.test | 14 +++-- test/scanstatus2.test | 137 ++++++++++++++++++++++++++++++++++++++++++ 17 files changed, 546 insertions(+), 137 deletions(-) create mode 100644 test/scanstatus2.test diff --git a/manifest b/manifest index 61b48477c1..deb137d25e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\stweaks\sto\sthe\sOPFS\sVFS.\sResolve\sa\smissing\sresult\scode\swhich\slead\sto\sa\snull\sderef\sin\sxFileSize(). -D 2022-12-02T18:56:37.557 +C Enhance\sthe\ssqlite3_stmt_scanstatus()\sAPI\sand\sadd\ssqlite3_stmt_scanstatus_v2().\sFor\screation\sof\seasier\sto\sread\squery\sperformance\sreports. +D 2022-12-02T20:32:22.142 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -595,7 +595,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c a56a7ad1163a9888d46cd5820be2e65354fb1aa04ed6909f7c3e5831e0ee2c29 F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 9e7fadc664b938c18f006be0d4f6669888f9302756ee204420c7eccaed5435a6 +F src/expr.c 7b44a97e14ebb679c14ced7c3db420fd6293c33f8bd668fef37746e76afdc746 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -645,17 +645,17 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 -F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c -F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 +F src/select.c b1301741aed7014beafb4cbc524d725865a890c7619d784e4ffb04379259ae6c +F src/shell.c.in 0e45a91da5426563c5b0c8f66fd51d5a5fcb3cc17faa44cd4ea8cc386bde146b +F src/sqlite.h.in 3ba99a3c2b414b9ad36d06e2f273ab6077f5f74e4d79630aee047394b92f7c7f F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 4ddd98e423276714479f9f22dbbda050e8ef99aa97e7e26bf0bdf58acef0ca42 +F src/sqliteInt.h 0c9934acd88e0fa14f0d4743233b4cd7bd7bbc84099ba3cad50d8e69676ce2b9 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 4e64ba300a5a26e0f1170e09032429faeb65e45e8f3d1a7833e8edb69fc2979e -F src/test1.c 09ad3f2bf5f0efe70eb16e460054954b36be8ed09dfcafa0642b37d25d379f6f +F src/test1.c 98f4a4525e10b0df164ba7f33950824348bb29fd582101e0f93b1af72f493154 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -715,13 +715,13 @@ F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c 4e42c338d982e3f139004467bf0f2e9d679d519c7c75718fcf4ff1811401b03c +F src/util.c 313f3154e2b85a447326f5dd15de8d31a4df6ab0c3579bd58f426ff634ec9050 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 00648bd76fb2145c2d890312112bda446569da64ca70aaf1f2282cc6c2b22d14 -F src/vdbe.h 58675f47dcf3105bab182c3ad3726efd60ffd003e954386904ac9107d0d2b743 -F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f -F src/vdbeapi.c b75fe402b8c0db8839aa9968f917bae49361a364ed28069b27b19ac7761e48b0 -F src/vdbeaux.c 8ebe337e82d99cf3b01cd4fd67103a5b0054d53fee8456b90dbeba46ebf97ceb +F src/vdbe.c 5f5cc749c999d65e2115bb3ec555f7ac0ecfbda7f47526272d53a032e9d083b1 +F src/vdbe.h 247b4ffaad9ba2b2944a2b3084936ae5df684a42151552f32f074ec847c1402b +F src/vdbeInt.h 9628718a289cf3bf490db1435019773ccc0874e1a505502ea477c5a98e649128 +F src/vdbeapi.c 592c3d7be7c23d202cf5bfb96fc970ced7aaaa1b51052868f1deb91bbbb0add8 +F src/vdbeaux.c 600edaa2d20eb28aea9707148f4edcf12b349418fd2e3f1de92e5d23fdfc0eb8 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -734,7 +734,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c bf470b5d1ba03af8d558a0c98cc1fa97b330e03a198a7af61895e5a2e8d93f20 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c ee52c2781c36004d23c85bf111063b78fc16e5e1b6a0d424326af8bf90babb0b +F src/wherecode.c f7121f4c29e45ba7d38c91448e250ae5197a4b75e9fa75996ba6d5c81a3cc6e5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1456,7 +1456,8 @@ F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 -F test/scanstatus.test 9a0ed37ab6d57b50567282788fffdf832d9b16739ecc41bff9d77a8d767cf317 +F test/scanstatus.test 7dbcfd6adc6a8df6abc59f83d6da5a27e1bce0b2f6fa55147c8176d7c44e0450 +F test/scanstatus2.test c618c5843257e290647dd63672c16b0f6488faebc64ba9b2ec5b4de3bb39a7cc F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2065,8 +2066,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d36021d107d3afca190ad61c3380536ad0cc2d493d345d48f9f9c1191741128 -R 6eba0a795b491027bc2604925d67afb7 -U stephan -Z 379ecd060fe213536d8ac858396e84c5 +P 57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 +R 49b81422f59a6a7338c393b5956057f1 +T *branch * scanstatus_v2 +T *sym-scanstatus_v2 * +T -sym-trunk * +U dan +Z e87234282222fcab550cd1e86b10e65b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c043a6432..23a6edc914 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 \ No newline at end of file +55800833645739efeddcacef464c623931cb6aeb43f4219b4e4faf473c25c8bb \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index d30ae97668..07e44a1876 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3257,6 +3257,9 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ SelectDest dest; /* How to deal with SELECT result */ int nReg; /* Registers to allocate */ Expr *pLimit; /* New limit expression */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; +#endif Vdbe *v = pParse->pVdbe; assert( v!=0 ); @@ -3309,7 +3312,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ** In both cases, the query is augmented with "LIMIT 1". Any ** preexisting limit is discarded in place of the new LIMIT 1. */ - ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY %d", + ExplainQueryPlan2(addrExplain, (pParse, 1, "%sSCALAR SUBQUERY %d", addrOnce?"":"CORRELATED ", pSel->selId)); nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); @@ -3353,6 +3356,8 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ if( addrOnce ){ sqlite3VdbeJumpHere(v, addrOnce); } + sqlite3VdbeScanStatusEnd(v, addrExplain); + /* Subroutine return */ assert( ExprUseYSub(pExpr) ); diff --git a/src/select.c b/src/select.c index 0cd28b543e..7403725252 100644 --- a/src/select.c +++ b/src/select.c @@ -7277,6 +7277,9 @@ int sqlite3Select( ** the same view can reuse the materialization. */ int topAddr; int onceAddr = 0; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; +#endif pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp0(v, OP_Goto); @@ -7292,7 +7295,8 @@ int sqlite3Select( VdbeNoopComment((v, "materialize %!S", pItem)); } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem)); + + ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); dest.zAffSdst = sqlite3TableAffinityStr(db, pItem->pTab); sqlite3Select(pParse, pSub, &dest); sqlite3DbFree(db, dest.zAffSdst); @@ -7301,6 +7305,7 @@ int sqlite3Select( if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); + sqlite3VdbeScanStatusEnd(v, addrExplain); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ diff --git a/src/shell.c.in b/src/shell.c.in index a829fa2387..7cbd6b3c47 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -2082,7 +2082,7 @@ static void eqp_render_level(ShellState *p, int iEqpId){ /* ** Display and reset the EXPLAIN QUERY PLAN data */ -static void eqp_render(ShellState *p){ +static void eqp_render(ShellState *p, i64 nCycle){ EQPGraphRow *pRow = p->sGraph.pRow; if( pRow ){ if( pRow->zText[0]=='-' ){ @@ -2093,6 +2093,8 @@ static void eqp_render(ShellState *p){ utf8_printf(p->out, "%s\n", pRow->zText+3); p->sGraph.pRow = pRow->pNext; sqlite3_free(pRow); + }else if( nCycle>0 ){ + utf8_printf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); }else{ utf8_printf(p->out, "QUERY PLAN\n"); } @@ -2970,6 +2972,35 @@ static int display_stats( return 0; } + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + } + } + ret++; + } + return ret; +} +#endif + /* ** Display scan stats. */ @@ -2981,40 +3012,70 @@ static void display_scanstats( UNUSED_PARAMETER(db); UNUSED_PARAMETER(pArg); #else - int i, k, n, mx; - raw_printf(pArg->out, "-------- scanstats --------\n"); - mx = 0; - for(k=0; k<=mx; k++){ - double rEstLoop = 1.0; - for(i=n=0; 1; i++){ - sqlite3_stmt *p = pArg->pStmt; - sqlite3_int64 nLoop, nVisit; - double rEst; - int iSid; - const char *zExplain; - if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ - break; + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *p = pArg->pStmt; + int ii = 0; + i64 nTotal = 0; + int nWidth = 0; + eqp_reset(pArg); + + for(ii=0; 1; ii++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + n = strlen(z) + scanStatsHeight(p, ii)*3; + if( n>nWidth ) nWidth = n; + } + nWidth += 4; + + sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(ii=0; 1; ii++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *z = 0; + char *zText = 0; + double rEst = 0.0; + + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + + zText = sqlite3_mprintf("%s", z); + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + char *z = 0; + if( nCycle>=0 ){ + z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, + nCycle, ((nCycle*100)+nTotal/2) / nTotal + ); } - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); - if( iSid>mx ) mx = iSid; - if( iSid!=k ) continue; - if( n==0 ){ - rEstLoop = (double)nLoop; - if( k>0 ) raw_printf(pArg->out, "-------- subquery %d -------\n", k); + if( nLoop>=0 ){ + z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); } - n++; - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - utf8_printf(pArg->out, "Loop %2d: %s\n", n, zExplain); - rEstLoop *= rEst; - raw_printf(pArg->out, - " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst + if( nRow>=0 ){ + z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); + } + + zText = sqlite3_mprintf( + "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z ); } + + eqp_append(pArg, iId, iPid, zText); + sqlite3_free(zText); } - raw_printf(pArg->out, "---------------------------\n"); + + eqp_render(pArg, nTotal); #endif } @@ -3938,10 +3999,10 @@ static int shell_exec( int iEqpId = sqlite3_column_int(pExplain, 0); int iParentId = sqlite3_column_int(pExplain, 1); if( zEQPLine==0 ) zEQPLine = ""; - if( zEQPLine[0]=='-' ) eqp_render(pArg); + if( zEQPLine[0]=='-' ) eqp_render(pArg, 0); eqp_append(pArg, iEqpId, iParentId, zEQPLine); } - eqp_render(pArg); + eqp_render(pArg, 0); } sqlite3_finalize(pExplain); sqlite3_free(zEQP); @@ -3990,7 +4051,7 @@ static int shell_exec( bind_prepared_stmt(pArg, pStmt); exec_prepared_stmt(pArg, pStmt); explain_data_delete(pArg); - eqp_render(pArg); + eqp_render(pArg, 0); /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 369d6b6022..91466399be 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -9946,6 +9946,8 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 #define SQLITE_SCANSTAT_SELECTID 5 +#define SQLITE_SCANSTAT_PARENTID 6 +#define SQLITE_SCANSTAT_NCYCLE 7 /* ** CAPI3REF: Prepared Statement Scan Status @@ -9985,6 +9987,16 @@ int sqlite3_stmt_scanstatus( void *pOut /* Result written here */ ); +int sqlite3_stmt_scanstatus_v2( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + int flags, /* Mask of flags defined below */ + void *pOut /* Result written here */ +); + +#define SQLITE_SCANSTAT_COMPLEX 0x0001 + /* ** CAPI3REF: Zero Scan-Status Counters ** METHOD: sqlite3_stmt diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4711e4f094..5a19058bd1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -5565,7 +5565,9 @@ const char **sqlite3CompileOptions(int *pnOpt); int sqlite3KvvfsInit(void); #endif -#if defined(VDBE_PROFILE) || defined(SQLITE_PERFORMANCE_TRACE) +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) sqlite3_uint64 sqlite3Hwtime(void); #endif diff --git a/src/test1.c b/src/test1.c index c127d49870..a314e90d84 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2188,7 +2188,7 @@ static int SQLITE_TCLAPI test_stmt_status( #ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** Usage: sqlite3_stmt_scanstatus STMT IDX +** Usage: sqlite3_stmt_scanstatus ?-flags FLAGS? STMT IDX */ static int SQLITE_TCLAPI test_stmt_scanstatus( void * clientData, @@ -2203,36 +2203,99 @@ static int SQLITE_TCLAPI test_stmt_scanstatus( const char *zExplain; sqlite3_int64 nLoop; sqlite3_int64 nVisit; + sqlite3_int64 nCycle; double rEst; int res; + int flags = 0; + int iSelectId = 0; + int iParentId = 0; - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "STMT IDX"); + if( objc==5 ){ + struct Flag { + const char *zFlag; + int flag; + } aTbl[] = { + {"complex", SQLITE_SCANSTAT_COMPLEX}, + {0, 0} + }; + + Tcl_Obj **aFlag = 0; + int nFlag = 0; + int ii; + + if( Tcl_ListObjGetElements(interp, objv[2], &nFlag, &aFlag) ){ + return TCL_ERROR; + } + for(ii=0; iiaOp; /* Copy of p->aOp */ Op *pOp = aOp; /* Current operation */ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) - Op *pOrigOp; /* Value of pOp at the top of the loop */ -#endif #ifdef SQLITE_DEBUG + Op *pOrigOp; /* Value of pOp at the top of the loop */ int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ #endif int rc = SQLITE_OK; /* Value to return */ @@ -727,8 +724,8 @@ int sqlite3VdbeExec( Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ -#ifdef VDBE_PROFILE - u64 start; /* CPU clock count at start of opcode */ +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 *pnCycle = 0; #endif /*** INSERT STACK UNION HERE ***/ @@ -791,12 +788,17 @@ int sqlite3VdbeExec( assert( rc==SQLITE_OK ); assert( pOp>=aOp && pOp<&aOp[p->nOp]); -#ifdef VDBE_PROFILE - start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); -#endif nVmStep++; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - if( p->anExec ) p->anExec[(int)(pOp-aOp)]++; +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + if( p->anExec ){ + assert( p->anExec && p->anCycle ); + p->anExec[(int)(pOp-aOp)]++; + pnCycle = &p->anCycle[pOp-aOp]; +# ifdef VDBE_PROFILE + if( sqlite3NProfileCnt==0 ) +# endif + *pnCycle -= sqlite3Hwtime(); + } #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -858,7 +860,7 @@ int sqlite3VdbeExec( } } #endif -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#ifdef SQLITE_DEBUG pOrigOp = pOp; #endif @@ -7150,8 +7152,9 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) pFrame->anExec = p->anExec; + pFrame->anCycle = p->anCycle; #endif #ifdef SQLITE_DEBUG pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; @@ -7189,8 +7192,9 @@ case OP_Program: { /* jump */ memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8); p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) p->anExec = 0; + p->anCycle = 0; #endif #ifdef SQLITE_DEBUG /* Verify that second and subsequent executions of the same trigger do not @@ -8727,11 +8731,16 @@ default: { /* This is really OP_Noop, OP_Explain */ *****************************************************************************/ } -#ifdef VDBE_PROFILE - { - u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); - if( endTime>start ) pOrigOp->cycles += endTime - start; - pOrigOp->cnt++; +#if defined(VDBE_PROFILE) + assert( pnCycle ); + if( pnCycle ){ + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; + } +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; } #endif @@ -8808,6 +8817,14 @@ abort_due_to_error: ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: +#if defined(VDBE_PROFILE) + if( pnCycle ){ + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + } +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ) *pnCycle += sqlite3Hwtime(); +#endif + #ifndef SQLITE_OMIT_PROGRESS_CALLBACK while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ nProgressLimit += db->nProgressOps; diff --git a/src/vdbe.h b/src/vdbe.h index a244468b52..16690f0855 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -67,10 +67,6 @@ struct VdbeOp { #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS char *zComment; /* Comment to improve readability */ #endif -#ifdef VDBE_PROFILE - u32 cnt; /* Number of times this instruction was executed */ - u64 cycles; /* Total time spent executing this instruction */ -#endif #ifdef SQLITE_VDBE_COVERAGE u32 iSrcLine; /* Source-code line that generated this opcode ** with flags in the upper 8 bits */ @@ -205,14 +201,20 @@ void sqlite3VdbeEndCoroutine(Vdbe*,int); #endif VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp,int iLineno); #ifndef SQLITE_OMIT_EXPLAIN - void sqlite3VdbeExplain(Parse*,u8,const char*,...); + int sqlite3VdbeExplain(Parse*,u8,const char*,...); void sqlite3VdbeExplainPop(Parse*); int sqlite3VdbeExplainParent(Parse*); # define ExplainQueryPlan(P) sqlite3VdbeExplain P +# ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define ExplainQueryPlan2(V,P) (V = sqlite3VdbeExplain P) +# else +# define ExplainQueryPlan2(V,P) ExplainQueryPlan(P) +# endif # define ExplainQueryPlanPop(P) sqlite3VdbeExplainPop(P) # define ExplainQueryPlanParent(P) sqlite3VdbeExplainParent(P) #else # define ExplainQueryPlan(P) +# define ExplainQueryPlan2(V,P) # define ExplainQueryPlanPop(P) # define ExplainQueryPlanParent(P) 0 # define sqlite3ExplainBreakpoint(A,B) /*no-op*/ @@ -385,8 +387,10 @@ int sqlite3VdbeBytecodeVtabInit(sqlite3*); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +void sqlite3VdbeScanStatusEnd(Vdbe*, int); #else -# define sqlite3VdbeScanStatus(a,b,c,d,e) +# define sqlite3VdbeScanStatus(a,b,c,d,e,f) +# define sqlite3VdbeScanStatusEnd(a,b) #endif #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 0c6301c8f1..b8642da508 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -172,6 +172,7 @@ struct VdbeFrame { VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ i64 *anExec; /* Event counters from parent frame */ + u64 *anCycle; /* Cycle counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ u8 *aOnce; /* Bitmask used by OP_Once */ @@ -391,6 +392,7 @@ typedef unsigned bft; /* Bit Field Type */ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ + int addrEndRange; /* End of range (inclusive) of times to sum */ int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ int iSelectID; /* The "Select-ID" for this loop */ @@ -482,8 +484,11 @@ struct Vdbe { u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ AuxData *pAuxData; /* Linked list of auxdata allocations */ -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) i64 *anExec; /* Number of times each op has been executed */ + u64 *anCycle; /* Sum of sqlite3HwTime() spent in each opcode */ +#endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS int nScan; /* Entries in aScan[] */ ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ #endif diff --git a/src/vdbeapi.c b/src/vdbeapi.c index f67ca828ca..90b0cc60c8 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2111,23 +2111,60 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ /* ** Return status data for a single loop within query pStmt. */ -int sqlite3_stmt_scanstatus( +int sqlite3_stmt_scanstatus_v2( sqlite3_stmt *pStmt, /* Prepared statement being queried */ - int idx, /* Index of loop to report on */ + int iScan, /* Index of loop to report on */ int iScanStatusOp, /* Which metric to return */ + int flags, void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; ScanStatus *pScan; - if( idx<0 || idx>=p->nScan ) return 1; - pScan = &p->aScan[idx]; + int idx; + + /* If the v2 flag is clear, then this function must ignore any ScanStatus + ** structures with ScanStatus.addrLoop set to 0. */ + if( iScan<0 ){ + int ii; + if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ + i64 res = 0; + for(ii=0; iinOp; ii++){ + res += p->anCycle[ii]; + } + *(i64*)pOut = res; + return 0; + } + return 1; + } + if( flags & SQLITE_SCANSTAT_COMPLEX ){ + idx = iScan; + pScan = &p->aScan[idx]; + }else{ + for(idx=0; idxnScan; idx++){ + pScan = &p->aScan[idx]; + if( pScan->addrLoop ){ + iScan--; + if( iScan<0 ) break; + } + } + } + if( idx>=p->nScan ) return 1; + switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + if( pScan->addrLoop>0 ){ + *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_NVISIT: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + if( pScan->addrVisit>0 ){ + *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_EST: { @@ -2160,6 +2197,26 @@ int sqlite3_stmt_scanstatus( } break; } + case SQLITE_SCANSTAT_PARENTID: { + if( pScan->addrExplain ){ + *(int*)pOut = p->aOp[ pScan->addrExplain ].p2; + }else{ + *(int*)pOut = -1; + } + break; + } + case SQLITE_SCANSTAT_NCYCLE: { + i64 res = -1; + if( pScan->addrEndRange ){ + int ii; + res = 0; + for(ii=pScan->addrExplain; ii<=pScan->addrEndRange; ii++){ + res += p->anCycle[ii]; + } + } + *(i64*)pOut = res; + break; + } default: { return 1; } @@ -2167,11 +2224,25 @@ int sqlite3_stmt_scanstatus( return 0; } +/* +** Return status data for a single loop within query pStmt. +*/ +int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int iScan, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + return sqlite3_stmt_scanstatus_v2(pStmt, iScan, iScanStatusOp, 0, pOut); +} + + /* ** Zero all counters associated with the sqlite3_stmt_scanstatus() data. */ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; memset(p->anExec, 0, p->nOp * sizeof(i64)); + memset(p->anCycle, 0, p->nOp * sizeof(u64)); } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index fd196f37bc..e7cd61f05d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -268,10 +268,6 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ test_addop_breakpoint(i, &p->aOp[i]); } #endif -#ifdef VDBE_PROFILE - pOp->cycles = 0; - pOp->cnt = 0; -#endif #ifdef SQLITE_VDBE_COVERAGE pOp->iSrcLine = 0; #endif @@ -439,8 +435,9 @@ void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ ** If the bPush flag is true, then make this opcode the parent for ** subsequent Explains until sqlite3VdbeExplainPop() is called. */ -void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ -#ifndef SQLITE_DEBUG +int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ + int addr = 0; +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. ** But omit them (for performance) during production builds */ if( pParse->explain==2 ) @@ -455,13 +452,15 @@ void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ va_end(ap); v = pParse->pVdbe; iThis = v->nOp; - sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, + addr = sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, zMsg, P4_DYNAMIC); sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetLastOp(v)->p4.z); if( bPush){ pParse->addrExplain = iThis; } + sqlite3VdbeScanStatus(v, iThis, 0, 0, 0, 0); } + return addr; } /* @@ -1119,6 +1118,7 @@ void sqlite3VdbeScanStatus( aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ ScanStatus *pNew = &aNew[p->nScan++]; + memset(pNew, 0, sizeof(ScanStatus)); pNew->addrExplain = addrExplain; pNew->addrLoop = addrLoop; pNew->addrVisit = addrVisit; @@ -1127,6 +1127,19 @@ void sqlite3VdbeScanStatus( p->aScan = aNew; } } + +void sqlite3VdbeScanStatusEnd(Vdbe *p, int addrExplain){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + pScan->addrEndRange = sqlite3VdbeCurrentAddr(p)-1; + } +} #endif @@ -2424,7 +2437,7 @@ static void *allocSpace( ** running it. */ void sqlite3VdbeRewind(Vdbe *p){ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#if defined(SQLITE_DEBUG) int i; #endif assert( p!=0 ); @@ -2452,10 +2465,8 @@ void sqlite3VdbeRewind(Vdbe *p){ p->iStatement = 0; p->nFkConstraint = 0; #ifdef VDBE_PROFILE - for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; - } + memset(p->anExec, 0, sizeof(i64)*p->nOp); + memset(p->anCycle, 0, sizeof(u64)*p->nOp); #endif } @@ -2563,8 +2574,9 @@ void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); + p->anCycle = allocSpace(&x, 0, p->nOp*sizeof(u64)); #endif if( x.nNeeded ){ x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); @@ -2574,8 +2586,9 @@ void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); + p->anCycle = allocSpace(&x, p->anCycle, p->nOp*sizeof(u64)); #endif } } @@ -2591,8 +2604,9 @@ void sqlite3VdbeMakeReady( p->nMem = nMem; initMemArray(p->aMem, nMem, db, MEM_Undefined); memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) memset(p->anExec, 0, p->nOp*sizeof(i64)); + memset(p->anCycle, 0, p->nOp*sizeof(u64)); #endif } sqlite3VdbeRewind(p); @@ -2651,8 +2665,9 @@ static void closeCursorsInFrame(Vdbe *p){ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; closeCursorsInFrame(v); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) v->anExec = pFrame->anExec; + v->anCycle = pFrame->anCycle; #endif v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; @@ -3485,10 +3500,12 @@ int sqlite3VdbeReset(Vdbe *p){ } for(i=0; inOp; i++){ char zHdr[100]; + i64 cnt = p->anExec[i]; + i64 cycles = p->anCycle[i]; sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", - p->aOp[i].cnt, - p->aOp[i].cycles, - p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 + cnt, + cycles, + cnt>0 ? cycles/cnt : 0 ); fprintf(out, "%s", zHdr); sqlite3VdbePrintOp(out, i, &p->aOp[i]); diff --git a/src/wherecode.c b/src/wherecode.c index c731cc82f4..b744556fa8 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -270,6 +270,8 @@ int sqlite3WhereExplainBloomFilter( zMsg = sqlite3StrAccumFinish(&str); ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), pParse->addrExplain, 0, zMsg,P4_DYNAMIC); + + sqlite3VdbeScanStatus(v, sqlite3VdbeCurrentAddr(v)-1, 0, 0, 0, 0); return ret; } #endif /* SQLITE_OMIT_EXPLAIN */ diff --git a/test/scanstatus.test b/test/scanstatus.test index 46249f665d..caad70c5b4 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -36,7 +36,9 @@ proc do_scanstatus_test {tn res} { while {1} { set r [sqlite3_stmt_scanstatus $stmt $idx] if {[llength $r]==0} break - lappend ret {*}$r + foreach v {nLoop nVisit nEst zName zExplain} { + lappend ret $v [dict get $r $v] + } incr idx } @@ -312,8 +314,8 @@ do_execsql_test 5.1.1 { SELECT count(*) FROM t1 WHERE a IN (SELECT b FROM t1 AS ii) } {2} do_scanstatus_test 5.1.2 { - nLoop 1 nVisit 10 nEst 10.0 zName t1bc - zExplain {SCAN ii USING COVERING INDEX t1bc} + nLoop 1 nVisit 10 nEst 10.0 zName t1 + zExplain {SCAN ii} nLoop 1 nVisit 2 nEst 8.0 zName sqlite_autoindex_t1_1 zExplain {SEARCH t1 USING COVERING INDEX sqlite_autoindex_t1_1 (a=?)} } @@ -341,15 +343,15 @@ do_eqp_test 5.4.1 { SELECT count(*) FROM t1, t2 WHERE y = c; } { QUERY PLAN - |--SCAN t1 USING COVERING INDEX t1bc + |--SCAN t1 `--SEARCH t2 USING COVERING INDEX t2xy (ANY(x) AND y=?) } do_execsql_test 5.4.2 { SELECT count(*) FROM t1, t2 WHERE y = c; } {200} do_scanstatus_test 5.4.3 { - nLoop 1 nVisit 10 nEst 10.0 zName t1bc - zExplain {SCAN t1 USING COVERING INDEX t1bc} + nLoop 1 nVisit 10 nEst 10.0 zName t1 + zExplain {SCAN t1} nLoop 10 nVisit 200 nEst 56.0 zName t2xy zExplain {SEARCH t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} } diff --git a/test/scanstatus2.test b/test/scanstatus2.test new file mode 100644 index 0000000000..001797df2e --- /dev/null +++ b/test/scanstatus2.test @@ -0,0 +1,137 @@ +# 2014 November 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix scanstatus2 + +ifcapable !scanstatus { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES('a', 'b'); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t2 VALUES('e', 'f'); +} + +proc do_zexplain_test {v2 tn sql res} { + db eval $sql + set stmt [db version -last-stmt-ptr] + set idx 0 + set ret [list] + + set cmd sqlite3_stmt_scanstatus + set f [list] + if {$v2} { lappend f complex } + + while {1} { + set r [sqlite3_stmt_scanstatus -flags $f $stmt $idx] + if {[llength $r]==0} break + lappend ret [dict get $r zExplain] + incr idx + } + uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] +} + +proc get_cycles {stmt} { + set r [sqlite3_stmt_scanstatus $stmt -1] + dict get $r nCycle +} + +proc foreach_scan {varname stmt body} { + upvar $varname var + + for {set ii 0} {1} {incr ii} { + set r [sqlite3_stmt_scanstatus -flags complex $stmt $ii] + if {[llength $r]==0} break + array set var $r + uplevel $body + } +} + +proc get_eqp_graph {stmt iPar nIndent} { + set res "" + foreach_scan A $stmt { + if {$A(iParentId)==$iPar} { + set txt $A(zExplain) + if {$A(nCycle)>=0} { + append txt " (nCycle=$A(nCycle))" + } + append res "[string repeat - $nIndent]$txt\n" + append res [get_eqp_graph $stmt $A(iSelectId) [expr $nIndent+2]] + } + } + set res +} + +proc get_graph {stmt} { + set nCycle [get_cycles $stmt] + set res "QUERY (nCycle=$nCycle)\n" + append res [get_eqp_graph $stmt 0 2] +} + +proc do_graph_test {tn sql res} { + db eval $sql + set stmt [db version -last-stmt-ptr] + + set graph [string trim [get_graph $stmt]] + set graph [regsub -all {nCycle=[0-9]+} $graph nCycle=nnn] + uplevel [list do_test $tn [list set {} $graph] [string trim $res]] +} + + +do_zexplain_test 0 1.1 { + SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 +} { + {SCAN t2} + {SCAN t1} +} +do_zexplain_test 1 1.2 { + SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 +} { + {SCAN t2} + {CORRELATED SCALAR SUBQUERY 1} + {SCAN t1} +} + +do_graph_test 1.3 { + SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 +} { +QUERY (nCycle=nnn) +--SCAN t2 +--CORRELATED SCALAR SUBQUERY 1 (nCycle=nnn) +----SCAN t1 +} + +do_graph_test 1.4 { + WITH v2(x,y) AS MATERIALIZED ( + SELECT x,y FROM t2 + ) + SELECT * FROM t1, v2 ORDER BY y; +} { +QUERY (nCycle=nnn) +--MATERIALIZE v2 (nCycle=nnn) +----SCAN t2 +--SCAN v2 +--SCAN t1 +--USE TEMP B-TREE FOR ORDER BY +} + +finish_test + + From 1edd0a089c81558d199d075ddcf5845703c62667 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 Dec 2022 00:52:21 +0000 Subject: [PATCH 132/282] Tuning the query planner by adjusting the weights that predict the relative performance of sorting and index lookup. FossilOrigin-Name: 9f2806da4d88beceac2e81e05421f00481dd3dd100b096cd2ae6c828adb42ca7 --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/where.c | 18 ++++++++++++------ test/orderby1.test | 3 +++ test/where.test | 1 + 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 61b48477c1..a763dc6801 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\stweaks\sto\sthe\sOPFS\sVFS.\sResolve\sa\smissing\sresult\scode\swhich\slead\sto\sa\snull\sderef\sin\sxFileSize(). -D 2022-12-02T18:56:37.557 +C Tuning\sthe\squery\splanner\sby\sadjusting\sthe\sweights\sthat\spredict\sthe\srelative\nperformance\sof\ssorting\sand\sindex\slookup. +D 2022-12-03T00:52:21.776 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c bf470b5d1ba03af8d558a0c98cc1fa97b330e03a198a7af61895e5a2e8d93f20 +F src/where.c 5826b62ddcfc92979669cb5fb80f73d0df86bbfeefa1d757f5dc1f857cd628e7 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c ee52c2781c36004d23c85bf111063b78fc16e5e1b6a0d424326af8bf90babb0b F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -1368,7 +1368,7 @@ F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/optfuzz-db01.c 9f2fa80b8f84ebbf1f2e8b13421a4e0477fe300f6686fbd76cac1d2db66e0fdc F test/optfuzz-db01.txt 21f6bdeadc701cf11528276e2a55c70bfcb846ba42df327f979bd9e7b6ce7041 F test/optfuzz.c 690430a0bf0ad047d5a168bf52b05b2ee97aedaad8c14337e9eb5050faa64994 -F test/orderby1.test a4bba04b9c60a21e53486fbc173a596b29641a3b3a57a0f26a1cbef1360358e9 +F test/orderby1.test 02cfd870127a7342170b829175c5c53e9e7405744451ac1aeb2f7e2b0c18ca76 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 @@ -1884,7 +1884,7 @@ F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2a F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec F test/wapptest.tcl 1bea58a6a8e68a73f542ee4fca28b771b84ed803bd0c9e385087070b3d747b3c x -F test/where.test d13cd7c24e80009d2b54e2f7a8893c457afa49c64f99359c9eb3fe668ba1d9d4 +F test/where.test 98208c95b574269980132c347b4bdb8992c6d5fc30c1954938593336d12e7447 F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 @@ -2065,8 +2065,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d36021d107d3afca190ad61c3380536ad0cc2d493d345d48f9f9c1191741128 -R 6eba0a795b491027bc2604925d67afb7 -U stephan -Z 379ecd060fe213536d8ac858396e84c5 +P 57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 +R e47495a3cfe52da04033a615c7da71c2 +T *branch * qp-tuning +T *sym-qp-tuning * +T -sym-trunk * +U drh +Z bdcd0fcd9740989c0fc73b883156ab70 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c043a6432..d25f3c5d8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 \ No newline at end of file +9f2806da4d88beceac2e81e05421f00481dd3dd100b096cd2ae6c828adb42ca7 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c9eeabe8b6..f1d1df44c5 100644 --- a/src/where.c +++ b/src/where.c @@ -3472,7 +3472,7 @@ static int whereLoopAddBtree( sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; - sPk.szIdxRow = pTab->szTabRow; + sPk.szIdxRow = 1; /* Interior rows of IPK table are very small */ sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; @@ -4811,13 +4811,18 @@ static LogEst whereSortingCost( /* TUNING: Estimated cost of a full external sort, where N is ** the number of rows to sort is: ** - ** cost = (3.0 * N * log(N)). + ** cost = (K * N * log(N)). ** ** Or, if the order-by clause has X terms but only the last Y ** terms are out of order, then block-sorting will reduce the ** sorting cost to: ** - ** cost = (3.0 * N * log(N)) * (Y/X) + ** cost = (K * N * log(N)) * (Y/X) + ** + ** The constant K is 2.0 for an external sort that is built around + ** the OP_SorterInsert, OP_SorterSort, and OP_SorterData opcodes. + ** For a sort built using OP_IdxInsert and OP_Sort (which is slower + ** by a constant factor), the constant K is 4.0. ** ** The (Y/X) term is implemented using stack variable rScale ** below. @@ -4825,7 +4830,8 @@ static LogEst whereSortingCost( LogEst rScale, rSortCost; assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + rScale + 16; + rSortCost = nRow + rScale + 10; + if( pWInfo->wctrlFlags & WHERE_USE_LIMIT ) rSortCost += 10; /* Multiple by log(M) where M is the number of output rows. ** Use the LIMIT for M if it is smaller. Or if this sort is for @@ -4985,11 +4991,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo, nRowEst, nOrderBy, isOrdered ); } - /* TUNING: Add a small extra penalty (5) to sorting as an + /* TUNING: Add a small extra penalty (3) to sorting as an ** extra encouragment to the query planner to select a plan ** where the rows emerge in the correct order without any sorting ** required. */ - rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 5; + rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3; WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", diff --git a/test/orderby1.test b/test/orderby1.test index 9b5df4c753..7432b5473d 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -43,6 +43,7 @@ do_test 1.0 { (NULL, 1, 3, 'one-c'), (NULL, 2, 1, 'two-a'), (NULL, 3, 1, 'three-a'); + ANALYZE; COMMIT; } } {} @@ -180,6 +181,7 @@ do_test 2.0 { (1, 3, 'one-c'), (20, 1, 'two-a'), (3, 1, 'three-a'); + ANALYZE; COMMIT; } } {} @@ -327,6 +329,7 @@ do_test 3.0 { (NULL, 1, 3, 'one-c'), (NULL, 2, 1, 'two-a'), (NULL, 3, 1, 'three-a'); + ANALYZE; COMMIT; } } {} diff --git a/test/where.test b/test/where.test index e28861bc10..3f4aa8e1be 100644 --- a/test/where.test +++ b/test/where.test @@ -545,6 +545,7 @@ do_test where-6.1 { CREATE INDEX t3acb ON t3(a,c,b); INSERT INTO t3 SELECT w, 101-w, y FROM t1; SELECT count(*), sum(a), sum(b), sum(c) FROM t3; + ANALYZE; } } {100 5050 5050 348550} do_test where-6.2 { From a37fed0f6289faa31dad019785092c6e2f92a08b Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 01:59:03 +0000 Subject: [PATCH 133/282] Rework the oo1.DB's distinct-per-VFS post-open() step to accept either a batch of SQL or a callback function. Increase OPFS's busy timeout to 10s. FossilOrigin-Name: 9feefe253ac487cb52be6bdf91bdd305963266716baa08f2bf9505954ee76321 --- ext/wasm/api/sqlite3-api-oo1.js | 41 +++++++++++++++++-------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 39 ++++++++++++----------- ext/wasm/tests/opfs/concurrency/test.js | 2 +- manifest | 16 +++++----- manifest.uuid | 2 +- 5 files changed, 59 insertions(+), 41 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 45c2ba913b..e077b0c506 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -78,8 +78,10 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }.bind({counter: 0})); /** - A map of sqlite3_vfs pointers to SQL code to run when the DB - constructor opens a database with the given VFS. + A map of sqlite3_vfs pointers to SQL code or a callback function + to run when the DB constructor opens a database with the given + VFS. In the latter case, the call signature is (theDbObject,sqlite3Namespace) + and the callback is expected to throw on error. */ const __vfsPostOpenSql = Object.create(null); @@ -160,15 +162,6 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.sqlite3_trace_v2(pDb, capi.SQLITE_TRACE_STMT, __dbTraceToConsole, 0); } - // Check for per-VFS post-open SQL... - const pVfs = capi.sqlite3_js_db_vfs(pDb); - //console.warn("Opened db",fn,"with vfs",vfsName,pVfs); - if(!pVfs) toss3("Internal error: cannot get VFS for new db handle."); - const postInitSql = __vfsPostOpenSql[pVfs]; - if(postInitSql){ - rc = capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0); - checkSqlite3Rc(pDb, rc); - } }catch( e ){ if( pDb ) capi.sqlite3_close_v2(pDb); throw e; @@ -178,12 +171,34 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this.filename = fnJs; __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); + try{ + // Check for per-VFS post-open SQL/callback... + const pVfs = capi.sqlite3_js_db_vfs(pDb); + if(!pVfs) toss3("Internal error: cannot get VFS for new db handle."); + const postInitSql = __vfsPostOpenSql[pVfs]; + if(postInitSql instanceof Function){ + postInitSql(this, sqlite3); + }else if(postInitSql){ + checkSqlite3Rc( + pDb, capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0) + ); + } + }catch(e){ + this.close(); + throw e; + } }; /** Sets SQL which should be exec()'d on a DB instance after it is - opened with the given VFS pointer. This is intended only for use - by DB subclasses or sqlite3_vfs implementations. + opened with the given VFS pointer. The SQL may be any type + supported by the "flexible-string" function argument + conversion. Alternately, the 2nd argument may be a function, in + which case it is called with (theOo1DbObject,sqlite3Namespace) at + the end of the DB() constructor. The function must throw on + error, in which case the db is closed and the exception is + propagated. This function is intended only for use by DB + subclasses or sqlite3_vfs implementations. */ dbCtorHelper.setVfsPostOpenSql = function(pVfs, sql){ __vfsPostOpenSql[pVfs] = sql; diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 4dc145a614..f5a1eb6cc2 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -1163,24 +1163,27 @@ const installOpfsVfs = function callee(options){ OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, - [ - /* Truncate journal mode is faster than delete for - this vfs, per speedtest1. That gap seems to have closed with - Chrome version 108 or 109, but "persist" is very roughly 5-6% - faster than truncate in initial tests. */ - "pragma journal_mode=persist;", - /* Set a default busy-timeout handler to help OPFS dbs - deal with multi-tab/multi-worker contention. */ - "pragma busy_timeout=5000;", - /* - This vfs benefits hugely from cache on moderate/large - speedtest1 --size 50 and --size 100 workloads. We currently - rely on setting a non-default cache size when building - sqlite3.wasm. If that policy changes, the cache can - be set here. - */ - //"pragma cache_size=-16384;" - ].join("") + function(oo1Db, sqlite3){ + /* Set a relatively high default busy-timeout handler to + help OPFS dbs deal with multi-tab/multi-worker + contention. */ + sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); + sqlite3.capi.sqlite3_exec(oo1Db, [ + /* Truncate journal mode is faster than delete for + this vfs, per speedtest1. That gap seems to have closed with + Chrome version 108 or 109, but "persist" is very roughly 5-6% + faster than truncate in initial tests. */ + "pragma journal_mode=persist;", + /* + This vfs benefits hugely from cache on moderate/large + speedtest1 --size 50 and --size 100 workloads. We + currently rely on setting a non-default cache size when + building sqlite3.wasm. If that policy changes, the cache + can be set here. + */ + "pragma cache_size=-16384;" + ], 0, 0, 0); + } ); } diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index b994a8e79f..14cd6f514f 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -61,7 +61,7 @@ urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 ) || 1; options.interval = ( - urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 + urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 1000 ) || 1000; options.iterations = ( urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10 diff --git a/manifest b/manifest index 61b48477c1..809ca5f22f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\stweaks\sto\sthe\sOPFS\sVFS.\sResolve\sa\smissing\sresult\scode\swhich\slead\sto\sa\snull\sderef\sin\sxFileSize(). -D 2022-12-02T18:56:37.557 +C Rework\sthe\soo1.DB's\sdistinct-per-VFS\spost-open()\sstep\sto\saccept\seither\sa\sbatch\sof\sSQL\sor\sa\scallback\sfunction.\sIncrease\sOPFS's\sbusy\stimeout\sto\s10s. +D 2022-12-03T01:59:03.165 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -504,13 +504,13 @@ F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js b528207ba43f7740d1ade623f3f6b08a49f44ce7e9126915b78e1818c2466d8e -F ext/wasm/api/sqlite3-api-oo1.js c8b6c9ccb64cf93ca990ac689e98963735110aec21f98e04b55018f8e67b8147 +F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 F ext/wasm/api/sqlite3-api-prologue.js 42d6b316b542cf8e086f2f272460deb72dff184f1438a3377383cab99b08070b F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b8889f182ba7b5c1be8bcf6f47b538519e3fff0e98d7cf8b9d1fa4a9afdfb8ce +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 733bc939f93caef0df0b3ebfea14cbd528da580fdef1a35b1f69c2b3e044c7b7 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -557,7 +557,7 @@ F ext/wasm/tester1-worker.html ead6bdcc6cca221deb0dc9855a56f376351dbf2294fd7978c F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js e73a91eba4b59aaadd98f383c00a5101dbbbc52d937fff3162fc4761986f4a88 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 -F ext/wasm/tests/opfs/concurrency/test.js 9315339ed27849e65890eda924a516562936525a4f3f162fa71aeb489b9dc707 +F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 7ab655788bf0b52dce4538acbd5b11cdbe77edd36a14af5dec6dfe1ec4ab25fc @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d36021d107d3afca190ad61c3380536ad0cc2d493d345d48f9f9c1191741128 -R 6eba0a795b491027bc2604925d67afb7 +P 57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 +R c08d64a2fd44c82ec1d52c0b8202d402 U stephan -Z 379ecd060fe213536d8ac858396e84c5 +Z 36a132ccc79fdf9c5e77fe15b521856c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c043a6432..ec9f8c2463 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 \ No newline at end of file +9feefe253ac487cb52be6bdf91bdd305963266716baa08f2bf9505954ee76321 \ No newline at end of file From 1369d6be0268106576c2321bddfadc7c02d6d5f7 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 02:42:21 +0000 Subject: [PATCH 134/282] wasm snapshot and doc-related build automation tweaks. FossilOrigin-Name: 13eb1abd06f55fb88fc7f7be6149fd94b12057d9c38cc6b97bec3940e7e01f04 --- ext/wasm/GNUmakefile | 40 +++++++++++++++++++++------------------- ext/wasm/dist.make | 12 ++++++++++-- ext/wasm/index.html | 4 ++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 42 insertions(+), 32 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index d8ee9c758a..d2d68daa57 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -811,12 +811,6 @@ endif # /wasmfs ######################################################################## -######################################################################## -# Create main client downloadable zip file: -ifneq (,$(filter dist snapshot,$(MAKECMDGOALS))) -include dist.make -endif - ######################################################################## # Push files to public wasm-testing.sqlite.org server wasm-testing.include = *.js *.mjs *.html \ @@ -839,27 +833,35 @@ push-testing: ######################################################################## # If we find a copy of the sqlite.org/wasm docs checked out, copy # certain files over to it, noting that some need automatable edits... -WDOCS.home ?= ../../../wdoc +wasm.docs.home ?= ../../../wdoc +wasm.docs.found = $(if $(wildcard $(wasm.docs.home)/api-index.md),\ + $(wildcard $(wasm.docs.home)),) .PHONY: update-docs -ifneq (,$(wildcard $(WDOCS.home)/api-index.md)) -WDOCS.jswasm := $(WDOCS.home)/jswasm -update-docs: $(bin.stripccomments) $(sqlite3.js) $(sqlite3.wasm) - @echo "Copying files to the /wasm docs. Be sure to use an -Oz build for this!" - cp $(sqlite3.wasm) $(WDOCS.jswasm)/. - $(bin.stripccomments) -k -k < $(sqlite3.js) \ - | sed -e '/^[ \t]*$$/d' > $(WDOCS.jswasm)/sqlite3.js - cp demo-123.js demo-123.html demo-123-worker.html $(WDOCS.home) - sed -n -e '/EXTRACT_BEGIN/,/EXTRACT_END/p' \ - module-symbols.html > $(WDOCS.home)/module-symbols.html -else +ifeq (,$(wasm.docs.found)) update-docs: @echo "Cannot find wasm docs checkout."; \ - echo "Pass WDOCS.home=/path/to/wasm/docs/checkout or edit this makefile to suit."; \ + echo "Pass wasm.docs.home=/path/to/wasm/docs/checkout or edit this makefile to suit."; \ exit 127 +else +wasm.docs.jswasm := $(wasm.docs.home)/jswasm +update-docs: $(bin.stripccomments) $(sqlite3.js) $(sqlite3.wasm) + @echo "Copying files to the /wasm docs. Be sure to use an -Oz build for this!" + cp $(sqlite3.wasm) $(wasm.docs.jswasm)/. + $(bin.stripccomments) -k -k < $(sqlite3.js) \ + | sed -e '/^[ \t]*$$/d' > $(wasm.docs.jswasm)/sqlite3.js + cp demo-123.js demo-123.html demo-123-worker.html $(wasm.docs.home) + sed -n -e '/EXTRACT_BEGIN/,/EXTRACT_END/p' \ + module-symbols.html > $(wasm.docs.home)/module-symbols.html endif # end /wasm docs ######################################################################## +######################################################################## +# Create main client downloadable zip file: +ifneq (,$(filter dist snapshot,$(MAKECMDGOALS))) +include dist.make +endif + # Run local web server for the test/demo pages. httpd: althttpd -max-age 1 -enable-sab -page index.html diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index c8b9ee1fb3..e4dbad7c99 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -97,9 +97,17 @@ dist: \ ls -la $$arczip; \ set +e; \ unzip -lv $$arczip || echo "Missing unzip app? Not fatal." +ifeq (,$(wasm.docs.found)) snapshot: dist - @echo "Upload snapshot with:"; \ - echo "rsync -ve ssh $(dist-name-prefix)*.zip $(wasm-testing.dest)/snapshots/." + @echo "To upload the snapshot build to the wasm docs server:"; \ + echo "1) move $(dist-name-prefix)*.zip to the top of a wasm docs checkout."; \ + echo "2) run 'make uv-sync'" +else +snapshot: dist + @echo "Moving snapshot to [$(wasm.docs.found)]..."; \ + mv $(dist-name-prefix)*.zip $(wasm.docs.found)/. + @echo "Run 'make uv-sync' from $(wasm.docs.found) to upload it." +endif # We need a separate `clean` rule to account for weirdness in # a sub-make, where we get a copy of the $(dist-name) dir # copied into the new $(dist-name) dir. diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 33a115d2e9..a547ce50fa 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -88,9 +88,9 @@
  • speedtest1: a main-thread WASM build of speedtest1.
    • -
  • speedtest1-kvvfs: speedtest1 with the kvvfs.
    • +
  • speedtest1?vfs=kvvfs: speedtest1 with the kvvfs.
  • speedtest1-worker: an interactive Worker-thread variant of speedtest1.
    • -
  • speedtest1-worker-opfs: speedtest1-worker with the +
  • speedtest1-worker?vfs=opfs: speedtest1-worker with the OPFS VFS preselected and configured for a moderate workload.
    • diff --git a/manifest b/manifest index 809ca5f22f..dd28e4d280 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\sthe\soo1.DB's\sdistinct-per-VFS\spost-open()\sstep\sto\saccept\seither\sa\sbatch\sof\sSQL\sor\sa\scallback\sfunction.\sIncrease\sOPFS's\sbusy\stimeout\sto\s10s. -D 2022-12-03T01:59:03.165 +C wasm\ssnapshot\sand\sdoc-related\sbuild\sautomation\stweaks. +D 2022-12-03T02:42:21.560 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 54ab8da16a01e78bf5767c0e7bd57af07bfeb3a71fbecd63a39b3dbeec967c4e +F ext/wasm/GNUmakefile 6f64170c1c69a19ec76bb7f61bb69a4c009a3c3a6907617264140f69264a5b85 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -531,14 +531,14 @@ F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98 F ext/wasm/demo-worker1-promiser.js b85a2bb1b918db4f09dfa24419241cb3edad7791389425c2505092e9b715017d F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa -F ext/wasm/dist.make 994cc61822694b123d4357731072937a54153fbe5d9b12c6cb95d5562d2766dc +F ext/wasm/dist.make 701694188a78c9a24bf44cdf529063f4b3a0e892adc1d20ed1619252738943f1 F ext/wasm/fiddle.make 2812c44c9bafb5be9c8767963d1b9f374d77af7795fcaa06483c03e7059dea74 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js b4a0c8ab6c0983218543ca771c45f6075449f63a1dcf290ae5a681b2cba8800d F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 -F ext/wasm/index.html 618ad5cbc07b55556a4e09931773761698a38323387d392046bda907f1ce4c52 +F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e F ext/wasm/module-symbols.html b8eebafef8e536624bbe5f7a3da40c07a9062b843dfd3161a0bb72cbb6763dc5 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 -R c08d64a2fd44c82ec1d52c0b8202d402 +P 9feefe253ac487cb52be6bdf91bdd305963266716baa08f2bf9505954ee76321 +R 595dc0b8e35ed0115509bba594a7e5c6 U stephan -Z 36a132ccc79fdf9c5e77fe15b521856c +Z 192475a6131cf0ba0ca1840d5782ce18 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ec9f8c2463..09075c1212 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9feefe253ac487cb52be6bdf91bdd305963266716baa08f2bf9505954ee76321 \ No newline at end of file +13eb1abd06f55fb88fc7f7be6149fd94b12057d9c38cc6b97bec3940e7e01f04 \ No newline at end of file From 2564ca7d15328a8519248be22829702348253d8e Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 03:06:16 +0000 Subject: [PATCH 135/282] wasm build: rename the path to the wasm docs checkout, for clarity and consistency. FossilOrigin-Name: b820db32365b2ca8e2397fd6ea85883e4555ffd82948e248a0f98415b7328349 --- ext/wasm/GNUmakefile | 2 +- ext/wasm/module-symbols.html | 5 ++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index d2d68daa57..71c508f7c2 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -833,7 +833,7 @@ push-testing: ######################################################################## # If we find a copy of the sqlite.org/wasm docs checked out, copy # certain files over to it, noting that some need automatable edits... -wasm.docs.home ?= ../../../wdoc +wasm.docs.home ?= ../../../wasm wasm.docs.found = $(if $(wildcard $(wasm.docs.home)/api-index.md),\ $(wildcard $(wasm.docs.home)),) .PHONY: update-docs diff --git a/ext/wasm/module-symbols.html b/ext/wasm/module-symbols.html index 427d2dc8c3..1fbe2f3c17 100644 --- a/ext/wasm/module-symbols.html +++ b/ext/wasm/module-symbols.html @@ -92,9 +92,8 @@
  • This page runs in the main UI thread so cannot see features which are only available in a Worker thread. If this page were to function via a Worker, it would not be able to see - functionality only available in the main thread. Starting a - Worker here to fetch those symbols requires loading a second - copy of the sqlite3 WASM module and JS code. + functionality only available in the main thread. Either way, it + would be missing certain APIs.
    • diff --git a/manifest b/manifest index dd28e4d280..da738ab08b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm\ssnapshot\sand\sdoc-related\sbuild\sautomation\stweaks. -D 2022-12-03T02:42:21.560 +C wasm\sbuild:\srename\sthe\spath\sto\sthe\swasm\sdocs\scheckout,\sfor\sclarity\sand\sconsistency. +D 2022-12-03T03:06:16.365 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 6f64170c1c69a19ec76bb7f61bb69a4c009a3c3a6907617264140f69264a5b85 +F ext/wasm/GNUmakefile 004f3662abc5588d0e460893ad8f0fd94c970957159b6bba97087bac27d78007 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -541,7 +541,7 @@ F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486c F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e -F ext/wasm/module-symbols.html b8eebafef8e536624bbe5f7a3da40c07a9062b843dfd3161a0bb72cbb6763dc5 +F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 F ext/wasm/speedtest1-wasmfs.html bc28eb29b69a73864b8d7aae428448f8b7e1de81d8bfb9bba99541322054dbd0 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9feefe253ac487cb52be6bdf91bdd305963266716baa08f2bf9505954ee76321 -R 595dc0b8e35ed0115509bba594a7e5c6 +P 13eb1abd06f55fb88fc7f7be6149fd94b12057d9c38cc6b97bec3940e7e01f04 +R f5c92ee96049664c2a17edeac52c505f U stephan -Z 192475a6131cf0ba0ca1840d5782ce18 +Z a6b554242e7c212e38bd6fdb1ff7082c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 09075c1212..65b7f302f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13eb1abd06f55fb88fc7f7be6149fd94b12057d9c38cc6b97bec3940e7e01f04 \ No newline at end of file +b820db32365b2ca8e2397fd6ea85883e4555ffd82948e248a0f98415b7328349 \ No newline at end of file From 85ef4e434415ac8e674f4f92480a38cd7c3a135e Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 11:16:55 +0000 Subject: [PATCH 136/282] JavaScript: add sqlite3.wasm.realloc(), sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE, and related tests. FossilOrigin-Name: eeb84ba5de1152ef0f42105b8b285fdee9f5ad58281e60a4e0c8b1d6de1dead8 --- ext/wasm/GNUmakefile | 2 +- ext/wasm/api/sqlite3-api-glue.js | 1 + ext/wasm/api/sqlite3-api-prologue.js | 92 +++++++++++++++++++--------- ext/wasm/api/sqlite3-wasm.c | 10 +++ ext/wasm/tester1-worker.html | 2 +- ext/wasm/tester1.c-pp.js | 32 ++++++++++ manifest | 22 +++---- manifest.uuid | 2 +- 8 files changed, 121 insertions(+), 42 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 71c508f7c2..968d4f440c 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -489,7 +489,7 @@ emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) emcc.jsflags += $(emcc.environment) emcc.jsflags += -sSTACK_SIZE=1MB -# ^^^ ACHTUNG: emsdk 3.1.27 reduced the default stack size from 4MB to +# ^^^ ACHTUNG: emsdk 3.1.27 reduced the default stack size from 5MB to # a mere 64KB, which leads to silent memory corruption via the kvvfs # VFS, which requires twice that for its xRead() and xWrite() methods. ######################################################################## diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index e60baeb7f3..ac1e9fd6d6 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -605,6 +605,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); for(const t of ['access', 'blobFinalizers', 'dataTypes', 'encodings', 'fcntl', 'flock', 'ioCap', + 'limits', 'openFlags', 'prepareFlags', 'resultCodes', 'serialize', 'syncFlags', 'trace', 'udfFlags', 'version' diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index a99065663d..b6ff94dfe4 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -75,6 +75,10 @@ the `free(3)`-compatible routine for the WASM environment. Defaults to `"sqlite3_free"`. + - `reallocExportName`: the name of the function, in `exports`, of + the `realloc(3)`-compatible routine for the WASM + environment. Defaults to `"sqlite3_realloc"`. + - `wasmfsOpfsDir`[^1]: if the environment supports persistent storage using OPFS-over-WASMFS , this directory names the "mount point" for that directory. It must be prefixed by `/` and may @@ -110,6 +114,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( })(), allocExportName: 'sqlite3_malloc', deallocExportName: 'sqlite3_free', + reallocExportName: 'sqlite3_realloc', wasmfsOpfsDir: '/opfs' }, apiConfig || {}); @@ -284,12 +289,14 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( }; /** - Returns true if v appears to be one of our bind()-able - TypedArray types: Uint8Array or Int8Array. Support for - TypedArrays with element sizes >1 is TODO. + Returns true if v appears to be one of our bind()-able TypedArray + types: Uint8Array or Int8Array. Support for TypedArrays with + element sizes >1 is a potential TODO just waiting on a use case + to justify them. */ const isBindableTypedArray = (v)=>{ - return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + return v && (v instanceof Uint8Array || v instanceof Int8Array); + //v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); }; /** @@ -302,7 +309,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( isSQLableTypedArray() list. */ const isSQLableTypedArray = (v)=>{ - return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + return v && (v instanceof Uint8Array || v instanceof Int8Array); + //v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); }; /** Returns true if isBindableTypedArray(v) does, else throws with a message @@ -664,12 +672,12 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( "or config.memory (imported)."), /** - The API's one single point of access to the WASM-side memory - allocator. Works like malloc(3) (and is likely bound to - malloc()) but throws an WasmAllocError if allocation fails. It is - important that any code which might pass through the sqlite3 C - API NOT throw and must instead return SQLITE_NOMEM (or - equivalent, depending on the context). + The API's primary point of access to the WASM-side memory + allocator. Works like sqlite3_malloc() but throws a + WasmAllocError if allocation fails. It is important that any + code which might pass through the sqlite3 C API NOT throw and + must instead return SQLITE_NOMEM (or equivalent, depending on + the context). Very few cases in the sqlite3 JS APIs can result in client-defined functions propagating exceptions via the C-style @@ -681,7 +689,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( catch exceptions and convert them to appropriate error codes. For cases where non-throwing allocation is required, use - sqlite3.wasm.alloc.impl(), which is direct binding of the + this.alloc.impl(), which is direct binding of the underlying C-level allocator. Design note: this function is not named "malloc" primarily @@ -692,9 +700,27 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( alloc: undefined/*installed later*/, /** - The API's one single point of access to the WASM-side memory - deallocator. Works like free(3) (and is likely bound to - free()). + Rarely necessary in JS code, this routine works like + sqlite3_realloc(M,N), where M is either NULL or a pointer + obtained from this function or this.alloc() and N is the number + of bytes to reallocate the block to. Returns a pointer to the + reallocated block or 0 if allocation fails. + + If M is NULL and N is positive, this behaves like + this.alloc(N). If N is 0, it behaves like this.dealloc(). + Results are undefined if N is negative (sqlite3_realloc() + treats that as 0, but if this code is built with a different + allocator it may misbehave with negative values). + + Like this.alloc.impl(), this.realloc.impl() is a direct binding + to the underlying realloc() implementation which does not throw + exceptions, instead returning 0 on allocation error. + */ + realloc: undefined/*installed later*/, + + /** + The API's primary point of access to the WASM-side memory + deallocator. Works like sqlite3_free(). Design note: this function is not named "free" for the same reason that this.alloc() is not called this.malloc(). @@ -731,20 +757,30 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return pRet; }; - const keyAlloc = config.allocExportName, - keyDealloc = config.deallocExportName; - for(const key of [keyAlloc, keyDealloc]){ - const f = wasm.exports[key]; - if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); - } + { + // Set up allocators... + const keyAlloc = config.allocExportName, + keyDealloc = config.deallocExportName, + keyRealloc = config.reallocExportName; + for(const key of [keyAlloc, keyDealloc, keyRealloc]){ + const f = wasm.exports[key]; + if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); + } - wasm.alloc = function f(n){ - const m = f.impl(n); - if(!m) throw new WasmAllocError("Failed to allocate",n," bytes."); - return m; - }; - wasm.alloc.impl = wasm.exports[keyAlloc]; - wasm.dealloc = wasm.exports[keyDealloc]; + wasm.alloc = function f(n){ + const m = f.impl(n); + if(!m) throw new WasmAllocError("Failed to allocate",n," bytes."); + return m; + }; + wasm.alloc.impl = wasm.exports[keyAlloc]; + wasm.realloc = function f(m,n){ + const m2 = f.impl(m,n); + if(n && !m2) throw new WasmAllocError("Failed to reallocate",n," bytes."); + return n ? m2 : 0; + }; + wasm.realloc.impl = wasm.exports[keyRealloc]; + wasm.dealloc = wasm.exports[keyDealloc]; + } /** Reports info about compile-time options using diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 9acc8020e5..cbc4499760 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -112,6 +112,12 @@ # define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION #endif +/**********************************************************************/ +/* SQLITE_M... */ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 0x1fffffff +#endif + /**********************************************************************/ /* SQLITE_O... */ #ifndef SQLITE_OMIT_DEPRECATED @@ -497,6 +503,10 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_IOCAP_BATCH_ATOMIC); } _DefGroup; + DefGroup(limits) { + DefInt(SQLITE_MAX_ALLOCATION_SIZE); + } _DefGroup; + DefGroup(openFlags) { /* Noting that not all of these will have any effect in ** WASM-space. */ diff --git a/ext/wasm/tester1-worker.html b/ext/wasm/tester1-worker.html index ee03874df3..eba3fdeb4e 100644 --- a/ext/wasm/tester1-worker.html +++ b/ext/wasm/tester1-worker.html @@ -45,7 +45,7 @@ if(urlParams.has('esm')){ logHtml('warning',"Attempting to run an ES6 Worker Module, "+ "which is not supported by all browsers! "+ - "e.g. Firefox (as of 2022-11) cannot do this."); + "e.g. Firefox (as of 2022-12) cannot do this."); workerArgs.push("tester1.mjs",{type:"module"}); document.querySelectorAll('title,#color-target').forEach((e)=>{ e.innerText = "sqlite3 tester #1: ES6 Worker Module"; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 1db8499ef6..949d2365c4 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -424,6 +424,38 @@ self.sqlite3InitModule = sqlite3InitModule; } } + // alloc(), realloc(), allocFromTypedArray() + { + let m = w.alloc(14); + let m2 = w.realloc(m, 16); + T.assert(m === m2/* because of alignment */); + T.assert(0 === w.realloc(m, 0)); + m = m2 = 0; + + // Check allocation limits and allocator's responses... + T.assert('number' === typeof sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE); + const tooMuch = sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE + 1, + isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; + T.mustThrowMatching(()=>w.alloc(tooMuch), isAllocErr) + .assert(0 === w.alloc.impl(tooMuch)) + .mustThrowMatching(()=>w.realloc(0, tooMuch), isAllocErr) + .assert(0 === w.realloc.impl(0, tooMuch)); + + // Check allocFromTypedArray()... + const byteList = [11,22,33] + const u = new Uint8Array(byteList); + m = w.allocFromTypedArray(u); + for(let i = 0; i < u.length; ++i){ + T.assert(u[i] === byteList[i]) + .assert(u[i] === w.getMemValue(m + i, 'i8')); + } + w.dealloc(m); + T.mustThrowMatching( + ()=>w.allocFromTypedArray(1), + 'Value is not of a supported TypedArray type.' + ); + } + // isPtr32() { const ip = w.isPtr32; diff --git a/manifest b/manifest index da738ab08b..696930b4d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm\sbuild:\srename\sthe\spath\sto\sthe\swasm\sdocs\scheckout,\sfor\sclarity\sand\sconsistency. -D 2022-12-03T03:06:16.365 +C JavaScript:\sadd\ssqlite3.wasm.realloc(),\ssqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE,\sand\srelated\stests. +D 2022-12-03T11:16:55.292 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 004f3662abc5588d0e460893ad8f0fd94c970957159b6bba97087bac27d78007 +F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -503,16 +503,16 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js b528207ba43f7740d1ade623f3f6b08a49f44ce7e9126915b78e1818c2466d8e +F ext/wasm/api/sqlite3-api-glue.js 6fe39964605fda3b699f69365eed565b5172d29cab2c49bc057a43f9a93f9f36 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js 42d6b316b542cf8e086f2f272460deb72dff184f1438a3377383cab99b08070b +F ext/wasm/api/sqlite3-api-prologue.js 3d9550021269fd97636595ea2d2a1ec3ce00866f0a5d3b5fab94d4583afdafe0 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 733bc939f93caef0df0b3ebfea14cbd528da580fdef1a35b1f69c2b3e044c7b7 +F ext/wasm/api/sqlite3-wasm.c 69c2c1bf555dd25596137bf282d721657d5c49243061e0cb420375203107adcd F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -553,9 +553,9 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac -F ext/wasm/tester1-worker.html ead6bdcc6cca221deb0dc9855a56f376351dbf2294fd7978cd1609b3a56b245b +F ext/wasm/tester1-worker.html 29b1d87f7d51f70d61645719fee657f3787fe939bb695f27034c75404e8f1e6f F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js e73a91eba4b59aaadd98f383c00a5101dbbbc52d937fff3162fc4761986f4a88 +F ext/wasm/tester1.c-pp.js d25cea43933bf86590aab63038a6a0b6e7002ffba7e85d8df2720b7a69f85690 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 13eb1abd06f55fb88fc7f7be6149fd94b12057d9c38cc6b97bec3940e7e01f04 -R f5c92ee96049664c2a17edeac52c505f +P b820db32365b2ca8e2397fd6ea85883e4555ffd82948e248a0f98415b7328349 +R c074a7899efe7412c63b603c13447040 U stephan -Z a6b554242e7c212e38bd6fdb1ff7082c +Z 037a89bfec293517ee42832df5b904a5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 65b7f302f2..ba0da199e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b820db32365b2ca8e2397fd6ea85883e4555ffd82948e248a0f98415b7328349 \ No newline at end of file +eeb84ba5de1152ef0f42105b8b285fdee9f5ad58281e60a4e0c8b1d6de1dead8 \ No newline at end of file From fed255d794e80814d0a2f9bbd2083d03682245cb Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 11:51:29 +0000 Subject: [PATCH 137/282] Move kvvfs xRead()/xWrite() buffer from function-local stack memory to heap memory to accommodate changes in emsdk 3.1.27. FossilOrigin-Name: f1da32410ca7b808b3bef5f5a59766e7281e9e6ea343c8b979599bf1fc1060f5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_kv.c | 16 ++++++++++++---- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 696930b4d7..7147fdf43d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C JavaScript:\sadd\ssqlite3.wasm.realloc(),\ssqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE,\sand\srelated\stests. -D 2022-12-03T11:16:55.292 +C Move\skvvfs\sxRead()/xWrite()\sbuffer\sfrom\sfunction-local\sstack\smemory\sto\sheap\smemory\sto\saccommodate\schanges\sin\semsdk\s3.1.27. +D 2022-12-03T11:51:29.063 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -627,7 +627,7 @@ F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 F src/os.c 81c9c1c52eab711e27e33fd51fe5788488d3a02bc1a71439857abbee5d0d2c97 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 -F src/os_kv.c 0e59600d25b72034c7666b8b7dcc527f039b5d9c16f24a7eca4c08c66f63c364 +F src/os_kv.c 73f89ab97ecdb3216857d2acc8395103f89164eaadac87cce4e9e16445c89541 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 F src/os_unix.c 08191111a7040b8d5a6fff946f9fc9a11a0f83bac727c0415dfc5d030e1bc41f F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b820db32365b2ca8e2397fd6ea85883e4555ffd82948e248a0f98415b7328349 -R c074a7899efe7412c63b603c13447040 +P eeb84ba5de1152ef0f42105b8b285fdee9f5ad58281e60a4e0c8b1d6de1dead8 +R 3c59336372e40211f743691646d8f288 U stephan -Z 037a89bfec293517ee42832df5b904a5 +Z 413cf7b6bea840d13a8d33f3b8232f5f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ba0da199e4..d59f9b5e0d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eeb84ba5de1152ef0f42105b8b285fdee9f5ad58281e60a4e0c8b1d6de1dead8 \ No newline at end of file +f1da32410ca7b808b3bef5f5a59766e7281e9e6ea343c8b979599bf1fc1060f5 \ No newline at end of file diff --git a/src/os_kv.c b/src/os_kv.c index 322588be9a..45955d18f2 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -52,7 +52,9 @@ struct KVVfsFile { char *aJrnl; /* Journal content */ int szPage; /* Last known page size */ sqlite3_int64 szDb; /* Database file size. -1 means unknown */ + char *aData; /* Buffer to hold page data */ }; +#define SQLITE_KVOS_SZ 133073 /* ** Methods for KVVfsFile @@ -493,6 +495,7 @@ static int kvvfsClose(sqlite3_file *pProtoFile){ SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, pFile->isJournal ? "journal" : "db")); sqlite3_free(pFile->aJrnl); + sqlite3_free(pFile->aData); return SQLITE_OK; } @@ -541,7 +544,7 @@ static int kvvfsReadDb( unsigned int pgno; int got, n; char zKey[30]; - char aData[133073]; + char *aData = pFile->aData; assert( iOfst>=0 ); assert( iAmt>=0 ); SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); @@ -558,7 +561,8 @@ static int kvvfsReadDb( pgno = 1; } sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, aData, sizeof(aData)-1); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); if( got<0 ){ n = 0; }else{ @@ -566,7 +570,7 @@ static int kvvfsReadDb( if( iOfst+iAmt<512 ){ int k = iOfst+iAmt; aData[k*2] = 0; - n = kvvfsDecode(aData, &aData[2000], sizeof(aData)-2000); + n = kvvfsDecode(aData, &aData[2000], SQLITE_KVOS_SZ-2000); if( n>=iOfst+iAmt ){ memcpy(zBuf, &aData[2000+iOfst], iAmt); n = iAmt; @@ -625,7 +629,7 @@ static int kvvfsWriteDb( KVVfsFile *pFile = (KVVfsFile*)pProtoFile; unsigned int pgno; char zKey[30]; - char aData[131073]; + char *aData = pFile->aData; SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); assert( iAmt>=512 && iAmt<=65536 ); assert( (iAmt & (iAmt-1))==0 ); @@ -834,6 +838,10 @@ static int kvvfsOpen( }else{ pFile->zClass = "local"; } + pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); + if( pFile->aData==0 ){ + return SQLITE_NOMEM; + } pFile->aJrnl = 0; pFile->nJrnl = 0; pFile->szPage = -1; From 54ac04c8315f8ffc66991cea36d6daa580e8ad8b Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 13:05:33 +0000 Subject: [PATCH 138/282] Doc typo fix for SQLITE_MAX_ALLOCATION_SIZE in malloc.c. No code changes. FossilOrigin-Name: ed1ed21221b048ac5a5275cdfc4d9b2a406acdc7d4b648c3b61bcc822d88d955 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7147fdf43d..e9a01448a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\skvvfs\sxRead()/xWrite()\sbuffer\sfrom\sfunction-local\sstack\smemory\sto\sheap\smemory\sto\saccommodate\schanges\sin\semsdk\s3.1.27. -D 2022-12-03T11:51:29.063 +C Doc\stypo\sfix\sfor\sSQLITE_MAX_ALLOCATION_SIZE\sin\smalloc.c.\sNo\scode\schanges. +D 2022-12-03T13:05:33.513 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -609,7 +609,7 @@ F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 25663175950c5c4404b9377840b7b4c6fe5c53b415caf43634c62f442c02a9a7 F src/main.c 954490392b74fb215378af3c75a9e1f4f559f19cb1567e5d77f3fbbb63909b4d -F src/malloc.c 3d4ec162214024ee071d85711b93bec25cd3371280aee3702b63bcf312ca8238 +F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eeb84ba5de1152ef0f42105b8b285fdee9f5ad58281e60a4e0c8b1d6de1dead8 -R 3c59336372e40211f743691646d8f288 +P f1da32410ca7b808b3bef5f5a59766e7281e9e6ea343c8b979599bf1fc1060f5 +R ecd21c0eba1077a6366f5dd7664db4b8 U stephan -Z 413cf7b6bea840d13a8d33f3b8232f5f +Z d0c7ece3afa4de468292a6daac64615d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d59f9b5e0d..c125e07548 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1da32410ca7b808b3bef5f5a59766e7281e9e6ea343c8b979599bf1fc1060f5 \ No newline at end of file +ed1ed21221b048ac5a5275cdfc4d9b2a406acdc7d4b648c3b61bcc822d88d955 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 68d9f5f55c..48c4600606 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -279,7 +279,7 @@ static void mallocWithAlarm(int n, void **pp){ ** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 ** This provides a 256-byte safety margin for defense against 32-bit ** signed integer overflow bugs when computing memory allocation sizes. -** Parnoid applications might want to reduce the maximum allocation size +** Paranoid applications might want to reduce the maximum allocation size ** further for an even larger safety margin. 0x3fffffff or 0x0fffffff ** or even smaller would be reasonable upper bounds on the size of a memory ** allocations for most applications. From 09c27a59db621e621e32e4e90aa2b564dc84c574 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 13:10:58 +0000 Subject: [PATCH 139/282] Rename wasm.xWrap.resultAdapter() X:free entries to X:dealloc for consistency with wasm.dealloc(). Add an undocumented feature to replace wasm.alloc/dealloc/realloc() with the C-standard allocators (after an allocator misuse led down a several-hour rabbit hole trying to discover a mis-free() violation). Related test updates. FossilOrigin-Name: d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 1 + ext/wasm/api/sqlite3-api-prologue.js | 26 +++++++---- ext/wasm/api/sqlite3-wasm.c | 2 +- ext/wasm/common/whwasmutil.js | 18 +++---- ext/wasm/tester1.c-pp.js | 52 ++++++++++++++------- manifest | 20 ++++---- manifest.uuid | 2 +- 7 files changed, 73 insertions(+), 48 deletions(-) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index 1f7908e3b8..bbf16215ae 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -93,3 +93,4 @@ _sqlite3_vfs_register _sqlite3_vfs_unregister _malloc _free +_realloc diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index b6ff94dfe4..a0e978c952 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -112,12 +112,23 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( } return !!self.BigInt64Array; })(), - allocExportName: 'sqlite3_malloc', - deallocExportName: 'sqlite3_free', - reallocExportName: 'sqlite3_realloc', - wasmfsOpfsDir: '/opfs' + wasmfsOpfsDir: '/opfs', + /** + useStdAlloc is just for testing an allocator discrepancy. The + docs guarantee that this is false in the canonical builds. For + 99% of purposes it doesn't matter which allocators we use, but + it becomes significant with, e.g., sqlite3_deserialize() + and certain wasm.xWrap.resultAdapter()s. + */ + useStdAlloc: false }, apiConfig || {}); + Object.assign(config, { + allocExportName: config.useStdAlloc ? 'malloc' : 'sqlite3_malloc', + deallocExportName: config.useStdAlloc ? 'free' : 'sqlite3_free', + reallocExportName: config.useStdAlloc ? 'realloc' : 'sqlite3_realloc' + }, config); + [ // If any of these config options are functions, replace them with // the result of calling that function... @@ -768,15 +779,12 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( } wasm.alloc = function f(n){ - const m = f.impl(n); - if(!m) throw new WasmAllocError("Failed to allocate",n," bytes."); - return m; + return f.impl(n) || WasmAllocError.toss("Failed to allocate",n," bytes."); }; wasm.alloc.impl = wasm.exports[keyAlloc]; wasm.realloc = function f(m,n){ const m2 = f.impl(m,n); - if(n && !m2) throw new WasmAllocError("Failed to reallocate",n," bytes."); - return n ? m2 : 0; + return n ? (m2 || WasmAllocError.toss("Failed to reallocate",n," bytes.")) : 0; }; wasm.realloc.impl = wasm.exports[keyRealloc]; wasm.dealloc = wasm.exports[keyDealloc]; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index cbc4499760..f6499243a6 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1204,7 +1204,7 @@ void sqlite3_wasm_test_stack_overflow(int recurse){ /* For testing the 'string-free' whwasmutil.xWrap() conversion. */ SQLITE_WASM_KEEP char * sqlite3_wasm_test_str_hello(int fail){ - char * s = fail ? 0 : (char *)malloc(6); + char * s = fail ? 0 : (char *)sqlite3_malloc(6); if(s){ memcpy(s, "hello", 5); s[5] = 0; diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 0190433773..8cb4833247 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -1267,7 +1267,7 @@ self.WhWasmUtilInstaller = function(target){ - If v is a string, scopeAlloc() a new C-string from it and return that temp string's pointer. - - Else return the value from the arg adaptor defined for ptrIR. + - Else return the value from the arg adapter defined for ptrIR. TODO? Permit an Int8Array/Uint8Array and convert it to a string? Would that be too much magic concentrated in one place, ready to @@ -1279,12 +1279,12 @@ self.WhWasmUtilInstaller = function(target){ return v ? xcv.arg[ptrIR](v) : null; }; xcv.result.string = xcv.result.utf8 = (i)=>target.cstringToJs(i); - xcv.result['string:free'] = xcv.result['utf8:free'] = (i)=>{ + xcv.result['string:dealloc'] = xcv.result['utf8:dealloc'] = (i)=>{ try { return i ? target.cstringToJs(i) : null } finally{ target.dealloc(i) } }; xcv.result.json = (i)=>JSON.parse(target.cstringToJs(i)); - xcv.result['json:free'] = (i)=>{ + xcv.result['json:dealloc'] = (i)=>{ try{ return i ? JSON.parse(target.cstringToJs(i)) : null } finally{ target.dealloc(i) } } @@ -1383,7 +1383,7 @@ self.WhWasmUtilInstaller = function(target){ true. - `f32` (`float`), `f64` (`double`) (args and results): pass - their argument to Number(). i.e. the adaptor does not currently + their argument to Number(). i.e. the adapter does not currently distinguish between the two types of floating-point numbers. - `number` (results): converts the result to a JS Number using @@ -1411,7 +1411,7 @@ self.WhWasmUtilInstaller = function(target){ const C-string, encoded as UTF-8, copies it to a JS string, and returns that JS string. - - `string:free` or `utf8:free) (results): treats the result value + - `string:dealloc` or `utf8:dealloc) (results): treats the result value as a non-const UTF-8 C-string, ownership of which has just been transfered to the caller. It copies the C-string to a JS string, frees the C-string, and returns the JS string. If such @@ -1422,7 +1422,7 @@ self.WhWasmUtilInstaller = function(target){ required. For example: ```js - target.xWrap.resultAdaptor('string:my_free',(i)=>{ + target.xWrap.resultAdapter('string:my_free',(i)=>{ try { return i ? target.cstringToJs(i) : null } finally{ target.exports.my_free(i) } }; @@ -1432,9 +1432,9 @@ self.WhWasmUtilInstaller = function(target){ returns the result of passing the converted-to-JS string to JSON.parse(). Returns `null` if the C-string is a NULL pointer. - - `json:free` (results): works exactly like `string:free` but + - `json:dealloc` (results): works exactly like `string:dealloc` but returns the same thing as the `json` adapter. Note the - warning in `string:free` regarding maching allocators and + warning in `string:dealloc` regarding maching allocators and deallocators. The type names for results and arguments are validated when @@ -1545,7 +1545,7 @@ self.WhWasmUtilInstaller = function(target){ */ target.xWrap.resultAdapter = function f(typeName, adapter){ return __xAdapter(f, arguments.length, typeName, adapter, - 'resultAdaptor()', xcv.result); + 'resultAdapter()', xcv.result); }; /** diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 949d2365c4..2820555ffd 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -344,9 +344,20 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////// T.g('Basic sanity checks') - .t("JS wasm-side allocator === sqlite3_malloc()", function(sqlite3){ - T.assert(wasm.alloc.impl === wasm.exports.sqlite3_malloc) - .assert(wasm.dealloc === wasm.exports.sqlite3_free); + .t({ + name: "JS wasm-side allocator", + test: function(sqlite3){ + if(sqlite3.config.useStdAlloc){ + warn("Using system allocator. This violates the docs."); + T.assert(wasm.alloc.impl === wasm.exports.malloc) + .assert(wasm.dealloc === wasm.exports.free) + .assert(wasm.realloc.impl === wasm.exports.realloc); + }else{ + T.assert(wasm.alloc.impl === wasm.exports.sqlite3_malloc) + .assert(wasm.dealloc === wasm.exports.sqlite3_free) + .assert(wasm.realloc.impl === wasm.exports.sqlite3_realloc); + } + } }) .t('Namespace object checks', function(sqlite3){ const wasmCtypes = wasm.ctype; @@ -434,12 +445,14 @@ self.sqlite3InitModule = sqlite3InitModule; // Check allocation limits and allocator's responses... T.assert('number' === typeof sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE); - const tooMuch = sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE + 1, - isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; - T.mustThrowMatching(()=>w.alloc(tooMuch), isAllocErr) - .assert(0 === w.alloc.impl(tooMuch)) - .mustThrowMatching(()=>w.realloc(0, tooMuch), isAllocErr) - .assert(0 === w.realloc.impl(0, tooMuch)); + if(!sqlite3.config.useStdAlloc){ + const tooMuch = sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE + 1, + isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; + T.mustThrowMatching(()=>w.alloc(tooMuch), isAllocErr) + .assert(0 === w.alloc.impl(tooMuch)) + .mustThrowMatching(()=>w.realloc(0, tooMuch), isAllocErr) + .assert(0 === w.realloc.impl(0, tooMuch)); + } // Check allocFromTypedArray()... const byteList = [11,22,33] @@ -553,11 +566,12 @@ self.sqlite3InitModule = sqlite3InitModule; //log("allocCString()..."); { - const cstr = w.allocCString("hällo, world"); - const n = w.cstrlen(cstr); - T.assert(13 === n) + const jstr = "hällo, world!"; + const [cstr, n] = w.allocCString(jstr, true); + T.assert(14 === n) .assert(0===w.getMemValue(cstr+n)) - .assert(chr('d')===w.getMemValue(cstr+n-1)); + .assert(chr('!')===w.getMemValue(cstr+n-1)); + w.dealloc(cstr); } //log("scopedAlloc() and friends..."); @@ -639,11 +653,13 @@ self.sqlite3InitModule = sqlite3InitModule; rc = w.xCallWrapped('sqlite3_wasm_enum_json','utf8'); T.assert('string'===typeof rc).assert(rc.length>300); if(haveWasmCTests()){ - fw = w.xWrap('sqlite3_wasm_test_str_hello', 'utf8:free',['i32']); - rc = fw(0); - T.assert('hello'===rc); - rc = fw(1); - T.assert(null===rc); + if(!sqlite3.config.useStdAlloc){ + fw = w.xWrap('sqlite3_wasm_test_str_hello', 'utf8:dealloc',['i32']); + rc = fw(0); + T.assert('hello'===rc); + rc = fw(1); + T.assert(null===rc); + } if(w.bigIntEnabled){ w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); diff --git a/manifest b/manifest index e9a01448a6..ee12cfe5ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Doc\stypo\sfix\sfor\sSQLITE_MAX_ALLOCATION_SIZE\sin\smalloc.c.\sNo\scode\schanges. -D 2022-12-03T13:05:33.513 +C Rename\swasm.xWrap.resultAdapter()\sX:free\sentries\sto\sX:dealloc\sfor\sconsistency\swith\swasm.dealloc().\sAdd\san\sundocumented\sfeature\sto\sreplace\swasm.alloc/dealloc/realloc()\swith\sthe\sC-standard\sallocators\s(after\san\sallocator\smisuse\sled\sdown\sa\sseveral-hour\srabbit\shole\strying\sto\sdiscover\sa\smis-free()\sviolation).\sRelated\stest\supdates. +D 2022-12-03T13:10:58.483 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api eeb1300945c3871e46904e026a04a41d6fc458fcaa74ccee4e719ebb8ccb3087 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d @@ -505,14 +505,14 @@ F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a9578430388 F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js 6fe39964605fda3b699f69365eed565b5172d29cab2c49bc057a43f9a93f9f36 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js 3d9550021269fd97636595ea2d2a1ec3ce00866f0a5d3b5fab94d4583afdafe0 +F ext/wasm/api/sqlite3-api-prologue.js b06eb09246deb56612b4905eb2ec4bcc8b10f0472fa661cd54c5d6cd1888d8b9 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 69c2c1bf555dd25596137bf282d721657d5c49243061e0cb420375203107adcd +F ext/wasm/api/sqlite3-wasm.c b0babf8435f31d21f28454fb81433aa538c68b23d0a4a251f0666fdec4e71f59 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -521,7 +521,7 @@ F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c07 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/common/testing.css 35889709547d89a6109ff83b25c11bbc91d8dd43aab8722e428655ca98880a06 -F ext/wasm/common/whwasmutil.js dbe625a22bf0962cde1f958f2be604d27d2f97ee1b4e6ee0f19c6480aa36aeed +F ext/wasm/common/whwasmutil.js c1bc5715cd96728929cc31d788b16152ccbd6b2e111d2e88fbc9725247e67b4f F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 29b1d87f7d51f70d61645719fee657f3787fe939bb695f27034c75404e8f1e6f F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js d25cea43933bf86590aab63038a6a0b6e7002ffba7e85d8df2720b7a69f85690 +F ext/wasm/tester1.c-pp.js 956acfd5675e61fa7c52a19e5b16603da61e753e309abe91284e3b1d39598840 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f1da32410ca7b808b3bef5f5a59766e7281e9e6ea343c8b979599bf1fc1060f5 -R ecd21c0eba1077a6366f5dd7664db4b8 +P ed1ed21221b048ac5a5275cdfc4d9b2a406acdc7d4b648c3b61bcc822d88d955 +R 04802c5e85209b72659bc288df2e94a1 U stephan -Z d0c7ece3afa4de468292a6daac64615d +Z 23bb6930a31350fa896c81241f2934b6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c125e07548..4e29a56c7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed1ed21221b048ac5a5275cdfc4d9b2a406acdc7d4b648c3b61bcc822d88d955 \ No newline at end of file +d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 \ No newline at end of file From cbcc08ddc4cf5a1c6b21d5e1febed6ed64962939 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 14:58:45 +0000 Subject: [PATCH 140/282] sqlite3.wasm.allocFromTypedArray() now optionally accepts an ArrayBuffer as its argument. FossilOrigin-Name: 75a1a796f86d289c7275666fab19013934775dcccaed44a1a61d1749a6cb99c9 --- ext/wasm/api/sqlite3-api-prologue.js | 10 ++++++---- ext/wasm/tester1.c-pp.js | 17 +++++++++++------ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index a0e978c952..e1376f23bb 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -748,18 +748,20 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( returned pointer must eventually be passed to wasm.dealloc() to clean it up. + The argument may be a Uint8Array, Int8Array, or ArrayBuffer, + and it throws if passed any other type. + As a special case, to avoid further special cases where this is used, if srcTypedArray.byteLength is 0, it allocates a single byte and sets it to the value 0. Even in such cases, calls must behave as if the allocated memory has exactly srcTypedArray.byteLength bytes. - - ACHTUNG: this currently only works for Uint8Array and - Int8Array types and will throw if srcTypedArray is of - any other type. */ wasm.allocFromTypedArray = function(srcTypedArray){ + if(srcTypedArray instanceof ArrayBuffer){ + srcTypedArray = new Uint8Array(srcTypedArray); + } affirmBindableTypedArray(srcTypedArray); const pRet = wasm.alloc(srcTypedArray.byteLength || 1); wasm.heapForSize(srcTypedArray.constructor).set( diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 2820555ffd..f019b591e4 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -348,7 +348,9 @@ self.sqlite3InitModule = sqlite3InitModule; name: "JS wasm-side allocator", test: function(sqlite3){ if(sqlite3.config.useStdAlloc){ - warn("Using system allocator. This violates the docs."); + warn("Using system allocator. This violates the docs and", + "may cause grief with certain APIs", + "(e.g. sqlite3_deserialize())."); T.assert(wasm.alloc.impl === wasm.exports.malloc) .assert(wasm.dealloc === wasm.exports.free) .assert(wasm.realloc.impl === wasm.exports.realloc); @@ -463,6 +465,11 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(u[i] === w.getMemValue(m + i, 'i8')); } w.dealloc(m); + m = w.allocFromTypedArray(u.buffer); + for(let i = 0; i < u.length; ++i){ + T.assert(u[i] === byteList[i]) + .assert(u[i] === w.getMemValue(m + i, 'i8')); + } T.mustThrowMatching( ()=>w.allocFromTypedArray(1), 'Value is not of a supported TypedArray type.' @@ -1463,7 +1470,7 @@ self.sqlite3InitModule = sqlite3InitModule; const n = db.selectValue(sql); T.assert(n>0 && db2.selectValue(sql) === n); }finally{ - if(db2) db2.close(); + db2.close(); wasm.sqlite3_wasm_vfs_unlink(pVfs, filename); } } @@ -1502,8 +1509,7 @@ self.sqlite3InitModule = sqlite3InitModule; assert(T.eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). assert(T.eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))); - let blobArg = new Uint8Array(2); - blobArg.set([0x68, 0x69], 0); + let blobArg = new Uint8Array([0x68, 0x69]); let blobRc = db.selectValue("select asis(?1)", blobArg); T.assert(blobRc instanceof Uint8Array). assert(2 === blobRc.length). @@ -1513,8 +1519,7 @@ self.sqlite3InitModule = sqlite3InitModule; assert(2 === blobRc.length). assert(0x68==blobRc[0] && 0x69==blobRc[1]); - blobArg = new Int8Array(2); - blobArg.set([0x68, 0x69]); + blobArg = new Int8Array([0x68, 0x69]); //debug("blobArg=",blobArg); blobRc = db.selectValue("select asis(?1)", blobArg); T.assert(blobRc instanceof Uint8Array). diff --git a/manifest b/manifest index ee12cfe5ad..4bf54d1ec3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\swasm.xWrap.resultAdapter()\sX:free\sentries\sto\sX:dealloc\sfor\sconsistency\swith\swasm.dealloc().\sAdd\san\sundocumented\sfeature\sto\sreplace\swasm.alloc/dealloc/realloc()\swith\sthe\sC-standard\sallocators\s(after\san\sallocator\smisuse\sled\sdown\sa\sseveral-hour\srabbit\shole\strying\sto\sdiscover\sa\smis-free()\sviolation).\sRelated\stest\supdates. -D 2022-12-03T13:10:58.483 +C sqlite3.wasm.allocFromTypedArray()\snow\soptionally\saccepts\san\sArrayBuffer\sas\sits\sargument. +D 2022-12-03T14:58:45.066 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,7 +505,7 @@ F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a9578430388 F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js 6fe39964605fda3b699f69365eed565b5172d29cab2c49bc057a43f9a93f9f36 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js b06eb09246deb56612b4905eb2ec4bcc8b10f0472fa661cd54c5d6cd1888d8b9 +F ext/wasm/api/sqlite3-api-prologue.js 6dc183bdfdc213dc9f3f8f58089bd895525e399f1826e7d3f32d28b6abc582e8 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 29b1d87f7d51f70d61645719fee657f3787fe939bb695f27034c75404e8f1e6f F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js 956acfd5675e61fa7c52a19e5b16603da61e753e309abe91284e3b1d39598840 +F ext/wasm/tester1.c-pp.js b33f50216ad78a7d5a88967b8613fc2bc76b57fda2fcbceb58d2d739a315c2c5 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ed1ed21221b048ac5a5275cdfc4d9b2a406acdc7d4b648c3b61bcc822d88d955 -R 04802c5e85209b72659bc288df2e94a1 +P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 +R 36bd64116645d9b16e5ee1bfbfa56d1c U stephan -Z 23bb6930a31350fa896c81241f2934b6 +Z 62ef618676c9b9a1f0d769e57183adb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e29a56c7f..53576ea413 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 \ No newline at end of file +75a1a796f86d289c7275666fab19013934775dcccaed44a1a61d1749a6cb99c9 \ No newline at end of file From 9baafdabf77c8354c72dfe506ef7774573fa2c37 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 15:41:29 +0000 Subject: [PATCH 141/282] Correct a memory leak in tester1.js. FossilOrigin-Name: e42b052dd754e577b8429e2430821c5f88096170ffa58b4e5e3cb7ee19b6c37d --- ext/wasm/tester1.c-pp.js | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index f019b591e4..0aaee97693 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -470,6 +470,7 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert(u[i] === byteList[i]) .assert(u[i] === w.getMemValue(m + i, 'i8')); } + w.dealloc(m); T.mustThrowMatching( ()=>w.allocFromTypedArray(1), 'Value is not of a supported TypedArray type.' diff --git a/manifest b/manifest index 4bf54d1ec3..005404d5bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C sqlite3.wasm.allocFromTypedArray()\snow\soptionally\saccepts\san\sArrayBuffer\sas\sits\sargument. -D 2022-12-03T14:58:45.066 +C Correct\sa\smemory\sleak\sin\stester1.js. +D 2022-12-03T15:41:29.086 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 29b1d87f7d51f70d61645719fee657f3787fe939bb695f27034c75404e8f1e6f F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js b33f50216ad78a7d5a88967b8613fc2bc76b57fda2fcbceb58d2d739a315c2c5 +F ext/wasm/tester1.c-pp.js d096a8fadfd27caa680a4311b1d529551f8fe885a63dd27457c87b6008c64632 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 -R 36bd64116645d9b16e5ee1bfbfa56d1c +P 75a1a796f86d289c7275666fab19013934775dcccaed44a1a61d1749a6cb99c9 +R 7a48580039cdf0dcac94f303fc1d293a U stephan -Z 62ef618676c9b9a1f0d769e57183adb7 +Z a4c0098dd4ed05913f5ba3742d524e04 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 53576ea413..364d2c6b7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75a1a796f86d289c7275666fab19013934775dcccaed44a1a61d1749a6cb99c9 \ No newline at end of file +e42b052dd754e577b8429e2430821c5f88096170ffa58b4e5e3cb7ee19b6c37d \ No newline at end of file From 30cbcc0004fa7c272f35214d8ab022e30d2e7075 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 3 Dec 2022 16:09:32 +0000 Subject: [PATCH 142/282] Cause CLI .sha3sum to warn of text fields that do not survive CAST(CAST(t as BLOB) AS TEXT) due to invalid UTF encoding. FossilOrigin-Name: 123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/shell.c.in | 51 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ee12cfe5ad..34ffa10ecc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\swasm.xWrap.resultAdapter()\sX:free\sentries\sto\sX:dealloc\sfor\sconsistency\swith\swasm.dealloc().\sAdd\san\sundocumented\sfeature\sto\sreplace\swasm.alloc/dealloc/realloc()\swith\sthe\sC-standard\sallocators\s(after\san\sallocator\smisuse\sled\sdown\sa\sseveral-hour\srabbit\shole\strying\sto\sdiscover\sa\smis-free()\sviolation).\sRelated\stest\supdates. -D 2022-12-03T13:10:58.483 +C Cause\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sdo\snot\ssurvive\sCAST(CAST(t\sas\sBLOB)\sAS\sTEXT)\sdue\sto\sinvalid\sUTF\sencoding. +D 2022-12-03T16:09:32.554 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 -F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c +F src/shell.c.in aa3e4c78ae25c24562b0c3497a88875baa61425714437143170b2c462842035d F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2065,8 +2065,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ed1ed21221b048ac5a5275cdfc4d9b2a406acdc7d4b648c3b61bcc822d88d955 -R 04802c5e85209b72659bc288df2e94a1 -U stephan -Z 23bb6930a31350fa896c81241f2934b6 +P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 +R 9f83c1fb14dddfc272778480afabb0a3 +T *branch * sha3sum_text_validation +T *sym-sha3sum_text_validation * +T -sym-trunk * +U larrybr +Z 1ba97ab620561a8dccaf913734ec9094 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e29a56c7f..b35a727cc3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 \ No newline at end of file +123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a829fa2387..1f0ac24c81 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10077,6 +10077,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_stmt *pStmt; /* For querying tables names */ char *zSql; /* SQL to be run */ char *zSep; /* Separator */ + char *zRevText; /* Query for reversible to-blob-to-text check */ ShellText sSql; /* Complete SQL for the query to run the hash */ ShellText sQuery; /* Set of queries used to read all content */ open_db(p, 0); @@ -10114,17 +10115,35 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( bSchema ){ - zSql = "SELECT lower(name) FROM sqlite_schema" + zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" " UNION ALL SELECT 'sqlite_schema'" " ORDER BY 1 collate nocase"; }else{ - zSql = "SELECT lower(name) FROM sqlite_schema" + zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" " AND name NOT LIKE 'sqlite_%'" " ORDER BY 1 collate nocase"; } sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + zRevText = sqlite3_mprintf( + /* lower-case query is first run, producing upper-case query. */ + "with text_cols as materialized(" + "select tname, cname " + "from (" + " select ss.tname as tname, ti.name as cname, lower(type) as lca" + " from (%s) ss join pragma_table_info(tname) ti" + " where (instr(lca,'int')=0 and " + " instr(lca,'char')+instr(lca,'clob')+instr(lca,'text')>0)" + "))" + "select 'SELECT total(bad_text_count) AS bad_text_count" + " FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query" + " from (select 'SELECT COUNT(*) AS bad_text_count" + " FROM '||tname||' WHERE '" + " ||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname," + " ' OR ') as query, tname from text_cols group by tname)", + zSql); + shell_check_oom(zRevText); initText(&sQuery); initText(&sSql); appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); @@ -10180,7 +10199,35 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ shell_exec(p, zSql, 0); } + { + int lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); + if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); + assert(lrc==SQLITE_OK); + lrc = SQLITE_ROW==sqlite3_step(pStmt); + assert(lrc!=0); + if( lrc ){ + const char *zGenQuery = sqlite3_column_text(pStmt,0); + sqlite3_stmt *pCheckStmt; + double countIrreversible; + lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); + if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); + assert(lrc==SQLITE_OK); + lrc = SQLITE_ROW==sqlite3_step(pCheckStmt); + assert(lrc!=0); + countIrreversible = sqlite3_column_double(pCheckStmt, 0); + sqlite3_finalize(pStmt); + sqlite3_finalize(pCheckStmt); + if( countIrreversible>0 ){ + int n = (int)(countIrreversible + 0.5); + utf8_printf(stderr, + "Digest includes %d invalidly encoded text field%s.\n", + n, (n>1)? "s": ""); + } + } + + } sqlite3_free(zSql); + sqlite3_free(zRevText); }else #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) From e4fa4794be9a938ae28bf4208eac272956691574 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 Dec 2022 17:09:15 +0000 Subject: [PATCH 143/282] Further improvements to the estimated cost of sorting. Take into account the number of columns to be sorted. FossilOrigin-Name: f3290cf83b7c02d17d85d8942954f052b486c370cd5ec732969da9061dc1d19a --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/where.c | 49 +++++++++++++++++++++++++++++++------------------ 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index a763dc6801..dc964b2b92 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tuning\sthe\squery\splanner\sby\sadjusting\sthe\sweights\sthat\spredict\sthe\srelative\nperformance\sof\ssorting\sand\sindex\slookup. -D 2022-12-03T00:52:21.776 +C Further\simprovements\sto\sthe\sestimated\scost\sof\ssorting.\s\sTake\sinto\saccount\nthe\snumber\sof\scolumns\sto\sbe\ssorted. +D 2022-12-03T17:09:15.127 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 5826b62ddcfc92979669cb5fb80f73d0df86bbfeefa1d757f5dc1f857cd628e7 +F src/where.c 32875f4f738b1b32f648e8fd52df23a5ba28744a2b4209b262cac09efc2a8569 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c ee52c2781c36004d23c85bf111063b78fc16e5e1b6a0d424326af8bf90babb0b F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2065,11 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 -R e47495a3cfe52da04033a615c7da71c2 -T *branch * qp-tuning -T *sym-qp-tuning * -T -sym-trunk * +P 9f2806da4d88beceac2e81e05421f00481dd3dd100b096cd2ae6c828adb42ca7 +R 7460b69c2cad54c6cd480cae6fbd9d9a U drh -Z bdcd0fcd9740989c0fc73b883156ab70 +Z b965371360d6f8e63aa93b345d42429e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d25f3c5d8c..b086c5dd4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f2806da4d88beceac2e81e05421f00481dd3dd100b096cd2ae6c828adb42ca7 \ No newline at end of file +f3290cf83b7c02d17d85d8942954f052b486c370cd5ec732969da9061dc1d19a \ No newline at end of file diff --git a/src/where.c b/src/where.c index f1d1df44c5..a9bb092ef2 100644 --- a/src/where.c +++ b/src/where.c @@ -4803,12 +4803,12 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** order. */ static LogEst whereSortingCost( - WhereInfo *pWInfo, - LogEst nRow, - int nOrderBy, - int nSorted + WhereInfo *pWInfo, /* Query planning context */ + LogEst nRow, /* Estimated number of rows to sort */ + int nOrderBy, /* Number of ORDER BY clause terms */ + int nSorted /* Number of initial ORDER BY terms naturally in order */ ){ - /* TUNING: Estimated cost of a full external sort, where N is + /* Estimated cost of a full external sort, where N is ** the number of rows to sort is: ** ** cost = (K * N * log(N)). @@ -4819,27 +4819,40 @@ static LogEst whereSortingCost( ** ** cost = (K * N * log(N)) * (Y/X) ** - ** The constant K is 2.0 for an external sort that is built around - ** the OP_SorterInsert, OP_SorterSort, and OP_SorterData opcodes. - ** For a sort built using OP_IdxInsert and OP_Sort (which is slower - ** by a constant factor), the constant K is 4.0. + ** The constant K is at least 2.0 but will be larger if there are a + ** large number of columns to be sorted, as the sorting time is + ** proportional to the amount of content to be sorted. The algorithm + ** does not currently distinguish between fat columns (BLOBs and TEXTs) + ** and skinny columns (INTs). It just uses the number of columns as + ** an approximation for the row width. ** - ** The (Y/X) term is implemented using stack variable rScale - ** below. + ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort + ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert. */ - LogEst rScale, rSortCost; - assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); - rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + rScale + 10; - if( pWInfo->wctrlFlags & WHERE_USE_LIMIT ) rSortCost += 10; + LogEst rSortCost, nCol; + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->pEList!=0 ); + /* TUNING: sorting cost proportional to the number of output columns: */ + nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30); + rSortCost = nRow + nCol; + if( nSorted>0 ){ + /* Scale the result by (Y/X) */ + rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + } /* Multiple by log(M) where M is the number of output rows. ** Use the LIMIT for M if it is smaller. Or if this sort is for ** a DISTINCT operator, M will be the number of distinct output ** rows, so fudge it downwards a bit. */ - if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimitiLimit; + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){ + rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */ + if( nSorted!=0 ){ + rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */ + } + if( pWInfo->iLimitiLimit; + } }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){ /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT ** reduces the number of output rows by a factor of 2 */ From fcf4243c36f87b989409ceb22da542fdf7375dcc Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 Dec 2022 17:23:29 +0000 Subject: [PATCH 144/282] Add a test case to show that ticket [e8b674241947eb3b] has been fixed. FossilOrigin-Name: e20de6d450c2b4d5bde737f625de16ff53262c22ce7aa6917b64f1665170d33f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/sort.test | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index dc964b2b92..f0872a157d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\simprovements\sto\sthe\sestimated\scost\sof\ssorting.\s\sTake\sinto\saccount\nthe\snumber\sof\scolumns\sto\sbe\ssorted. -D 2022-12-03T17:09:15.127 +C Add\sa\stest\scase\sto\sshow\sthat\sticket\s[e8b674241947eb3b]\shas\sbeen\sfixed. +D 2022-12-03T17:23:29.589 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1525,7 +1525,7 @@ F test/snapshot_fault.test f6c5ef7cb93bf92fbb4e864ecc5c87df7d3a250064838822db5b4 F test/snapshot_up.test a0a29c4cf33475fcef07c3f8e64af795e24ab91b4cc68295863402a393cdd41c F test/soak.test 18944cf21b94a7fe0df02016a6ee1e9632bc4e8d095a0cb49d95e15d5cca2d5c F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 -F test/sort.test c2adc635c2564241fefec0b3a68391ef6868fd3b +F test/sort.test f86751134159abb5e5fd4381a0d7038c91013638cd1e3fa1d7850901f6df6196 F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0 F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5 F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f2806da4d88beceac2e81e05421f00481dd3dd100b096cd2ae6c828adb42ca7 -R 7460b69c2cad54c6cd480cae6fbd9d9a +P f3290cf83b7c02d17d85d8942954f052b486c370cd5ec732969da9061dc1d19a +R e8ab3ed0037b8a828ad83ab76a453ef0 U drh -Z b965371360d6f8e63aa93b345d42429e +Z 8bdcb4a69164adf0860f9006c8d2a11c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b086c5dd4e..86db9897dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f3290cf83b7c02d17d85d8942954f052b486c370cd5ec732969da9061dc1d19a \ No newline at end of file +e20de6d450c2b4d5bde737f625de16ff53262c22ce7aa6917b64f1665170d33f \ No newline at end of file diff --git a/test/sort.test b/test/sort.test index d73ecea480..e6da6c6baf 100644 --- a/test/sort.test +++ b/test/sort.test @@ -595,4 +595,36 @@ do_execsql_test 17.1 { SELECT * FROM sqlite_master ORDER BY sql; } {} +# 2022-12-03 Ticket e8b674241947eb3b +# Improve estimates for the cost of sorting relative +# to the cost of doing an index lookup, so as to get +# a better query plan. See the ticket for a deetailed +# example. +# +reset_db +do_execsql_test 18.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<50) + -- increase to 5000 for actual test data ----^^ + INSERT INTO t1(a,b,c) SELECT x, random()%5000, random()%5000 FROM c; + CREATE TABLE t2(d,e,f); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<500) + -- increase to 50000 for actual test data -----^^^ + INSERT INTO t2(d,e,f) SELECT + NULLIF(0, random()%2), random()%5000, random()%5000 + FROM c; + ANALYZE; + UPDATE sqlite_stat1 SET stat='50000' WHERE tbl='t2'; + UPDATE sqlite_stat1 SET stat='5000' WHERE tbl='t1'; + ANALYZE sqlite_schema; +} {} +do_execsql_test 18.2 { + EXPLAIN QUERY PLAN + SELECT a FROM t1 JOIN t2 + WHERE a IN (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) + AND a=CASE WHEN d IS NOT NULL THEN e ELSE f END + ORDER BY a; +} {/.*SCAN t2.*SEARCH t1.*/} +# ^^^^^^^--^^^^^^^^^--- t2 should be the outer loop. + finish_test From f6f01f15dd9ff495ac59f5abc3bbe15f4d8e88c8 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Dec 2022 18:16:25 +0000 Subject: [PATCH 145/282] Add CYCLES scanstat measurement to "USE TEMP B-TREE FOR ORDER BY" lines. FossilOrigin-Name: 365011ae8b7e3fcaa2c4ea9601231a3ef2223e60d7a53ec33013109dca22ad58 --- manifest | 27 ++++++++++++--------------- manifest.uuid | 2 +- src/expr.c | 5 ++--- src/select.c | 27 ++++++++++++++++++++++++--- src/vdbe.h | 4 ++-- src/vdbeInt.h | 10 +++++++++- src/vdbeapi.c | 12 +++++++++--- src/vdbeaux.c | 16 ++++++++++++++-- test/scanstatus2.test | 6 ++++++ 9 files changed, 79 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index deb137d25e..520ead0acf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\ssqlite3_stmt_scanstatus()\sAPI\sand\sadd\ssqlite3_stmt_scanstatus_v2().\sFor\screation\sof\seasier\sto\sread\squery\sperformance\sreports. -D 2022-12-02T20:32:22.142 +C Add\sCYCLES\sscanstat\smeasurement\sto\s"USE\sTEMP\sB-TREE\sFOR\sORDER\sBY"\slines. +D 2022-12-03T18:16:25.876 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -595,7 +595,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c a56a7ad1163a9888d46cd5820be2e65354fb1aa04ed6909f7c3e5831e0ee2c29 F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 7b44a97e14ebb679c14ced7c3db420fd6293c33f8bd668fef37746e76afdc746 +F src/expr.c 02a24db96257d0d0dfc82401a6720ae3156cd20b60b84db9523aaf87884a3254 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -645,7 +645,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c b1301741aed7014beafb4cbc524d725865a890c7619d784e4ffb04379259ae6c +F src/select.c f6674cdcb839f118cf931b0c49e94d5fd37007a616239f62882d02df51cacbe7 F src/shell.c.in 0e45a91da5426563c5b0c8f66fd51d5a5fcb3cc17faa44cd4ea8cc386bde146b F src/sqlite.h.in 3ba99a3c2b414b9ad36d06e2f273ab6077f5f74e4d79630aee047394b92f7c7f F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -718,10 +718,10 @@ F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 313f3154e2b85a447326f5dd15de8d31a4df6ab0c3579bd58f426ff634ec9050 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c 5f5cc749c999d65e2115bb3ec555f7ac0ecfbda7f47526272d53a032e9d083b1 -F src/vdbe.h 247b4ffaad9ba2b2944a2b3084936ae5df684a42151552f32f074ec847c1402b -F src/vdbeInt.h 9628718a289cf3bf490db1435019773ccc0874e1a505502ea477c5a98e649128 -F src/vdbeapi.c 592c3d7be7c23d202cf5bfb96fc970ced7aaaa1b51052868f1deb91bbbb0add8 -F src/vdbeaux.c 600edaa2d20eb28aea9707148f4edcf12b349418fd2e3f1de92e5d23fdfc0eb8 +F src/vdbe.h 77b3fb0abe5c5edaca849816584f0fd019c0f91e80410ef2d65f57fcf290ddd8 +F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 +F src/vdbeapi.c fed69b846866f1c74a17b8217497f379d1815256220006c072867737c43a741c +F src/vdbeaux.c 7dcaff5933e4fbe5983be435b7cd2951d5f3ebd0f2acaf70fed4271dbfb57981 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -1457,7 +1457,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 7dbcfd6adc6a8df6abc59f83d6da5a27e1bce0b2f6fa55147c8176d7c44e0450 -F test/scanstatus2.test c618c5843257e290647dd63672c16b0f6488faebc64ba9b2ec5b4de3bb39a7cc +F test/scanstatus2.test 7c4fb011e8018b638a39acfa8a93599b991ad0e56872df4556abf375d9663e95 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2066,11 +2066,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 57dd593ef0efa17dfb3a9f4eac36d5b8b879e271de817d8cd94a8c8b56d31870 -R 49b81422f59a6a7338c393b5956057f1 -T *branch * scanstatus_v2 -T *sym-scanstatus_v2 * -T -sym-trunk * +P 55800833645739efeddcacef464c623931cb6aeb43f4219b4e4faf473c25c8bb +R 3092b93947ce4f84b005f0cef9aa6957 U dan -Z e87234282222fcab550cd1e86b10e65b +Z 8d82637b2296891fd9a0f7cdb6b7a23f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 23a6edc914..eec563c182 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55800833645739efeddcacef464c623931cb6aeb43f4219b4e4faf473c25c8bb \ No newline at end of file +365011ae8b7e3fcaa2c4ea9601231a3ef2223e60d7a53ec33013109dca22ad58 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 07e44a1876..1528877e23 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3258,7 +3258,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ int nReg; /* Registers to allocate */ Expr *pLimit; /* New limit expression */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - int addrExplain; + int addrExplain; /* Address of OP_Explain instruction */ #endif Vdbe *v = pParse->pVdbe; @@ -3356,8 +3356,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ if( addrOnce ){ sqlite3VdbeJumpHere(v, addrOnce); } - sqlite3VdbeScanStatusEnd(v, addrExplain); - + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); /* Subroutine return */ assert( ExprUseYSub(pExpr) ); diff --git a/src/select.c b/src/select.c index 7403725252..7d010c6244 100644 --- a/src/select.c +++ b/src/select.c @@ -65,6 +65,10 @@ struct SortCtx { } aDefer[4]; #endif struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrPushStart; /* First instruction to push data into sorter */ + int addrPushEnd; /* Last instruction that pushes data into sorter */ +#endif }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -721,6 +725,10 @@ static void pushOntoSorter( */ assert( nData==1 || regData==regOrigData || regOrigData==0 ); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPushStart = sqlite3VdbeCurrentAddr(v); +#endif + if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nPrefixReg; @@ -821,6 +829,9 @@ static void pushOntoSorter( sqlite3VdbeChangeP2(v, iSkip, pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPushEnd = sqlite3VdbeCurrentAddr(v)-1; +#endif } /* @@ -1647,6 +1658,16 @@ static void generateSortTail( int bSeq; /* True if sorter record includes seq. no. */ int nRefKey = 0; struct ExprList_item *aOutEx = p->pEList->a; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ +#endif + + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") + ); + sqlite3VdbeScanStatusRange( + v, addrExplain, pSort->addrPushStart, pSort->addrPushEnd + ); assert( addrBreak<0 ); if( pSort->labelBkOut ){ @@ -1759,6 +1780,7 @@ static void generateSortTail( VdbeComment((v, "%s", aOutEx[i].zEName)); } } + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -1820,6 +1842,7 @@ static void generateSortTail( }else{ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + sqlite3VdbeScanStatusRange(v, addrExplain, sqlite3VdbeCurrentAddr(v)-1, -1); if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); } @@ -7305,7 +7328,7 @@ int sqlite3Select( if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); - sqlite3VdbeScanStatusEnd(v, addrExplain); + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ @@ -8060,8 +8083,6 @@ int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, - sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); assert( p->pEList==pEList ); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } diff --git a/src/vdbe.h b/src/vdbe.h index 16690f0855..8d0c8123f3 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -387,10 +387,10 @@ int sqlite3VdbeBytecodeVtabInit(sqlite3*); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); -void sqlite3VdbeScanStatusEnd(Vdbe*, int); +void sqlite3VdbeScanStatusRange(Vdbe*, int, int, int); #else # define sqlite3VdbeScanStatus(a,b,c,d,e,f) -# define sqlite3VdbeScanStatusEnd(a,b) +# define sqlite3VdbeScanStatusRange(a,b,c,d) #endif #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) diff --git a/src/vdbeInt.h b/src/vdbeInt.h index b8642da508..df47e05448 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -388,11 +388,19 @@ typedef unsigned bft; /* Bit Field Type */ /* The ScanStatus object holds a single value for the ** sqlite3_stmt_scanstatus() interface. +** +** aAddrRange[]: +** This array is used by ScanStatus elements associated with EQP +** notes that make an SQLITE_SCANSTAT_NCYCLE value available. It is +** an array of up to 3 ranges of VM addresses for which the Vdbe.anCycle[] +** values should be summed to calculate the NCYCLE value. Each pair of +** integer addresses is a start and end address (both inclusive) for a range +** instructions. A start value of 0 indicates an empty range. */ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ - int addrEndRange; /* End of range (inclusive) of times to sum */ + int aAddrRange[6]; int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ int iSelectID; /* The "Select-ID" for this loop */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 90b0cc60c8..c00980d6b2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2207,11 +2207,17 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_NCYCLE: { i64 res = -1; - if( pScan->addrEndRange ){ + if( pScan->aAddrRange[0] ){ int ii; res = 0; - for(ii=pScan->addrExplain; ii<=pScan->addrEndRange; ii++){ - res += p->anCycle[ii]; + for(ii=0; iiaAddrRange); ii+=2){ + int iIns = pScan->aAddrRange[ii]; + int iEnd = pScan->aAddrRange[ii+1]; + if( iIns==0 ) break; + while( iIns<=iEnd ){ + res += p->anCycle[iIns]; + iIns++; + } } } *(i64*)pOut = res; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e7cd61f05d..74d3866c48 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1128,7 +1128,12 @@ void sqlite3VdbeScanStatus( } } -void sqlite3VdbeScanStatusEnd(Vdbe *p, int addrExplain){ +void sqlite3VdbeScanStatusRange( + Vdbe *p, + int addrExplain, + int addrStart, + int addrEnd +){ ScanStatus *pScan = 0; int ii; for(ii=p->nScan-1; ii>=0; ii--){ @@ -1137,7 +1142,14 @@ void sqlite3VdbeScanStatusEnd(Vdbe *p, int addrExplain){ pScan = 0; } if( pScan ){ - pScan->addrEndRange = sqlite3VdbeCurrentAddr(p)-1; + if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; + for(ii=0; iiaAddrRange); ii+=2){ + if( pScan->aAddrRange[ii]==0 ){ + pScan->aAddrRange[ii] = addrStart; + pScan->aAddrRange[ii+1] = addrEnd; + break; + } + } } } #endif diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 001797df2e..e3a8daaab5 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -94,6 +94,12 @@ proc do_graph_test {tn sql res} { uplevel [list do_test $tn [list set {} $graph] [string trim $res]] } +proc puts_graph {sql} { + db eval $sql + set stmt [db version -last-stmt-ptr] + puts [string trim [get_graph $stmt]] +} + do_zexplain_test 0 1.1 { SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 From 878b44c1283bd1433d36e675d34a19711b5f1d13 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 Dec 2022 18:21:30 +0000 Subject: [PATCH 146/282] Add TOTAL rows to the output generated by tool/vdbe_profile.tcl. FossilOrigin-Name: 9800586393c9d3b82459ef657620d245a7985ef5fa389b8a9ea633d6a29c7299 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/vdbe_profile.tcl | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f0872a157d..6d338b47c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sto\sshow\sthat\sticket\s[e8b674241947eb3b]\shas\sbeen\sfixed. -D 2022-12-03T17:23:29.589 +C Add\sTOTAL\srows\sto\sthe\soutput\sgenerated\sby\stool/vdbe_profile.tcl. +D 2022-12-03T18:21:30.267 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2040,7 +2040,7 @@ F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh 1612bd947750e21e7b47befad5f6b3825b06cce0705441f903bf35ced65ae9b9 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 1dcb7632e57cf57105248029e6e162fddaf6c0fccb3bb9e6215603752c5a2d4a -F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f +F tool/vdbe_profile.tcl 3ac5a4a9449f4baf77059358ea050db3e34395ccf59c5464d29b91746d5b961e F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d58dc38367cc776550f90327e205d7946802d4004fb9f291fd8b81256bc1eedd F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f3290cf83b7c02d17d85d8942954f052b486c370cd5ec732969da9061dc1d19a -R e8ab3ed0037b8a828ad83ab76a453ef0 +P e20de6d450c2b4d5bde737f625de16ff53262c22ce7aa6917b64f1665170d33f +R aefabaf5ca745982d6ec40be787f17b5 U drh -Z 8bdcb4a69164adf0860f9006c8d2a11c +Z 6a37ae50ecfc47b777551be2941b67ad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 86db9897dd..207858a35b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e20de6d450c2b4d5bde737f625de16ff53262c22ce7aa6917b64f1665170d33f \ No newline at end of file +9800586393c9d3b82459ef657620d245a7985ef5fa389b8a9ea633d6a29c7299 \ No newline at end of file diff --git a/tool/vdbe_profile.tcl b/tool/vdbe_profile.tcl index a0dc99ec33..b7240e3567 100644 --- a/tool/vdbe_profile.tcl +++ b/tool/vdbe_profile.tcl @@ -66,6 +66,8 @@ foreach stmt $allstmt { puts "********************************************************************" puts [string trim $sql($stmt)] puts "Execution count: $cnt($stmt)" + set tcx 0 + set ttx 0 for {set i 0} {[info exists stat($stmt,$i)]} {incr i} { foreach {cx tx detail} $stat($stmt,$i) break if {$cx==0} { @@ -74,7 +76,11 @@ foreach stmt $allstmt { set ax [expr {$tx/$cx}] } puts [format {%8d %12d %12d %4d %s} $cx $tx $ax $i $detail] + incr tcx $cx + incr ttx $tx } + set tax [expr {$tcx>0?$ttx/$tcx:0}] + puts [format {%8d %12d %12d TOTAL} $tcx $ttx $tax] } puts "********************************************************************" puts "OPCODES:" From 1d8f4e6ce8c86ac83b4331362ba2e8c2e2014164 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 Dec 2022 19:04:09 +0000 Subject: [PATCH 147/282] Increase the nominal row size for IPK index lookups slightly, for better balance. FossilOrigin-Name: 1a61c500add4a2bfe80c0c691d559cfca166dc5f8262651a58da7ec16a51d430 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6d338b47c4..5751a22ada 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sTOTAL\srows\sto\sthe\soutput\sgenerated\sby\stool/vdbe_profile.tcl. -D 2022-12-03T18:21:30.267 +C Increase\sthe\snominal\srow\ssize\sfor\sIPK\sindex\slookups\sslightly,\sfor\sbetter\nbalance. +D 2022-12-03T19:04:09.723 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 32875f4f738b1b32f648e8fd52df23a5ba28744a2b4209b262cac09efc2a8569 +F src/where.c cf893bd9e48cc4f761beb490e2016cfec7791b0778808a84f2d3d6340085f0d5 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c ee52c2781c36004d23c85bf111063b78fc16e5e1b6a0d424326af8bf90babb0b F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e20de6d450c2b4d5bde737f625de16ff53262c22ce7aa6917b64f1665170d33f -R aefabaf5ca745982d6ec40be787f17b5 +P 9800586393c9d3b82459ef657620d245a7985ef5fa389b8a9ea633d6a29c7299 +R bdb8fcd5ad681651f9b1330561cafc6c U drh -Z 6a37ae50ecfc47b777551be2941b67ad +Z f5c8d079aea1d9c7b44238c2c41ad282 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 207858a35b..f3e63033d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9800586393c9d3b82459ef657620d245a7985ef5fa389b8a9ea633d6a29c7299 \ No newline at end of file +1a61c500add4a2bfe80c0c691d559cfca166dc5f8262651a58da7ec16a51d430 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a9bb092ef2..34a6f348d9 100644 --- a/src/where.c +++ b/src/where.c @@ -3472,7 +3472,7 @@ static int whereLoopAddBtree( sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; - sPk.szIdxRow = 1; /* Interior rows of IPK table are very small */ + sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */ sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; From ad23a47acdc3258e988c91eac48eb8ce7a79fd53 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Dec 2022 21:24:26 +0000 Subject: [PATCH 148/282] Enhance SQLITE_SCANSTAT_NCYCLE so that it reports on virtual tables. FossilOrigin-Name: 622d8eb3724bee617b55d6fb71f1a2d683db6858065adced6bf3ce9525bcd6b5 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/shell.c.in | 2 +- src/vdbeapi.c | 28 ++++++++++++++++++++++------ src/wherecode.c | 3 +++ test/scanstatus2.test | 17 +++++++++++++++++ 6 files changed, 53 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 520ead0acf..7ca1c989db 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sCYCLES\sscanstat\smeasurement\sto\s"USE\sTEMP\sB-TREE\sFOR\sORDER\sBY"\slines. -D 2022-12-03T18:16:25.876 +C Enhance\sSQLITE_SCANSTAT_NCYCLE\sso\sthat\sit\sreports\son\svirtual\stables. +D 2022-12-03T21:24:26.253 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f6674cdcb839f118cf931b0c49e94d5fd37007a616239f62882d02df51cacbe7 -F src/shell.c.in 0e45a91da5426563c5b0c8f66fd51d5a5fcb3cc17faa44cd4ea8cc386bde146b +F src/shell.c.in 8cfe0c7dbd3ac8f9eca40fdad85d90f22362a89ab9ad8d64826fd0ad5e408d13 F src/sqlite.h.in 3ba99a3c2b414b9ad36d06e2f273ab6077f5f74e4d79630aee047394b92f7c7f F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -720,7 +720,7 @@ F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c 5f5cc749c999d65e2115bb3ec555f7ac0ecfbda7f47526272d53a032e9d083b1 F src/vdbe.h 77b3fb0abe5c5edaca849816584f0fd019c0f91e80410ef2d65f57fcf290ddd8 F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 -F src/vdbeapi.c fed69b846866f1c74a17b8217497f379d1815256220006c072867737c43a741c +F src/vdbeapi.c 35b4ca406f9a4bfd23fb7016b6d36280a5c1d68372b69271d6fbfcceead4b8ab F src/vdbeaux.c 7dcaff5933e4fbe5983be435b7cd2951d5f3ebd0f2acaf70fed4271dbfb57981 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a @@ -734,7 +734,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c bf470b5d1ba03af8d558a0c98cc1fa97b330e03a198a7af61895e5a2e8d93f20 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c f7121f4c29e45ba7d38c91448e250ae5197a4b75e9fa75996ba6d5c81a3cc6e5 +F src/wherecode.c d3146e215f3a716c0e153ded68bfcc747aad550e114a79729bda887cf4ea6f00 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1457,7 +1457,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 7dbcfd6adc6a8df6abc59f83d6da5a27e1bce0b2f6fa55147c8176d7c44e0450 -F test/scanstatus2.test 7c4fb011e8018b638a39acfa8a93599b991ad0e56872df4556abf375d9663e95 +F test/scanstatus2.test 799129ea953ba1235d4d5e7f59375cdff79f9185142773ed5d9e1c2c14500833 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2066,8 +2066,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 55800833645739efeddcacef464c623931cb6aeb43f4219b4e4faf473c25c8bb -R 3092b93947ce4f84b005f0cef9aa6957 +P 365011ae8b7e3fcaa2c4ea9601231a3ef2223e60d7a53ec33013109dca22ad58 +R d4e70f090a6234401994809de206bf5b U dan -Z 8d82637b2296891fd9a0f7cdb6b7a23f +Z 77f61cda4725a7748ad22a19d7f4eb4e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index eec563c182..b1c968e110 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -365011ae8b7e3fcaa2c4ea9601231a3ef2223e60d7a53ec33013109dca22ad58 \ No newline at end of file +622d8eb3724bee617b55d6fb71f1a2d683db6858065adced6bf3ce9525bcd6b5 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 7cbd6b3c47..d3ef84e44f 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3054,7 +3054,7 @@ static void display_scanstats( zText = sqlite3_mprintf("%s", z); if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ char *z = 0; - if( nCycle>=0 ){ + if( nCycle>=0 && nTotal>0 ){ z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, nCycle, ((nCycle*100)+nTotal/2) / nTotal ); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index c00980d6b2..2b9b1087f2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2206,17 +2206,33 @@ int sqlite3_stmt_scanstatus_v2( break; } case SQLITE_SCANSTAT_NCYCLE: { - i64 res = -1; - if( pScan->aAddrRange[0] ){ + i64 res = 0; + if( pScan->aAddrRange[0]==0 ){ + res = -1; + }else{ int ii; - res = 0; for(ii=0; iiaAddrRange); ii+=2){ int iIns = pScan->aAddrRange[ii]; int iEnd = pScan->aAddrRange[ii+1]; if( iIns==0 ) break; - while( iIns<=iEnd ){ - res += p->anCycle[iIns]; - iIns++; + if( iIns>0 ){ + while( iIns<=iEnd ){ + res += p->anCycle[iIns]; + iIns++; + } + }else{ + int iOp; + for(iOp=0; iOpnOp; iOp++){ + Op *pOp = &p->aOp[iOp]; + if( pOp->p1!=iEnd ) continue; + if( pOp->opcode!=OP_VFilter && pOp->opcode!=OP_VColumn + && pOp->opcode!=OP_Rowid && pOp->opcode!=OP_VOpen + && pOp->opcode!=OP_VNext + ){ + continue; + } + res += p->anCycle[iOp]; + } } } } diff --git a/src/wherecode.c b/src/wherecode.c index b744556fa8..54c79baf97 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -302,6 +302,9 @@ void sqlite3WhereAddScanStatus( sqlite3VdbeScanStatus( v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj ); + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } } #endif diff --git a/test/scanstatus2.test b/test/scanstatus2.test index e3a8daaab5..a7dfabc4ac 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -138,6 +138,23 @@ QUERY (nCycle=nnn) --USE TEMP B-TREE FOR ORDER BY } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a); + INSERT INTO ft VALUES('abc'); + INSERT INTO ft VALUES('def'); + INSERT INTO ft VALUES('ghi'); +} + +explain_i { + SELECT * FROM ft('def') + } +do_graph_test 2.1 { + SELECT * FROM ft('def') +} { +} + finish_test From a81c5b3a88558859abe593c8e539920d55ae68eb Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 4 Dec 2022 08:16:33 +0000 Subject: [PATCH 149/282] Export sqlite3_result_zeroblob/zeroblob64() to wasm. FossilOrigin-Name: a60e56627fc0ef8831429941d429ee02c6ee51ce5a2c1af581dc5bc5a00d911e --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 2 ++ ext/wasm/api/sqlite3-api-prologue.js | 20 +++++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index bbf16215ae..096bf44018 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -66,6 +66,8 @@ _sqlite3_result_int _sqlite3_result_int64 _sqlite3_result_null _sqlite3_result_text +_sqlite3_result_zeroblob +_sqlite3_result_zeroblob64 _sqlite3_serialize _sqlite3_shutdown _sqlite3_sourceid diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index e1376f23bb..5ebe7af058 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -945,15 +945,16 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( the range of supported argument types. */ ["sqlite3_realloc", "*","*","int"], ["sqlite3_reset", "int", "sqlite3_stmt*"], - ["sqlite3_result_blob",undefined, "*", "*", "int", "*"], - ["sqlite3_result_double",undefined, "*", "f64"], - ["sqlite3_result_error",undefined, "*", "string", "int"], - ["sqlite3_result_error_code", undefined, "*", "int"], - ["sqlite3_result_error_nomem", undefined, "*"], - ["sqlite3_result_error_toobig", undefined, "*"], - ["sqlite3_result_int",undefined, "*", "int"], - ["sqlite3_result_null",undefined, "*"], - ["sqlite3_result_text",undefined, "*", "string", "int", "*"], + ["sqlite3_result_blob",undefined, "sqlite3_context*", "*", "int", "*"], + ["sqlite3_result_double",undefined, "sqlite3_context*", "f64"], + ["sqlite3_result_error",undefined, "sqlite3_context*", "string", "int"], + ["sqlite3_result_error_code", undefined, "sqlite3_context*", "int"], + ["sqlite3_result_error_nomem", undefined, "sqlite3_context*"], + ["sqlite3_result_error_toobig", undefined, "sqlite3_context*"], + ["sqlite3_result_int",undefined, "sqlite3_context*", "int"], + ["sqlite3_result_null",undefined, "sqlite3_context*"], + ["sqlite3_result_text",undefined, "sqlite3_context*", "string", "int", "*"], + ["sqlite3_result_zeroblob", undefined, "sqlite3_context*", "int"], ["sqlite3_serialize","*", "sqlite3*", "string", "*", "int"], ["sqlite3_shutdown", undefined], ["sqlite3_sourceid", "string"], @@ -1001,6 +1002,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_msize", "i64", "*"], ["sqlite3_realloc64", "*","*", "i64"], ["sqlite3_result_int64",undefined, "*", "i64"], + ["sqlite3_result_zeroblob64", "int", "*", "i64"], ["sqlite3_total_changes64", "i64", ["sqlite3*"]], ["sqlite3_uri_int64", "i64", ["sqlite3_filename", "string", "i64"]], ["sqlite3_value_int64","i64", "sqlite3_value*"], diff --git a/manifest b/manifest index 005404d5bc..d5c9041c3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\sa\smemory\sleak\sin\stester1.js. -D 2022-12-03T15:41:29.086 +C Export\ssqlite3_result_zeroblob/zeroblob64()\sto\swasm. +D 2022-12-04T08:16:33.208 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api eeb1300945c3871e46904e026a04a41d6fc458fcaa74ccee4e719ebb8ccb3087 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 89af0612bad5c651f69e629c7e9689be6d3c8a92a9010da5dd90a87c1d86817a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d @@ -505,7 +505,7 @@ F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a9578430388 F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js 6fe39964605fda3b699f69365eed565b5172d29cab2c49bc057a43f9a93f9f36 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js 6dc183bdfdc213dc9f3f8f58089bd895525e399f1826e7d3f32d28b6abc582e8 +F ext/wasm/api/sqlite3-api-prologue.js 697a5989ad52a9ba7bc60b5436589bd05885ee2201d84c38c5e9af3876af3ba4 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75a1a796f86d289c7275666fab19013934775dcccaed44a1a61d1749a6cb99c9 -R 7a48580039cdf0dcac94f303fc1d293a +P e42b052dd754e577b8429e2430821c5f88096170ffa58b4e5e3cb7ee19b6c37d +R f9f2ae474faa20c31be8ebf34d534350 U stephan -Z a4c0098dd4ed05913f5ba3742d524e04 +Z b8f6ec21a3c321177d9e95fcdc34f445 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 364d2c6b7e..9fb00490d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e42b052dd754e577b8429e2430821c5f88096170ffa58b4e5e3cb7ee19b6c37d \ No newline at end of file +a60e56627fc0ef8831429941d429ee02c6ee51ce5a2c1af581dc5bc5a00d911e \ No newline at end of file From 8af6d712daa725fcdfcf63fcbcfd017633c41e10 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sun, 4 Dec 2022 23:20:38 +0000 Subject: [PATCH 150/282] Fix safe mode authorizer callback to reject disallowed UDFs. Reported at [forum:/forumpost/07beac8056151b2f|Forum post 07beac8056151b2f]. FossilOrigin-Name: cefc032473ac5ad244c0b6402c541b2f76c0c65a041bda03bfbe7c0e2c11fac2 --- manifest | 17 +++++++++-------- manifest.uuid | 2 +- src/shell.c.in | 4 ++-- test/shell2.test | 12 ++++++++++++ 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d5c9041c3e..d765436257 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Export\ssqlite3_result_zeroblob/zeroblob64()\sto\swasm. -D 2022-12-04T08:16:33.208 +C Fix\ssafe\smode\sauthorizer\scallback\sto\sreject\sdisallowed\sUDFs.\sReported\sat\s[forum:/forumpost/07beac8056151b2f|Forum\spost\s07beac8056151b2f]. +D 2022-12-04T23:20:38.999 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 -F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c +F src/shell.c.in f6ab148f150dc0c8460be74a61566d37c65d43311e84963cc1a58df3fc277511 F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -1500,7 +1500,7 @@ F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test e4b4de56f454708e0747b52915135baa2cbfec4965406d6eaf02a4a5c22a9880 -F test/shell2.test c536c2aab4852608f8a606262330797abc4d964a4c2c782a7760f54ea1f17a6a +F test/shell2.test 1190b951373fdfe719bc6ac16962bc743dfa4355db8ae546c0bb9bf559a28d4a F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b @@ -2065,8 +2065,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e42b052dd754e577b8429e2430821c5f88096170ffa58b4e5e3cb7ee19b6c37d -R f9f2ae474faa20c31be8ebf34d534350 -U stephan -Z b8f6ec21a3c321177d9e95fcdc34f445 +P a60e56627fc0ef8831429941d429ee02c6ee51ce5a2c1af581dc5bc5a00d911e +Q +4d934f00634fa31827c0bf2503a5011117c1fe9fd2f41c2a4bf954fb416d9b0f +R 7ec03feb65d26dfd70421ee745ff5ed3 +U larrybr +Z e9f5a072853fed1de334998b75e8a6f0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9fb00490d6..e69f11ed61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a60e56627fc0ef8831429941d429ee02c6ee51ce5a2c1af581dc5bc5a00d911e \ No newline at end of file +cefc032473ac5ad244c0b6402c541b2f76c0c65a041bda03bfbe7c0e2c11fac2 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a829fa2387..0f8108e05f 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1880,7 +1880,7 @@ static int safeModeAuth( "zipfile", "zipfile_cds", }; - UNUSED_PARAMETER(zA2); + UNUSED_PARAMETER(zA1); UNUSED_PARAMETER(zA3); UNUSED_PARAMETER(zA4); switch( op ){ @@ -1895,7 +1895,7 @@ static int safeModeAuth( case SQLITE_FUNCTION: { int i; for(i=0; i Date: Mon, 5 Dec 2022 02:42:30 +0000 Subject: [PATCH 151/282] Use the smaller estimated row size for searching IPK tables, but use the original larger row size estimate for scanning, since the leaves can have large rows. FossilOrigin-Name: df3818997b822743ac407dde45c5fd75845ca40f461e31350d86963dffec6cd6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5751a22ada..f96cffda51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\snominal\srow\ssize\sfor\sIPK\sindex\slookups\sslightly,\sfor\sbetter\nbalance. -D 2022-12-03T19:04:09.723 +C Use\sthe\ssmaller\sestimated\srow\ssize\sfor\ssearching\sIPK\stables,\sbut\suse\sthe\noriginal\slarger\srow\ssize\sestimate\sfor\sscanning,\ssince\sthe\sleaves\scan\shave\slarge\nrows. +D 2022-12-05T02:42:30.931 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c cf893bd9e48cc4f761beb490e2016cfec7791b0778808a84f2d3d6340085f0d5 +F src/where.c 20f4f51d2d5fb19b984e6ea381b26cf627cc93e64dd9b2ce6a94531aec2f5916 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c ee52c2781c36004d23c85bf111063b78fc16e5e1b6a0d424326af8bf90babb0b F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9800586393c9d3b82459ef657620d245a7985ef5fa389b8a9ea633d6a29c7299 -R bdb8fcd5ad681651f9b1330561cafc6c +P 1a61c500add4a2bfe80c0c691d559cfca166dc5f8262651a58da7ec16a51d430 +R 2840e735a826428dc7a6eae8bd9d8397 U drh -Z f5c8d079aea1d9c7b44238c2c41ad282 +Z f598756dbab89af8b8470b4640ebca93 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f3e63033d3..e4080b53cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a61c500add4a2bfe80c0c691d559cfca166dc5f8262651a58da7ec16a51d430 \ No newline at end of file +df3818997b822743ac407dde45c5fd75845ca40f461e31350d86963dffec6cd6 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 34a6f348d9..a133799472 100644 --- a/src/where.c +++ b/src/where.c @@ -3092,7 +3092,15 @@ static int whereLoopAddBtreeIndex( ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ assert( pSrc->pTab->szTabRow>0 ); - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ + /* The pProbe->szIdxRow is low for an IPK table since the interior + ** pages are small. Thuse szIdxRow gives a good estimate of seek cost. + ** But the leaf pages are full-size, so pProbe->szIdxRow would badly + ** under-estimate the scanning cost. */ + rCostIdx = pNew->nOut + 16; + }else{ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + } pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); From e1774479727bd2093f7bfcf38fd52a28eca6ff04 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 05:30:03 +0000 Subject: [PATCH 152/282] Initial infrastructure for adding virtual table/table-valued function support to WASM. FossilOrigin-Name: c202d7a0398b9aabc2babba5c4c91a313f32bbf37549d419775642bb4aa3936a --- ext/wasm/api/sqlite3-api-glue.js | 48 ++++++--- ext/wasm/api/sqlite3-wasm.c | 164 ++++++++++++++++++++++++++++++- ext/wasm/common/whwasmutil.js | 26 +++-- ext/wasm/jaccwabyt/jaccwabyt.js | 40 +++++--- ext/wasm/jaccwabyt/jaccwabyt.md | 30 +++--- manifest | 26 ++--- manifest.uuid | 2 +- 7 files changed, 269 insertions(+), 67 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index ac1e9fd6d6..eddcfddd1e 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -603,13 +603,16 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } wasm.ctype = JSON.parse(wasm.cstringToJs(cJson)); //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); - for(const t of ['access', 'blobFinalizers', 'dataTypes', - 'encodings', 'fcntl', 'flock', 'ioCap', - 'limits', - 'openFlags', 'prepareFlags', 'resultCodes', - 'serialize', 'syncFlags', 'trace', 'udfFlags', - 'version' - ]){ + const defineGroups = ['access', 'blobFinalizers', 'dataTypes', + 'encodings', 'fcntl', 'flock', 'ioCap', + 'limits', + 'openFlags', 'prepareFlags', 'resultCodes', + 'serialize', 'syncFlags', 'trace', 'udfFlags', + 'version' ]; + if(wasm.bigIntEnabled){ + defineGroups.push('vtab'); + } + for(const t of defineGroups){ for(const e of Object.entries(wasm.ctype[t])){ // ^^^ [k,v] there triggers a buggy code transformation via // one of the Emscripten-driven optimizers. @@ -629,19 +632,33 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.sqlite3_js_rc_str = (rc)=>__rcMap[rc]; /* Bind all registered C-side structs... */ const notThese = Object.assign(Object.create(null),{ - // Structs NOT to register - WasmTestStruct: true - }); - if(!util.isUIThread()){ + // For each struct to NOT register, map its name to false: + WasmTestStruct: true, /* We remove the kvvfs VFS from Worker threads below. */ - notThese.sqlite3_kvvfs_methods = true; - } + sqlite3_kvvfs_methods: !util.isUIThread(), + sqlite3_index_info: !wasm.bigIntEnabled, + sqlite3_index_constraint: !wasm.bigIntEnabled, + sqlite3_index_orderby: !wasm.bigIntEnabled, + sqlite3_index_constraint_usage: !wasm.bigIntEnabled + }); for(const s of wasm.ctype.structs){ if(!notThese[s.name]){ capi[s.name] = sqlite3.StructBinder(s); } } - }/*end C constant imports*/ + if(capi.sqlite3_index_info){ + /* Move these inner structs into sqlite3_index_info. Binding + ** them to WASM requires that we create global-scope structs to + ** model them with, but those are no longer needed after we've + ** passed them to StructBinder. */ + for(const k of ['sqlite3_index_constraint', + 'sqlite3_index_orderby', + 'sqlite3_index_constraint_usage']){ + capi.sqlite3_index_info[k] = capi[k]; + delete capi[k]; + } + } + }/*end C constant and struct imports*/ const pKvvfs = capi.sqlite3_vfs_find("kvvfs"); if( pKvvfs ){/* kvvfs-specific glue */ @@ -652,8 +669,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ delete capi.sqlite3_kvvfs_methods; const kvvfsMakeKey = wasm.exports.sqlite3_wasm_kvvfsMakeKeyOnPstack, - pstack = wasm.pstack, - pAllocRaw = wasm.exports.sqlite3_wasm_pstack_alloc; + pstack = wasm.pstack; const kvvfsStorage = (zClass)=> ((115/*=='s'*/===wasm.getMemValue(zClass)) diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index f6499243a6..dc5dff62a7 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -368,7 +368,7 @@ void sqlite3_wasm_test_struct(WasmTestStruct * s){ */ SQLITE_WASM_KEEP const char * sqlite3_wasm_enum_json(void){ - static char aBuffer[1024 * 12] = {0} /* where the JSON goes */; + static char aBuffer[1024 * 16] = {0} /* where the JSON goes */; int n = 0, nChildren = 0, nStruct = 0 /* output counters for figuring out where commas go */; char * zPos = &aBuffer[1] /* skip first byte for now to help protect @@ -410,6 +410,14 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_ACCESS_READ)/*docs say this is unused*/; } _DefGroup; +#if 0 + /* TODO? Authorizer... */ + DefGroup(authorizer){ + DefInt(SQLITE_DENY); + DefInt(SQLITE_IGNORE); + } _DefGroup; +#endif + DefGroup(blobFinalizers) { /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as ** integers to avoid casting-related warnings. */ @@ -681,7 +689,28 @@ const char * sqlite3_wasm_enum_json(void){ DefStr(SQLITE_VERSION); DefStr(SQLITE_SOURCE_ID); } _DefGroup; - + + DefGroup(vtab) { + DefInt(SQLITE_INDEX_SCAN_UNIQUE); + DefInt(SQLITE_INDEX_CONSTRAINT_EQ); + DefInt(SQLITE_INDEX_CONSTRAINT_GT); + DefInt(SQLITE_INDEX_CONSTRAINT_LE); + DefInt(SQLITE_INDEX_CONSTRAINT_LT); + DefInt(SQLITE_INDEX_CONSTRAINT_GE); + DefInt(SQLITE_INDEX_CONSTRAINT_MATCH); + DefInt(SQLITE_INDEX_CONSTRAINT_LIKE); + DefInt(SQLITE_INDEX_CONSTRAINT_GLOB); + DefInt(SQLITE_INDEX_CONSTRAINT_REGEXP); + DefInt(SQLITE_INDEX_CONSTRAINT_NE); + DefInt(SQLITE_INDEX_CONSTRAINT_ISNOT); + DefInt(SQLITE_INDEX_CONSTRAINT_ISNOTNULL); + DefInt(SQLITE_INDEX_CONSTRAINT_ISNULL); + DefInt(SQLITE_INDEX_CONSTRAINT_IS); + DefInt(SQLITE_INDEX_CONSTRAINT_LIMIT); + DefInt(SQLITE_INDEX_CONSTRAINT_OFFSET); + DefInt(SQLITE_INDEX_CONSTRAINT_FUNCTION); + } _DefGroup; + #undef DefGroup #undef DefStr #undef DefInt @@ -793,6 +822,137 @@ const char * sqlite3_wasm_enum_json(void){ } _StructBinder; #undef CurrentStruct + +#define CurrentStruct sqlite3_vtab + StructBinder { + M(pModule, "p"); + M(nRef, "i"); + M(zErrMsg, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_vtab_cursor + StructBinder { + M(pVtab, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_module + StructBinder { + M(iVersion, "i"); + M(xCreate, "i(ppippp)"); + M(xConnect, "i(ppippp)"); + M(xBestIndex, "i(pp)"); + M(xDisconnect, "i(p)"); + M(xDestroy, "i(p)"); + M(xOpen, "i(pp)"); + M(xClose, "i(p)"); + M(xFilter, "i(pisip)"); + M(xNext, "i(p)"); + M(xEof, "i(p)"); + M(xColumn, "i(ppi)"); + M(xRowid, "i(pp)"); + M(xUpdate, "i(pipp)"); + M(xBegin, "i(p)"); + M(xSync, "i(p)"); + M(xCommit, "i(p)"); + M(xRollback, "i(p)"); + M(xFindFunction, "i(pispp)"); + M(xRename, "i(ps)"); + // ^^^ v1. v2+ follows... + M(xSavepoint, "i(pi)"); + M(xRelease, "i(pi)"); + M(xRollbackTo, "i(pi)"); + // ^^^ v2. v3+ follows... + M(xShadowName, "i(s)"); + } _StructBinder; +#undef CurrentStruct + /* + module/vtab todos: + + - sqlite3_create_module() + - sqlite3_create_module_v2() + - sqlite3_drop_modules() + - sqlite3_declare_vtab() + - sqlite3_overload_function() + */ + + /** + ** Workaround: in order to map the various inner structs from + ** sqlite3_index_info, we have to uplift those into constructs we + ** can access by type name. These structs _must_ match their + ** in-sqlite3_index_info counterparts byte for byte. + */ + typedef struct { + int iColumn; + unsigned char op; + unsigned char usable; + int iTermOffset; + } sqlite3_index_constraint; + typedef struct { + int iColumn; + unsigned char desc; + } sqlite3_index_orderby; + typedef struct { + int argvIndex; + unsigned char omit; + } sqlite3_index_constraint_usage; + { /* Validate that the above struct sizeof()s match + ** expectations. We could improve upon this by + ** checking the offsetof() for each member. */ + const sqlite3_index_info siiCheck; +#define IndexSzCheck(T,M) \ + (sizeof(T) == sizeof(*siiCheck.M)) + if(!IndexSzCheck(sqlite3_index_constraint,aConstraint) + || !IndexSzCheck(sqlite3_index_orderby,aOrderBy) + || !IndexSzCheck(sqlite3_index_constraint_usage,aConstraintUsage)){ + assert(!"sizeof mismatch in sqlite3_index_... struct(s)"); + return 0; + } +#undef IndexSzCheck + } + +#define CurrentStruct sqlite3_index_constraint + StructBinder { + M(iColumn, "i"); + M(op, "C"); + M(usable, "C"); + M(iTermOffset, "i"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_index_orderby + StructBinder { + M(iColumn, "i"); + M(desc, "C"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_index_constraint_usage + StructBinder { + M(argvIndex, "i"); + M(omit, "C"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_index_info + StructBinder { + M(nConstraint, "i"); + M(aConstraint, "p"); + M(nOrderBy, "i"); + M(aOrderBy, "p"); + M(aConstraintUsage, "p"); + M(idxNum, "i"); + M(idxStr, "p"); + M(needToFreeIdxStr, "i"); + M(orderByConsumed, "i"); + M(estimatedCost, "d"); + M(estimatedRows, "j"); + M(idxFlags, "i"); + M(colUsed, "j"); + } _StructBinder; +#undef CurrentStruct + #if SQLITE_WASM_TESTS #define CurrentStruct WasmTestStruct StructBinder { diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 8cb4833247..84d829daba 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -1081,10 +1081,9 @@ self.WhWasmUtilInstaller = function(target){ // impl for allocMainArgv() and scopedAllocMainArgv(). const __allocMainArgv = function(isScoped, list){ - if(!list.length) toss("Cannot allocate empty array."); const pList = target[ isScoped ? 'scopedAlloc' : 'alloc' - ](list.length * target.ptrSizeof); + ]((list.length + 1) * target.ptrSizeof); let i = 0; list.forEach((e)=>{ target.setPtrValue(pList + (target.ptrSizeof * i++), @@ -1092,26 +1091,33 @@ self.WhWasmUtilInstaller = function(target){ isScoped ? 'scopedAllocCString' : 'allocCString' ](""+e)); }); + target.setPtrValue(pList + (target.ptrSizeof * i), 0); return pList; }; /** Creates an array, using scopedAlloc(), suitable for passing to a C-level main() routine. The input is a collection with a length - property and a forEach() method. A block of memory list.length - entries long is allocated and each pointer-sized block of that - memory is populated with a scopedAllocCString() conversion of the - (""+value) of each element. Returns a pointer to the start of the - list, suitable for passing as the 2nd argument to a C-style - main() function. + property and a forEach() method. A block of memory + (list.length+1) entries long is allocated and each pointer-sized + block of that memory is populated with a scopedAllocCString() + conversion of the (""+value) of each element, with the exception + that the final entry is a NULL pointer. Returns a pointer to the + start of the list, suitable for passing as the 2nd argument to a + C-style main() function. - Throws if list.length is falsy or scopedAllocPush() is not active. + Throws if scopedAllocPush() is not active. + + Design note: the returned array is allocated with an extra NULL + pointer entry to accommodate certain APIs, but client code which + does not need that functionality should treat the returned array + as list.length entries long. */ target.scopedAllocMainArgv = (list)=>__allocMainArgv(true, list); /** Identical to scopedAllocMainArgv() but uses alloc() instead of - scopedAllocMainArgv + scopedAlloc(). */ target.allocMainArgv = (list)=>__allocMainArgv(false, list); diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index dee7258c51..81b4c1bd4f 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -121,6 +121,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ at SIG s[0]. Throws for an unknown SIG. */ const sigIR = function(s){ switch(sigLetter(s)){ + case 'c': case 'C': return 'i8'; case 'i': return 'i32'; case 'p': case 'P': case 's': return ptrIR; case 'j': return 'i64'; @@ -133,6 +134,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ unknown SIG. */ const sigSizeof = function(s){ switch(sigLetter(s)){ + case 'c': case 'C': return 1; case 'i': return 4; case 'p': case 'P': case 's': return ptrSizeof; case 'j': return 8; @@ -168,6 +170,8 @@ self.Jaccwabyt = function StructBinderFactory(config){ break; } case 'i': return 'getInt32'; + case 'c': return 'getInt8'; + case 'C': return 'getUint8'; case 'j': return affirmBigIntArray() && 'getBigInt64'; case 'f': return 'getFloat32'; case 'd': return 'getFloat64'; @@ -186,6 +190,8 @@ self.Jaccwabyt = function StructBinderFactory(config){ break; } case 'i': return 'setInt32'; + case 'c': return 'setInt8'; + case 'C': return 'setUint8'; case 'j': return affirmBigIntArray() && 'setBigInt64'; case 'f': return 'setFloat32'; case 'd': return 'setFloat64'; @@ -199,7 +205,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ */ const sigDVSetWrapper = function(s){ switch(sigLetter(s)) { - case 'i': case 'f': case 'd': return Number; + case 'i': case 'f': case 'c': case 'C': case 'd': return Number; case 'j': return affirmBigIntArray() && BigInt; case 'p': case 'P': case 's': switch(ptrSizeof){ @@ -361,7 +367,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ framework's native format or in Emscripten format. */ const __memberSignature = function f(obj,memberName,emscriptenFormat=false){ - if(!f._) f._ = (x)=>x.replace(/[^vipPsjrd]/g,"").replace(/[pPs]/g,'i'); + if(!f._) f._ = (x)=>x.replace(/[^vipPsjrdcC]/g,"").replace(/[pPscC]/g,'i'); const m = __lookupMember(obj.structInfo, memberName, true); return emscriptenFormat ? f._(m.signature) : m.signature; }; @@ -570,7 +576,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ /*cache all available getters/setters/set-wrappers for direct reuse in each accessor function. */ f._ = {getters: {}, setters: {}, sw:{}}; - const a = ['i','p','P','s','f','d','v()']; + const a = ['i','c','C','p','P','s','f','d','v()']; if(bigIntEnabled) a.push('j'); a.forEach(function(v){ //const ir = sigIR(v); @@ -579,8 +585,8 @@ self.Jaccwabyt = function StructBinderFactory(config){ f._.sw[v] = sigDVSetWrapper(v) /* BigInt or Number ctor to wrap around values for conversion */; }); - const rxSig1 = /^[ipPsjfd]$/, - rxSig2 = /^[vipPsjfd]\([ipPsjfd]*\)$/; + const rxSig1 = /^[ipPsjfdcC]$/, + rxSig2 = /^[vipPsjfdcC]\([ipPsjfdcC]*\)$/; f.sigCheck = function(obj, name, key,sig){ if(Object.prototype.hasOwnProperty.call(obj, key)){ toss(obj.structName,'already has a property named',key+'.'); @@ -594,7 +600,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ f.sigCheck(ctor.prototype, name, key, descr.signature); descr.key = key; descr.name = name; - const sizeOf = sigSizeof(descr.signature); const sigGlyph = sigLetter(descr.signature); const xPropName = sPropName(ctor.prototype.structName,key); const dbg = ctor.prototype.debugFlags.__flags; @@ -610,10 +615,10 @@ self.Jaccwabyt = function StructBinderFactory(config){ prop.get = function(){ if(dbg.getter){ log("debug.getter:",f._.getters[sigGlyph],"for", sigIR(sigGlyph), - xPropName,'@', this.pointer,'+',descr.offset,'sz',sizeOf); + xPropName,'@', this.pointer,'+',descr.offset,'sz',descr.sizeof); } let rc = ( - new DataView(heap().buffer, this.pointer + descr.offset, sizeOf) + new DataView(heap().buffer, this.pointer + descr.offset, descr.sizeof) )[f._.getters[sigGlyph]](0, isLittleEndian); if(dbg.getter) log("debug.getter:",xPropName,"result =",rc); if(rc && isAutoPtrSig(descr.signature)){ @@ -628,7 +633,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ prop.set = function(v){ if(dbg.setter){ log("debug.setter:",f._.setters[sigGlyph],"for", sigIR(sigGlyph), - xPropName,'@', this.pointer,'+',descr.offset,'sz',sizeOf, v); + xPropName,'@', this.pointer,'+',descr.offset,'sz',descr.sizeof, v); } if(!this.pointer){ toss("Cannot set struct property on disposed instance."); @@ -644,7 +649,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ toss("Invalid value for pointer-type",xPropName+'.'); } ( - new DataView(heap().buffer, this.pointer + descr.offset, sizeOf) + new DataView(heap().buffer, this.pointer + descr.offset, descr.sizeof) )[f._.setters[sigGlyph]](0, f._.sw[sigGlyph](v), isLittleEndian); }; } @@ -665,13 +670,18 @@ self.Jaccwabyt = function StructBinderFactory(config){ if(!structName) toss("Struct name is required."); let lastMember = false; Object.keys(structInfo.members).forEach((k)=>{ + // Sanity checks of sizeof/offset info... const m = structInfo.members[k]; if(!m.sizeof) toss(structName,"member",k,"is missing sizeof."); - else if(0!==(m.sizeof%4)){ - toss(structName,"member",k,"sizeof is not aligned."); - } - else if(0!==(m.offset%4)){ - toss(structName,"member",k,"offset is not aligned."); + else if(m.sizeof>1){ // offsets of size-1 members may be odd values. + if(0!==(m.sizeof%4)){ + console.warn("Invalid struct description =",m,"from",structInfo.members); + toss(structName,"member",k,"sizeof is not aligned. sizeof="+m.sizeof); + } + if(0!==(m.offset%4)){ + console.warn("Invalid struct description =",m,"from",structInfo.members); + toss(structName,"member",k,"offset is not aligned. offset="+m.offset); + } } if(!lastMember || lastMember.offset < m.offset) lastMember = m; }); diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index edcba260a7..73f6bfa947 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -281,21 +281,29 @@ supported letters are: signature entry. - **`f`** = `float` (4 bytes) - **`d`** = `double` (8 bytes) -- **`p`** = `int32` (but see below!) +- **`c`** = `int8` (char - see notes below!) +- **`C`** = `int8` (unsigned char - see notes below!) +- **`p`** = `int32` (see notes below!) - **`P`** = Like `p` but with extra handling. Described below. -- **`s`** = like `int32` but is a _hint_ that it's a pointer to a string - so that _some_ (very limited) contexts may treat it as such, noting - such algorithms must, for lack of information to the contrary, - assume both that the encoding is UTF-8 and that the pointer's member - is NUL-terminated. If that is _not_ the case for a given string - member, do not use `s`: use `i` or `p` instead and do any string - handling yourself. +- **`s`** = like `int32` but is a _hint_ that it's a pointer to a + string so that _some_ (very limited) contexts may treat it as such, + noting that such algorithms must, for lack of information to the + contrary, assume both that the encoding is UTF-8 and that the + pointer's member is NUL-terminated. If that is _not_ the case for a + given string member, do not use `s`: use `i` or `p` instead and do + any string handling yourself. Noting that: - All of these types are numeric. Attempting to set any struct-bound property to a non-numeric value will trigger an exception except in cases explicitly noted otherwise. +- "Char" types: WASM does not define an `int8` type, nor does it + distinguish between signed and unsigned. This API treats `c` as + `int8` and `C` as `uint8` for purposes of getting and setting values + when using the `DataView` class. It is _not_ recommended that client + code use these types in new WASM-capable code, but they were added + for the sake of binding some immutable legacy code to WASM. > Sidebar: Emscripten's public docs do not mention `p`, but their generated code includes `p` as an alias for `i`, presumably to mean @@ -317,12 +325,12 @@ Signatures in the form `x(...)` denote function-pointer members and form `x()`. For function-type signatures, the strings are formulated such that they can be passed to Emscripten's `addFunction()` after stripping out the `(` and `)` characters. For good measure, to match -the public Emscripten docs, `p` should also be replaced with `i`. In -JavaScript that might look like: +the public Emscripten docs, `p`, `c`, and `C`, should also be replaced +with `i`. In JavaScript that might look like: > ``` -signature.replace(/[^vipPsjfd]/g,'').replace(/[pPs]/g,'i'); +signature.replace(/[^vipPsjfdcC]/g,'').replace(/[pPscC]/g,'i'); ``` diff --git a/manifest b/manifest index 27a4331f16..7d4a4ffa9e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\squery\splanner\scost\sestimates.\s\sFix\sfor\sticket\s[e8b674241947eb3b]. -D 2022-12-05T02:52:37.959 +C Initial\sinfrastructure\sfor\sadding\svirtual\stable/table-valued\sfunction\ssupport\sto\sWASM. +D 2022-12-05T05:30:03.152 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,7 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js 6fe39964605fda3b699f69365eed565b5172d29cab2c49bc057a43f9a93f9f36 +F ext/wasm/api/sqlite3-api-glue.js 6028d0c3e6f475a513040a45612238b749ec6c155181d5bd029d66577ab4d0d6 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 F ext/wasm/api/sqlite3-api-prologue.js 697a5989ad52a9ba7bc60b5436589bd05885ee2201d84c38c5e9af3876af3ba4 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f @@ -512,7 +512,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c b0babf8435f31d21f28454fb81433aa538c68b23d0a4a251f0666fdec4e71f59 +F ext/wasm/api/sqlite3-wasm.c 5120fb3419aba02d20cbe1e645b58dae5faeaaae8ccd46b8931ae04d311df9e5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -521,7 +521,7 @@ F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c07 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/common/testing.css 35889709547d89a6109ff83b25c11bbc91d8dd43aab8722e428655ca98880a06 -F ext/wasm/common/whwasmutil.js c1bc5715cd96728929cc31d788b16152ccbd6b2e111d2e88fbc9725247e67b4f +F ext/wasm/common/whwasmutil.js 1bc1c973662db7d52763512e67c2369fdbfe9e8bae33069ac89f4fbe40d678b2 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 @@ -539,8 +539,8 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff -F ext/wasm/jaccwabyt/jaccwabyt.js 95f573de1826474c9605dda620ee622fcb1673ae74f191eb324c0853aa4dcb66 -F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e +F ext/wasm/jaccwabyt/jaccwabyt.js 7c41784c442aa67f0e86e7c14aa51b3a28e3b21eb579c71c16af3c988fbf966f +F ext/wasm/jaccwabyt/jaccwabyt.md 50df3ccb7a773634000496a2ccb805dd25cf8cd963696a7deec3209a68c6093b F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 @@ -2065,9 +2065,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cefc032473ac5ad244c0b6402c541b2f76c0c65a041bda03bfbe7c0e2c11fac2 df3818997b822743ac407dde45c5fd75845ca40f461e31350d86963dffec6cd6 -R 3c93957174c9bb89beba8a6d7c9fdaab -T +closed df3818997b822743ac407dde45c5fd75845ca40f461e31350d86963dffec6cd6 -U drh -Z 2d83b2d52ce6d0637e1b84504b86ac00 +P 1b779afa3ed2f35a110e460fc6ed13cba744db85b9924149ab028b100d1e1e12 +R b429259a443dd7b0ac8ceede26128588 +T *branch * wasm-vtab +T *sym-wasm-vtab * +T -sym-trunk * Cancelled\sby\sbranch. +U stephan +Z bd980b10d448d088ee0cbb8238fc2a43 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 452c8ac5d7..f70604b10c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b779afa3ed2f35a110e460fc6ed13cba744db85b9924149ab028b100d1e1e12 \ No newline at end of file +c202d7a0398b9aabc2babba5c4c91a313f32bbf37549d419775642bb4aa3936a \ No newline at end of file From 864c3c029b869e5bfc58fcdf35f883e38c2cfa0d Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 05:45:00 +0000 Subject: [PATCH 153/282] Remove some dead code. Improve some error checks and comments. FossilOrigin-Name: 6712fbe46a97867cea309f78a274edbb6bd166a505b41e18a580306da0e063db --- ext/wasm/api/sqlite3-api-glue.js | 5 ++-- ext/wasm/jaccwabyt/jaccwabyt.js | 42 ++++++++++---------------------- manifest | 17 ++++++------- manifest.uuid | 2 +- 4 files changed, 24 insertions(+), 42 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index eddcfddd1e..582b0c0a30 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -632,10 +632,11 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.sqlite3_js_rc_str = (rc)=>__rcMap[rc]; /* Bind all registered C-side structs... */ const notThese = Object.assign(Object.create(null),{ - // For each struct to NOT register, map its name to false: + // For each struct to NOT register, map its name to true: WasmTestStruct: true, - /* We remove the kvvfs VFS from Worker threads below. */ + /* We unregister the kvvfs VFS from Worker threads below. */ sqlite3_kvvfs_methods: !util.isUIThread(), + /* sqlite3_index_info and friends require int64: */ sqlite3_index_info: !wasm.bigIntEnabled, sqlite3_index_constraint: !wasm.bigIntEnabled, sqlite3_index_orderby: !wasm.bigIntEnabled, diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index 81b4c1bd4f..7c8d42b331 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -130,34 +130,9 @@ self.Jaccwabyt = function StructBinderFactory(config){ } toss("Unhandled signature IR:",s); }; - /** Returns the sizeof value for the given SIG. Throws for an - unknown SIG. */ - const sigSizeof = function(s){ - switch(sigLetter(s)){ - case 'c': case 'C': return 1; - case 'i': return 4; - case 'p': case 'P': case 's': return ptrSizeof; - case 'j': return 8; - case 'f': return 4 /* C-side floats, not JS-side */; - case 'd': return 8; - } - toss("Unhandled signature sizeof:",s); - }; + const affirmBigIntArray = BigInt64Array ? ()=>true : ()=>toss('BigInt64Array is not available.'); - /** Returns the (signed) TypedArray associated with the type - described by the given SIG. Throws for an unknown SIG. */ - /********** - const sigTypedArray = function(s){ - switch(sigIR(s)) { - case 'i32': return Int32Array; - case 'i64': return affirmBigIntArray() && BigInt64Array; - case 'float': return Float32Array; - case 'double': return Float64Array; - } - toss("Unhandled signature TypedArray:",s); - }; - **************/ /** Returns the name of a DataView getter method corresponding to the given SIG. */ const sigDVGetter = function(s){ @@ -217,6 +192,8 @@ self.Jaccwabyt = function StructBinderFactory(config){ toss("Unhandled DataView set wrapper for signature:",s); }; + /** Returns the given struct and member name in a form suitable for + debugging and error output. */ const sPropName = (s,k)=>s+'::'+k; const __propThrowOnSet = function(structName,propName){ @@ -673,13 +650,20 @@ self.Jaccwabyt = function StructBinderFactory(config){ // Sanity checks of sizeof/offset info... const m = structInfo.members[k]; if(!m.sizeof) toss(structName,"member",k,"is missing sizeof."); - else if(m.sizeof>1){ // offsets of size-1 members may be odd values. + else if(m.sizeof===1){ + (m.signature === 'c' || m.signature === 'C') || + toss("Unexpected sizeof==1 member", + sPropName(structInfo.name,k), + "with signature",m.signature); + }else{ + // sizes and offsets of size-1 members may be odd values, but + // others may not. if(0!==(m.sizeof%4)){ - console.warn("Invalid struct description =",m,"from",structInfo.members); + console.warn("Invalid struct member description =",m,"from",structInfo); toss(structName,"member",k,"sizeof is not aligned. sizeof="+m.sizeof); } if(0!==(m.offset%4)){ - console.warn("Invalid struct description =",m,"from",structInfo.members); + console.warn("Invalid struct member description =",m,"from",structInfo); toss(structName,"member",k,"offset is not aligned. offset="+m.offset); } } diff --git a/manifest b/manifest index 7d4a4ffa9e..ceb804eae8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\sinfrastructure\sfor\sadding\svirtual\stable/table-valued\sfunction\ssupport\sto\sWASM. -D 2022-12-05T05:30:03.152 +C Remove\ssome\sdead\scode.\sImprove\ssome\serror\schecks\sand\scomments. +D 2022-12-05T05:45:00.486 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,7 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js 6028d0c3e6f475a513040a45612238b749ec6c155181d5bd029d66577ab4d0d6 +F ext/wasm/api/sqlite3-api-glue.js 40504fa3d382b9181e20e299ac5318ef55345090b8f5098601d7f759d57d6418 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 F ext/wasm/api/sqlite3-api-prologue.js 697a5989ad52a9ba7bc60b5436589bd05885ee2201d84c38c5e9af3876af3ba4 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f @@ -539,7 +539,7 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff -F ext/wasm/jaccwabyt/jaccwabyt.js 7c41784c442aa67f0e86e7c14aa51b3a28e3b21eb579c71c16af3c988fbf966f +F ext/wasm/jaccwabyt/jaccwabyt.js 292d37ad3b5fcef420db7b449813c38f1656f490f43e214ea4895793158e7fce F ext/wasm/jaccwabyt/jaccwabyt.md 50df3ccb7a773634000496a2ccb805dd25cf8cd963696a7deec3209a68c6093b F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 @@ -2065,11 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1b779afa3ed2f35a110e460fc6ed13cba744db85b9924149ab028b100d1e1e12 -R b429259a443dd7b0ac8ceede26128588 -T *branch * wasm-vtab -T *sym-wasm-vtab * -T -sym-trunk * Cancelled\sby\sbranch. +P c202d7a0398b9aabc2babba5c4c91a313f32bbf37549d419775642bb4aa3936a +R acfab192817c95e0e585836998ec2507 U stephan -Z bd980b10d448d088ee0cbb8238fc2a43 +Z 572c98dbcc011253d2f467baefba9d4d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f70604b10c..cddf20fae5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c202d7a0398b9aabc2babba5c4c91a313f32bbf37549d419775642bb4aa3936a \ No newline at end of file +6712fbe46a97867cea309f78a274edbb6bd166a505b41e18a580306da0e063db \ No newline at end of file From 08fc64ea04d7b8ce458ba13608502159a2727738 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 07:51:25 +0000 Subject: [PATCH 154/282] More work on the JS side of the virtual table APIs. FossilOrigin-Name: cb9881ec001b0e2faf047e57acfd1722d2b546255a54e0f850f568edfe2df1cd --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 14 +++++ ext/wasm/api/sqlite3-api-glue.js | 64 ++++++++------------- ext/wasm/api/sqlite3-api-prologue.js | 22 +++++++ ext/wasm/api/sqlite3-wasm.c | 44 ++++++++++---- manifest | 18 +++--- manifest.uuid | 2 +- 6 files changed, 102 insertions(+), 62 deletions(-) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index 096bf44018..12ab94b2dc 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -7,6 +7,7 @@ _sqlite3_bind_null _sqlite3_bind_parameter_count _sqlite3_bind_parameter_index _sqlite3_bind_text +_sqlite3_busy_handler _sqlite3_busy_timeout _sqlite3_changes _sqlite3_changes64 @@ -24,14 +25,19 @@ _sqlite3_column_text _sqlite3_column_type _sqlite3_compileoption_get _sqlite3_compileoption_used +_sqlite3_complete _sqlite3_create_function _sqlite3_create_function_v2 +_sqlite3_create_module +_sqlite3_create_module_v2 _sqlite3_create_window_function _sqlite3_data_count _sqlite3_db_filename _sqlite3_db_handle _sqlite3_db_name +_sqlite3_declare_vtab _sqlite3_deserialize +_sqlite3_drop_modules _sqlite3_errmsg _sqlite3_error_offset _sqlite3_errstr @@ -50,6 +56,7 @@ _sqlite3_malloc64 _sqlite3_msize _sqlite3_open _sqlite3_open_v2 +_sqlite3_overload_function _sqlite3_prepare_v2 _sqlite3_prepare_v3 _sqlite3_randomness @@ -93,6 +100,13 @@ _sqlite3_value_type _sqlite3_vfs_find _sqlite3_vfs_register _sqlite3_vfs_unregister +_sqlite3_vtab_distinct +_sqlite3_vtab_in +_sqlite3_vtab_in_first +_sqlite3_vtab_in_next +_sqlite3_vtab_nochange +_sqlite3_vtab_on_conflict +_sqlite3_vtab_rhs_value _malloc _free _realloc diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 582b0c0a30..c1c91136ac 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -37,20 +37,6 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }); delete self.Jaccwabyt; - if(0){ - /* "The problem" is that the following isn't even remotely - type-safe. OTOH, nothing about WASM pointers is. */ - const argPointer = wasm.xWrap.argAdapter('*'); - wasm.xWrap.argAdapter('StructType', (v)=>{ - if(v && v.constructor && v instanceof StructBinder.StructType){ - v = v.pointer; - } - return wasm.isPtr(v) - ? argPointer(v) - : toss("Invalid (object) type for StructType-type argument."); - }); - } - {/* Convert Arrays and certain TypedArrays to strings for 'flexible-string'-type arguments */ const xString = wasm.xWrap.argAdapter('string'); @@ -68,15 +54,21 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `sqlite3_vfs*` via capi.sqlite3_vfs.pointer. */ const aPtr = wasm.xWrap.argAdapter('*'); + const nilType = function(){}; wasm.xWrap.argAdapter('sqlite3_filename', aPtr) ('sqlite3_stmt*', aPtr) ('sqlite3_context*', aPtr) ('sqlite3_value*', aPtr) ('void*', aPtr) - ('sqlite3*', (v)=>{ - if(sqlite3.oo1 && v instanceof sqlite3.oo1.DB) v = v.pointer; - return aPtr(v); - }) + ('sqlite3*', (v)=> + aPtr((v instanceof (sqlite3?.oo1?.DB || nilType)) + ? v.pointer : v)) + ('sqlite3_index_info*', (v)=> + aPtr((v instanceof (capi.sqlite3_index_info || nilType)) + ? v.pointer : v)) + ('sqlite3_module*', (v)=> + aPtr((v instanceof (capi.sqlite3_module || nilType)) + ? v.pointer : v)) /** `sqlite3_vfs*`: @@ -87,14 +79,13 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ ('sqlite3_vfs*', (v)=>{ if('string'===typeof v){ - const x = capi.sqlite3_vfs_find(v); /* A NULL sqlite3_vfs pointer will be treated as the default VFS in many contexts. We specifically do not want that behavior here. */ - if(!x) sqlite3.SQLite3Error.toss("Unknown sqlite3_vfs name:",v); - return x; - }else if(v instanceof sqlite3.capi.sqlite3_vfs) v = v.pointer; - return aPtr(v); + return capi.sqlite3_vfs_find(v) + || sqlite3.SQLite3Error.toss("Unknown sqlite3_vfs name:",v); + } + return aPtr((v instanceof capi.sqlite3_vfs) ? v.pointer : v); }); wasm.xWrap.resultAdapter('sqlite3*', aPtr) @@ -127,7 +118,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ : fI64Disabled(e[0]); } - /* There's no(?) need to expose bindingSignatures to clients, + /* There's no need to expose bindingSignatures to clients, implicitly making it part of the public interface. */ delete wasm.bindingSignatures; @@ -141,21 +132,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return errCode; }; } - }/*xWrap() bindings*/; - /** - When registering a VFS and its related components it may be - necessary to ensure that JS keeps a reference to them to keep - them from getting garbage collected. Simply pass each such value - to this function and a reference will be held to it for the life - of the app. - */ - capi.sqlite3_vfs_register.addReference = function f(...args){ - if(!f._) f._ = []; - f._.push(...args); - }; - /** Internal helper to assist in validating call argument counts in the hand-written sqlite3_xyz() wrappers. We do this only for @@ -603,10 +581,11 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } wasm.ctype = JSON.parse(wasm.cstringToJs(cJson)); //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); - const defineGroups = ['access', 'blobFinalizers', 'dataTypes', + const defineGroups = ['access', 'authorizer', + 'blobFinalizers', 'dataTypes', 'encodings', 'fcntl', 'flock', 'ioCap', - 'limits', - 'openFlags', 'prepareFlags', 'resultCodes', + 'limits', 'openFlags', + 'prepareFlags', 'resultCodes', 'serialize', 'syncFlags', 'trace', 'udfFlags', 'version' ]; if(wasm.bigIntEnabled){ @@ -658,7 +637,10 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.sqlite3_index_info[k] = capi[k]; delete capi[k]; } - } + capi.sqlite3_vtab_config = + (pDb, op, arg=0)=>wasm.exports.sqlite3_wasm_vtab_config( + wasm.xWrap.argAdapter('sqlite3*')(pDb), op, arg); + }/* end vtab-related setup */ }/*end C constant and struct imports*/ const pKvvfs = capi.sqlite3_vfs_find("kvvfs"); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 5ebe7af058..b66497e6ba 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -890,6 +890,10 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( the lines of sqlite3_prepare_v3(). The slightly problematic part is the final argument (text destructor). */ ], + //["sqlite3_busy_handler","int", "sqlite3*", "*", "*"], + // ^^^^ TODO: custom binding which auto-converts JS function arg + // to a WASM function, noting that calling it multiple times + // would introduce a leak. ["sqlite3_busy_timeout","int", "sqlite3*", "int"], ["sqlite3_close_v2", "int", "sqlite3*"], ["sqlite3_changes", "int", "sqlite3*"], @@ -904,6 +908,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], ["sqlite3_compileoption_get", "string", "int"], ["sqlite3_compileoption_used", "int", "string"], + ["sqlite3_complete", "int", "flexible-string"], /* sqlite3_create_function(), sqlite3_create_function_v2(), and sqlite3_create_window_function() use hand-written bindings to simplify handling of their function-type arguments. */ @@ -998,14 +1003,31 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], ["sqlite3_changes64","i64", ["sqlite3*"]], ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], + ["sqlite3_create_module", "int", + ["sqlite3*","string","sqlite3_module*","*"]], + ["sqlite3_create_module_v2", "int", + ["sqlite3*","string","sqlite3_module*","*","*"]], + ["sqlite3_declare_vtab", "int", ["sqlite3*", "flexible-string"]], + ["sqlite3_drop_modules", "int", ["sqlite3*", "**"]], ["sqlite3_malloc64", "*","i64"], ["sqlite3_msize", "i64", "*"], + ["sqlite3_overload_function", "int", ["sqlite3*","string","int"]], ["sqlite3_realloc64", "*","*", "i64"], ["sqlite3_result_int64",undefined, "*", "i64"], ["sqlite3_result_zeroblob64", "int", "*", "i64"], ["sqlite3_total_changes64", "i64", ["sqlite3*"]], ["sqlite3_uri_int64", "i64", ["sqlite3_filename", "string", "i64"]], ["sqlite3_value_int64","i64", "sqlite3_value*"], + //EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int) + ["sqlite3_vtab_distinct","int", "sqlite3_index_info*"], + ["sqlite3_vtab_in","int", "sqlite3_index_info*", "int", "int"], + ["sqlite3_vtab_in_first", "int", "sqlite3_value*", "**"], + ["sqlite3_vtab_in_next", "int", "sqlite3_value*", "**"], + /*["sqlite3_vtab_config" is variadic and requires a hand-written + proxy.] */ + ["sqlite3_vtab_nochange","int", "sqlite3_context*"], + ["sqlite3_vtab_on_conflict","int", "sqlite3*"], + ["sqlite3_vtab_rhs_value","int", "sqlite3_index_info*", "int", "**"] ]; /** diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index dc5dff62a7..96234bca3a 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -410,13 +410,11 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_ACCESS_READ)/*docs say this is unused*/; } _DefGroup; -#if 0 /* TODO? Authorizer... */ DefGroup(authorizer){ DefInt(SQLITE_DENY); DefInt(SQLITE_IGNORE); } _DefGroup; -#endif DefGroup(blobFinalizers) { /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as @@ -709,6 +707,14 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_INDEX_CONSTRAINT_LIMIT); DefInt(SQLITE_INDEX_CONSTRAINT_OFFSET); DefInt(SQLITE_INDEX_CONSTRAINT_FUNCTION); + DefInt(SQLITE_VTAB_CONSTRAINT_SUPPORT); + DefInt(SQLITE_VTAB_INNOCUOUS); + DefInt(SQLITE_VTAB_DIRECTONLY); + DefInt(SQLITE_ROLLBACK); + //DefInt(SQLITE_IGNORE); // Also used by sqlite3_authorizer() callback + DefInt(SQLITE_FAIL); + //DefInt(SQLITE_ABORT); // Also an error code + DefInt(SQLITE_REPLACE); } _DefGroup; #undef DefGroup @@ -867,15 +873,6 @@ const char * sqlite3_wasm_enum_json(void){ M(xShadowName, "i(s)"); } _StructBinder; #undef CurrentStruct - /* - module/vtab todos: - - - sqlite3_create_module() - - sqlite3_create_module_v2() - - sqlite3_drop_modules() - - sqlite3_declare_vtab() - - sqlite3_overload_function() - */ /** ** Workaround: in order to map the various inner structs from @@ -1269,6 +1266,31 @@ sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ return &sqlite3KvvfsMethods; } +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This is a proxy for the variadic sqlite3_vtab_config() which passes +** its argument on, or not, to sqlite3_vtab_config(), depending on the +** value of its 2nd argument. Returns the result of +** sqlite3_vtab_config(), or SQLITE_MISUSE if the 2nd arg is not a +** valid value. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ + switch(op){ + case SQLITE_VTAB_DIRECTONLY: + case SQLITE_VTAB_INNOCUOUS: + return sqlite3_vtab_config(pDb, op); + case SQLITE_VTAB_CONSTRAINT_SUPPORT: + return sqlite3_vtab_config(pDb, op, arg); + default: + return SQLITE_MISUSE; + } + +} + + #if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) #include diff --git a/manifest b/manifest index ceb804eae8..08e9801bce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sdead\scode.\sImprove\ssome\serror\schecks\sand\scomments. -D 2022-12-05T05:45:00.486 +C More\swork\son\sthe\sJS\sside\sof\sthe\svirtual\stable\sAPIs. +D 2022-12-05T07:51:25.186 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 89af0612bad5c651f69e629c7e9689be6d3c8a92a9010da5dd90a87c1d86817a +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 5b042c543c83134c20d222b7b8cd9e3401a5d552d709364d82461c064cd26089 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d @@ -503,16 +503,16 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js 40504fa3d382b9181e20e299ac5318ef55345090b8f5098601d7f759d57d6418 +F ext/wasm/api/sqlite3-api-glue.js dd7f3cc427154f280d9718dbc755ae4ed21c81d193f16b32184731c299c1d3be F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js 697a5989ad52a9ba7bc60b5436589bd05885ee2201d84c38c5e9af3876af3ba4 +F ext/wasm/api/sqlite3-api-prologue.js d2cb6dd5ca109b36c5038301a6881f6ca137b946350e79e701114e461ab56298 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 5120fb3419aba02d20cbe1e645b58dae5faeaaae8ccd46b8931ae04d311df9e5 +F ext/wasm/api/sqlite3-wasm.c 723522a6c2a2463884a83fa1cc7ae5770deaaf0856a1058cc1023b2bfa1c898b F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c202d7a0398b9aabc2babba5c4c91a313f32bbf37549d419775642bb4aa3936a -R acfab192817c95e0e585836998ec2507 +P 6712fbe46a97867cea309f78a274edbb6bd166a505b41e18a580306da0e063db +R 556ab7e7db905d4fcefcfcbe9067fc69 U stephan -Z 572c98dbcc011253d2f467baefba9d4d +Z c9a0ef228c6ee4583769e98c5e79436a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cddf20fae5..d7b2b1b5f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6712fbe46a97867cea309f78a274edbb6bd166a505b41e18a580306da0e063db \ No newline at end of file +cb9881ec001b0e2faf047e57acfd1722d2b546255a54e0f850f568edfe2df1cd \ No newline at end of file From fe8ecfd6966ec0c10705877c48f0df1d0f1587e4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 5 Dec 2022 10:50:06 +0000 Subject: [PATCH 155/282] Remove obsolete "experimental" comments on the virtual-table interface documentation in sqlite.h.in. FossilOrigin-Name: 28fc32abdcb1c6af19f0926a3a7317e15930a0ba89645d5ffbf6ce99d8463273 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 19 ------------------- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 27a4331f16..32b47a08bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\squery\splanner\scost\sestimates.\s\sFix\sfor\sticket\s[e8b674241947eb3b]. -D 2022-12-05T02:52:37.959 +C Remove\sobsolete\s"experimental"\scomments\son\sthe\svirtual-table\sinterface\ndocumentation\sin\ssqlite.h.in. +D 2022-12-05T10:50:06.702 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -647,7 +647,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 F src/shell.c.in f6ab148f150dc0c8460be74a61566d37c65d43311e84963cc1a58df3fc277511 -F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 +F src/sqlite.h.in 319be18909674dbab5d0c5372aec098671a35f8ab611cdfc41108df86ab8472b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f F src/sqliteInt.h 4ddd98e423276714479f9f22dbbda050e8ef99aa97e7e26bf0bdf58acef0ca42 @@ -2065,9 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cefc032473ac5ad244c0b6402c541b2f76c0c65a041bda03bfbe7c0e2c11fac2 df3818997b822743ac407dde45c5fd75845ca40f461e31350d86963dffec6cd6 -R 3c93957174c9bb89beba8a6d7c9fdaab -T +closed df3818997b822743ac407dde45c5fd75845ca40f461e31350d86963dffec6cd6 +P 1b779afa3ed2f35a110e460fc6ed13cba744db85b9924149ab028b100d1e1e12 +R 76fd9f81f55a4334f31e568ee103dfee U drh -Z 2d83b2d52ce6d0637e1b84504b86ac00 +Z 3d486c52105de074dd9d6853e636a4e9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 452c8ac5d7..c40d99deaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b779afa3ed2f35a110e460fc6ed13cba744db85b9924149ab028b100d1e1e12 \ No newline at end of file +28fc32abdcb1c6af19f0926a3a7317e15930a0ba89645d5ffbf6ce99d8463273 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 369d6b6022..1b9d71e139 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7014,15 +7014,6 @@ int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void)); */ void sqlite3_reset_auto_extension(void); -/* -** The interface to the virtual-table mechanism is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - /* ** Structures used by the virtual table interface */ @@ -7423,16 +7414,6 @@ int sqlite3_declare_vtab(sqlite3*, const char *zSQL); */ int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); -/* -** The interface to the virtual-table mechanism defined above (back up -** to a comment remarkably similar to this one) is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - /* ** CAPI3REF: A Handle To An Open BLOB ** KEYWORDS: {BLOB handle} {BLOB handles} From 0adef0937451b9c93688c00e26a792915361af5d Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 11:30:39 +0000 Subject: [PATCH 156/282] Export sqlite3_bind/value/result_pointer() to wasm. Add 'static-string' argument converter to support the lifetime requirements of bind/result_pointer()'s string argument. Correct an endless loop in wasm.cstrlen() when passed a non-C-string argument. FossilOrigin-Name: a94552434a657376d5ce1831de05c1b15fb153020848cd825fb0df413c3baa70 --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 3 ++ ext/wasm/api/sqlite3-api-glue.js | 33 ++++++++++++++++++++- ext/wasm/api/sqlite3-api-prologue.js | 5 ++++ ext/wasm/common/whwasmutil.js | 9 +++--- manifest | 18 +++++------ manifest.uuid | 2 +- 6 files changed, 55 insertions(+), 15 deletions(-) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index 12ab94b2dc..5c0419e9da 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -6,6 +6,7 @@ _sqlite3_bind_int64 _sqlite3_bind_null _sqlite3_bind_parameter_count _sqlite3_bind_parameter_index +_sqlite3_bind_pointer _sqlite3_bind_text _sqlite3_busy_handler _sqlite3_busy_timeout @@ -72,6 +73,7 @@ _sqlite3_result_error_toobig _sqlite3_result_int _sqlite3_result_int64 _sqlite3_result_null +_sqlite3_result_pointer _sqlite3_result_text _sqlite3_result_zeroblob _sqlite3_result_zeroblob64 @@ -95,6 +97,7 @@ _sqlite3_value_bytes _sqlite3_value_double _sqlite3_value_int _sqlite3_value_int64 +_sqlite3_value_pointer _sqlite3_value_text _sqlite3_value_type _sqlite3_vfs_find diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index c1c91136ac..dc9db9435f 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -43,7 +43,38 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.xWrap.argAdapter( 'flexible-string', (v)=>xString(util.flexibleString(v)) ); - } + + /** + The 'static-string' argument adapter treats its argument as + either... + + - WASM pointer: assumed to be a long-lived C-string which gets + returned as-is. + + - Anything else: gets coerced to a JS string for use as a map + key. If a matching entry is found (as described next), it is + returned, else wasm.allocCString() is used to create a a new + string, map its pointer to (''+v) for the remainder of the + application's life, and returns that pointer value for this + call and all future calls which are passed a + string-equivalent argument. + + Use case: sqlite3_bind_pointer() and sqlite3_result_pointer() + call for "a static string and preferably a string + literal". This converter is used to ensure that the string + value seen by those functions is long-lived and behaves as they + need it to. + */ + wasm.xWrap.argAdapter( + 'static-string', + function(v){ + if(wasm.isPtr(v)) return v; + v = ''+v; + let rc = this[v]; + return rc || (rc = this[v] = wasm.allocCString(v)); + }.bind(Object.create(null)) + ); + }/* special-case string-type argument conversions */ if(1){// WhWasmUtil.xWrap() bindings... /** diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index b66497e6ba..cf57329569 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -884,6 +884,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_bind_null",undefined, "sqlite3_stmt*", "int"], ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], + ["sqlite3_bind_pointer", "int", + "sqlite3_stmt*", "int", "*", "static-string", "*"], ["sqlite3_bind_text","int", "sqlite3_stmt*", "int", "string", "int", "int" /* We should arguably create a hand-written binding of bind_text() which does more flexible text conversion, along @@ -958,6 +960,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_result_error_toobig", undefined, "sqlite3_context*"], ["sqlite3_result_int",undefined, "sqlite3_context*", "int"], ["sqlite3_result_null",undefined, "sqlite3_context*"], + ["sqlite3_result_pointer",undefined, + "sqlite3_context*", "*", "static-string", "*"], ["sqlite3_result_text",undefined, "sqlite3_context*", "string", "int", "*"], ["sqlite3_result_zeroblob", undefined, "sqlite3_context*", "int"], ["sqlite3_serialize","*", "sqlite3*", "string", "*", "int"], @@ -980,6 +984,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_value_bytes","int", "sqlite3_value*"], ["sqlite3_value_double","f64", "sqlite3_value*"], ["sqlite3_value_int","int", "sqlite3_value*"], + ["sqlite3_value_pointer", "*", "sqlite3_value*", "static-string"], ["sqlite3_value_text", "string", "sqlite3_value*"], ["sqlite3_value_type", "int", "sqlite3_value*"], ["sqlite3_vfs_find", "*", "string"], diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 84d829daba..9bce82f1a9 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -725,11 +725,12 @@ self.WhWasmUtilInstaller = function(target){ Expects ptr to be a pointer into the WASM heap memory which refers to a NUL-terminated C-style string encoded as UTF-8. Returns the length, in bytes, of the string, as for `strlen(3)`. - As a special case, if !ptr then it it returns `null`. Throws if - ptr is out of range for target.heap8u(). + As a special case, if !ptr or if it's not a pointer then it + returns `null`. Throws if ptr is out of range for + target.heap8u(). */ target.cstrlen = function(ptr){ - if(!ptr) return null; + if(!ptr || !target.isPtr(ptr)) return null; const h = heapWrappers().HEAP8U; let pos = ptr; for( ; h[pos] !== 0; ++pos ){} @@ -753,7 +754,7 @@ self.WhWasmUtilInstaller = function(target){ refers to a NUL-terminated C-style string encoded as UTF-8. This function counts its byte length using cstrlen() then returns a JS-format string representing its contents. As a special case, if - ptr is falsy, `null` is returned. + ptr is falsy or not a pointer, `null` is returned. */ target.cstringToJs = function(ptr){ const n = target.cstrlen(ptr); diff --git a/manifest b/manifest index 08e9801bce..cd9e8c1ae2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\swork\son\sthe\sJS\sside\sof\sthe\svirtual\stable\sAPIs. -D 2022-12-05T07:51:25.186 +C Export\ssqlite3_bind/value/result_pointer()\sto\swasm.\sAdd\s'static-string'\sargument\sconverter\sto\ssupport\sthe\slifetime\srequirements\sof\sbind/result_pointer()'s\sstring\sargument.\sCorrect\san\sendless\sloop\sin\swasm.cstrlen()\swhen\spassed\sa\snon-C-string\sargument. +D 2022-12-05T11:30:39.470 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 5b042c543c83134c20d222b7b8cd9e3401a5d552d709364d82461c064cd26089 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 21a7c443f496f891e5362db75884a5c3c72058559bbfbcae707ec5bbefd80de9 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d @@ -503,9 +503,9 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js dd7f3cc427154f280d9718dbc755ae4ed21c81d193f16b32184731c299c1d3be +F ext/wasm/api/sqlite3-api-glue.js 170fd05a8064d2f23c253a45debe533b0c23872f9495c6d4d20463b700252770 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js d2cb6dd5ca109b36c5038301a6881f6ca137b946350e79e701114e461ab56298 +F ext/wasm/api/sqlite3-api-prologue.js d930ddf9c325405c97f9d0367020eac7cfa3da660c098585a11fd2466b1f5e16 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 @@ -521,7 +521,7 @@ F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c07 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/common/testing.css 35889709547d89a6109ff83b25c11bbc91d8dd43aab8722e428655ca98880a06 -F ext/wasm/common/whwasmutil.js 1bc1c973662db7d52763512e67c2369fdbfe9e8bae33069ac89f4fbe40d678b2 +F ext/wasm/common/whwasmutil.js 0de1e72494d52185d518892a3ac95d38b8e295d3699b64ddb36a3d46c11c8346 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6712fbe46a97867cea309f78a274edbb6bd166a505b41e18a580306da0e063db -R 556ab7e7db905d4fcefcfcbe9067fc69 +P cb9881ec001b0e2faf047e57acfd1722d2b546255a54e0f850f568edfe2df1cd +R 43c6939a23c10432b26f996c39515ebf U stephan -Z c9a0ef228c6ee4583769e98c5e79436a +Z e6790963ab3f2e980219be88a41c0b38 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d7b2b1b5f4..0e7dc35e19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb9881ec001b0e2faf047e57acfd1722d2b546255a54e0f850f568edfe2df1cd \ No newline at end of file +a94552434a657376d5ce1831de05c1b15fb153020848cd825fb0df413c3baa70 \ No newline at end of file From cf8f0d20463adf368cedb434d6d70cd255ab185a Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 11:54:13 +0000 Subject: [PATCH 157/282] Rename 'static-string' argument adapter to 'string:static'. Replace JS unit tests which were lost via editing a generated copy of tester1.js instead of the original tester1.c-pp.js input file. FossilOrigin-Name: 9d81d51d5a255b42f8416da850c992a9e4c8eebc940e0702a9262cfcaa6d7b2f --- ext/wasm/api/sqlite3-api-glue.js | 4 ++-- ext/wasm/api/sqlite3-api-prologue.js | 6 +++--- ext/wasm/tester1.c-pp.js | 20 ++++++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 34 insertions(+), 14 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index dc9db9435f..b460093f8d 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -45,7 +45,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ); /** - The 'static-string' argument adapter treats its argument as + The 'string:static' argument adapter treats its argument as either... - WASM pointer: assumed to be a long-lived C-string which gets @@ -66,7 +66,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ need it to. */ wasm.xWrap.argAdapter( - 'static-string', + 'string:static', function(v){ if(wasm.isPtr(v)) return v; v = ''+v; diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index cf57329569..4087adb504 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -885,7 +885,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], ["sqlite3_bind_pointer", "int", - "sqlite3_stmt*", "int", "*", "static-string", "*"], + "sqlite3_stmt*", "int", "*", "string:static", "*"], ["sqlite3_bind_text","int", "sqlite3_stmt*", "int", "string", "int", "int" /* We should arguably create a hand-written binding of bind_text() which does more flexible text conversion, along @@ -961,7 +961,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_result_int",undefined, "sqlite3_context*", "int"], ["sqlite3_result_null",undefined, "sqlite3_context*"], ["sqlite3_result_pointer",undefined, - "sqlite3_context*", "*", "static-string", "*"], + "sqlite3_context*", "*", "string:static", "*"], ["sqlite3_result_text",undefined, "sqlite3_context*", "string", "int", "*"], ["sqlite3_result_zeroblob", undefined, "sqlite3_context*", "int"], ["sqlite3_serialize","*", "sqlite3*", "string", "*", "int"], @@ -984,7 +984,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_value_bytes","int", "sqlite3_value*"], ["sqlite3_value_double","f64", "sqlite3_value*"], ["sqlite3_value_int","int", "sqlite3_value*"], - ["sqlite3_value_pointer", "*", "sqlite3_value*", "static-string"], + ["sqlite3_value_pointer", "*", "sqlite3_value*", "string:static"], ["sqlite3_value_text", "string", "sqlite3_value*"], ["sqlite3_value_type", "int", "sqlite3_value*"], ["sqlite3_vfs_find", "*", "string"], diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 0aaee97693..aa5fa41d2d 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -660,6 +660,26 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert(rc>0 && Number.isFinite(rc)); rc = w.xCallWrapped('sqlite3_wasm_enum_json','utf8'); T.assert('string'===typeof rc).assert(rc.length>300); + + + { // 'string:static' argAdapter() sanity checks... + let argAd = w.xWrap.argAdapter('string:static'); + let p0 = argAd('foo'), p1 = argAd('bar'); + T.assert(w.isPtr(p0) && w.isPtr(p1)) + .assert(p0 !== p1) + .assert(p0 === argAd('foo')) + .assert(p1 === argAd('bar')); + } + + // 'flexible-string' argAdapter() sanity checks... + w.scopedAllocCall(()=>{ + const argAd = w.xWrap.argAdapter('flexible-string'); + const cj = (v)=>w.cstringToJs(argAd(v)); + T.assert('Hi' === cj('Hi')) + .assert('hi' === cj(['h','i'])) + .assert('HI' === cj(new Uint8Array([72, 73]))); + }); + if(haveWasmCTests()){ if(!sqlite3.config.useStdAlloc){ fw = w.xWrap('sqlite3_wasm_test_str_hello', 'utf8:dealloc',['i32']); diff --git a/manifest b/manifest index cd9e8c1ae2..63d8142b60 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Export\ssqlite3_bind/value/result_pointer()\sto\swasm.\sAdd\s'static-string'\sargument\sconverter\sto\ssupport\sthe\slifetime\srequirements\sof\sbind/result_pointer()'s\sstring\sargument.\sCorrect\san\sendless\sloop\sin\swasm.cstrlen()\swhen\spassed\sa\snon-C-string\sargument. -D 2022-12-05T11:30:39.470 +C Rename\s'static-string'\sargument\sadapter\sto\s'string:static'.\sReplace\sJS\sunit\stests\swhich\swere\slost\svia\sediting\sa\sgenerated\scopy\sof\stester1.js\sinstead\sof\sthe\soriginal\stester1.c-pp.js\sinput\sfile. +D 2022-12-05T11:54:13.530 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,9 +503,9 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js 170fd05a8064d2f23c253a45debe533b0c23872f9495c6d4d20463b700252770 +F ext/wasm/api/sqlite3-api-glue.js 2c45a3c05badb11c37152d9d819156420a5c026ef9cf6fee97512b279b33bf44 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js d930ddf9c325405c97f9d0367020eac7cfa3da660c098585a11fd2466b1f5e16 +F ext/wasm/api/sqlite3-api-prologue.js 0dba5a99f795ce6bb88b621960cc3f5f3e6370ee6f16230a0b165136529fcf40 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 29b1d87f7d51f70d61645719fee657f3787fe939bb695f27034c75404e8f1e6f F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js d096a8fadfd27caa680a4311b1d529551f8fe885a63dd27457c87b6008c64632 +F ext/wasm/tester1.c-pp.js d03c8c50063ec82d38694db077fc75af390c8baa0f72eb52d3c4e4fde68ee95e F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb9881ec001b0e2faf047e57acfd1722d2b546255a54e0f850f568edfe2df1cd -R 43c6939a23c10432b26f996c39515ebf +P a94552434a657376d5ce1831de05c1b15fb153020848cd825fb0df413c3baa70 +R 3e5f8074bc013b334ba01f938431dc3c U stephan -Z e6790963ab3f2e980219be88a41c0b38 +Z 6b18e44631fce1838526a30c03df5312 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0e7dc35e19..5b3f6a2a2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a94552434a657376d5ce1831de05c1b15fb153020848cd825fb0df413c3baa70 \ No newline at end of file +9d81d51d5a255b42f8416da850c992a9e4c8eebc940e0702a9262cfcaa6d7b2f \ No newline at end of file From 8c696375b556cee3c123aba5aad2a497fa0be7e4 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 13:07:06 +0000 Subject: [PATCH 158/282] Remove SQLITE_EXPERIMENTAL tag from sqlite3_vtab_collation() and fix a related doc typo. FossilOrigin-Name: 35d670b3593a46e13ded541ef477fa35dac6bcb9c31e6cf4b90bd7fa550a8ee2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 32b47a08bf..9dc543142a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sobsolete\s"experimental"\scomments\son\sthe\svirtual-table\sinterface\ndocumentation\sin\ssqlite.h.in. -D 2022-12-05T10:50:06.702 +C Remove\sSQLITE_EXPERIMENTAL\stag\sfrom\ssqlite3_vtab_collation()\sand\sfix\sa\srelated\sdoc\stypo. +D 2022-12-05T13:07:06.892 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -647,7 +647,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 F src/shell.c.in f6ab148f150dc0c8460be74a61566d37c65d43311e84963cc1a58df3fc277511 -F src/sqlite.h.in 319be18909674dbab5d0c5372aec098671a35f8ab611cdfc41108df86ab8472b +F src/sqlite.h.in 14d1273e84a8a4d7cbfc044592c4f97ea04ecf59b2a684f3c7c1b2ab9e48ae0e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f F src/sqliteInt.h 4ddd98e423276714479f9f22dbbda050e8ef99aa97e7e26bf0bdf58acef0ca42 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1b779afa3ed2f35a110e460fc6ed13cba744db85b9924149ab028b100d1e1e12 -R 76fd9f81f55a4334f31e568ee103dfee -U drh -Z 3d486c52105de074dd9d6853e636a4e9 +P 28fc32abdcb1c6af19f0926a3a7317e15930a0ba89645d5ffbf6ce99d8463273 +R fd4f685ed0d35736bb0b09c84b60e02d +U stephan +Z 52cef66f712e32342d30b9b660ac21ec # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c40d99deaf..c0d4304ce2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28fc32abdcb1c6af19f0926a3a7317e15930a0ba89645d5ffbf6ce99d8463273 \ No newline at end of file +35d670b3593a46e13ded541ef477fa35dac6bcb9c31e6cf4b90bd7fa550a8ee2 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1b9d71e139..ba9aa5a933 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7255,7 +7255,7 @@ struct sqlite3_index_info { ** the [sqlite3_vtab_collation()] interface. For most real-world virtual ** tables, the collating sequence of constraints does not matter (for example ** because the constraints are numeric) and so the sqlite3_vtab_collation() -** interface is no commonly needed. +** interface is not commonly needed. */ #define SQLITE_INDEX_CONSTRAINT_EQ 2 #define SQLITE_INDEX_CONSTRAINT_GT 4 @@ -9617,7 +9617,7 @@ int sqlite3_vtab_nochange(sqlite3_context*); **

  • Otherwise, "BINARY" is returned. ** */ -SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); +const char *sqlite3_vtab_collation(sqlite3_index_info*,int); /* ** CAPI3REF: Determine if a virtual table query is DISTINCT From 9a49a97487c93708a0b720aae39b4123e9a02715 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 13:12:48 +0000 Subject: [PATCH 159/282] Export sqlite3_vtab_collation() to wasm. Rename 'flexible-string' JS argument adapter to 'string:flexible' for consistency. FossilOrigin-Name: 15f8042fddaeabab43dd187c463d3ccc56758cbf19bf2ca4837d9087a4850c1a --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 1 + ext/wasm/api/sqlite3-api-glue.js | 46 ++++++++++----------- ext/wasm/api/sqlite3-api-oo1.js | 14 +++---- ext/wasm/api/sqlite3-api-prologue.js | 8 ++-- ext/wasm/tester1.c-pp.js | 4 +- manifest | 20 ++++----- manifest.uuid | 2 +- 7 files changed, 48 insertions(+), 47 deletions(-) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index 5c0419e9da..f5aabfe079 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -103,6 +103,7 @@ _sqlite3_value_type _sqlite3_vfs_find _sqlite3_vfs_register _sqlite3_vfs_unregister +_sqlite3_vtab_collation _sqlite3_vtab_distinct _sqlite3_vtab_in _sqlite3_vtab_in_first diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index b460093f8d..ddfc1daf43 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -38,10 +38,10 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ delete self.Jaccwabyt; {/* Convert Arrays and certain TypedArrays to strings for - 'flexible-string'-type arguments */ + 'string:flexible'-type arguments */ const xString = wasm.xWrap.argAdapter('string'); wasm.xWrap.argAdapter( - 'flexible-string', (v)=>xString(util.flexibleString(v)) + 'string:flexible', (v)=>xString(util.flexibleString(v)) ); /** @@ -176,29 +176,9 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ (1===n?"":'s')+"."); }; - /** - Helper for flexible-string conversions which require a - byte-length counterpart argument. Passed a value and its - ostensible length, this function returns [V,N], where V - is either v or a transformed copy of v and N is either n, - -1, or the byte length of v (if it's a byte array). - */ - const __flexiString = function(v,n){ - if('string'===typeof v){ - n = -1; - }else if(util.isSQLableTypedArray(v)){ - n = v.byteLength; - v = util.typedArrayToString(v); - }else if(Array.isArray(v)){ - v = v.join(""); - n = -1; - } - return [v, n]; - }; - if(1){/* Special-case handling of sqlite3_exec() */ const __exec = wasm.xWrap("sqlite3_exec", "int", - ["sqlite3*", "flexible-string", "*", "*", "**"]); + ["sqlite3*", "string:flexible", "*", "*", "**"]); /* Documented in the api object's initializer. */ capi.sqlite3_exec = function f(pDb, sql, callback, pVoid, pErrMsg){ if(f.length!==arguments.length){ @@ -549,6 +529,26 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(1){/* Special-case handling of sqlite3_prepare_v2() and sqlite3_prepare_v3() */ + /** + Helper for string:flexible conversions which require a + byte-length counterpart argument. Passed a value and its + ostensible length, this function returns [V,N], where V + is either v or a transformed copy of v and N is either n, + -1, or the byte length of v (if it's a byte array). + */ + const __flexiString = (v,n)=>{ + if('string'===typeof v){ + n = -1; + }else if(util.isSQLableTypedArray(v)){ + n = v.byteLength; + v = util.typedArrayToString(v); + }else if(Array.isArray(v)){ + v = v.join(""); + n = -1; + } + return [v, n]; + }; + /** Scope-local holder of the two impls of sqlite3_prepare_v2/v3(). */ diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index e077b0c506..b377efc242 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -192,13 +192,13 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Sets SQL which should be exec()'d on a DB instance after it is opened with the given VFS pointer. The SQL may be any type - supported by the "flexible-string" function argument - conversion. Alternately, the 2nd argument may be a function, in - which case it is called with (theOo1DbObject,sqlite3Namespace) at - the end of the DB() constructor. The function must throw on - error, in which case the db is closed and the exception is - propagated. This function is intended only for use by DB - subclasses or sqlite3_vfs implementations. + supported by the "string:flexible" function argument conversion. + Alternately, the 2nd argument may be a function, in which case it + is called with (theOo1DbObject,sqlite3Namespace) at the end of + the DB() constructor. The function must throw on error, in which + case the db is closed and the exception is propagated. This + function is intended only for use by DB subclasses or sqlite3_vfs + implementations. */ dbCtorHelper.setVfsPostOpenSql = function(pVfs, sql){ __vfsPostOpenSql[pVfs] = sql; diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 4087adb504..5f456edf27 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -602,7 +602,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( If the callback is not a JS function then this binding performs no translation of the callback, but the sql argument is still converted to a WASM string for the call using the - "flexible-string" argument converter. + "string:flexible" argument converter. */ sqlite3_exec: (pDb, sql, callback, pVoid, pErrMsg)=>{}/*installed later*/, @@ -910,7 +910,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], ["sqlite3_compileoption_get", "string", "int"], ["sqlite3_compileoption_used", "int", "string"], - ["sqlite3_complete", "int", "flexible-string"], + ["sqlite3_complete", "int", "string:flexible"], /* sqlite3_create_function(), sqlite3_create_function_v2(), and sqlite3_create_window_function() use hand-written bindings to simplify handling of their function-type arguments. */ @@ -1012,7 +1012,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3*","string","sqlite3_module*","*"]], ["sqlite3_create_module_v2", "int", ["sqlite3*","string","sqlite3_module*","*","*"]], - ["sqlite3_declare_vtab", "int", ["sqlite3*", "flexible-string"]], + ["sqlite3_declare_vtab", "int", ["sqlite3*", "string:flexible"]], ["sqlite3_drop_modules", "int", ["sqlite3*", "**"]], ["sqlite3_malloc64", "*","i64"], ["sqlite3_msize", "i64", "*"], @@ -1023,7 +1023,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_total_changes64", "i64", ["sqlite3*"]], ["sqlite3_uri_int64", "i64", ["sqlite3_filename", "string", "i64"]], ["sqlite3_value_int64","i64", "sqlite3_value*"], - //EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int) + ["sqlite3_vtab_collation","string","sqlite3_index_info*","int"], ["sqlite3_vtab_distinct","int", "sqlite3_index_info*"], ["sqlite3_vtab_in","int", "sqlite3_index_info*", "int", "int"], ["sqlite3_vtab_in_first", "int", "sqlite3_value*", "**"], diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index aa5fa41d2d..2d1b634e55 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -671,9 +671,9 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(p1 === argAd('bar')); } - // 'flexible-string' argAdapter() sanity checks... + // 'string:flexible' argAdapter() sanity checks... w.scopedAllocCall(()=>{ - const argAd = w.xWrap.argAdapter('flexible-string'); + const argAd = w.xWrap.argAdapter('string:flexible'); const cj = (v)=>w.cstringToJs(argAd(v)); T.assert('Hi' === cj('Hi')) .assert('hi' === cj(['h','i'])) diff --git a/manifest b/manifest index 63d8142b60..3084b5ab0b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\s'static-string'\sargument\sadapter\sto\s'string:static'.\sReplace\sJS\sunit\stests\swhich\swere\slost\svia\sediting\sa\sgenerated\scopy\sof\stester1.js\sinstead\sof\sthe\soriginal\stester1.c-pp.js\sinput\sfile. -D 2022-12-05T11:54:13.530 +C Export\ssqlite3_vtab_collation()\sto\swasm.\sRename\s'flexible-string'\sJS\sargument\sadapter\sto\s'string:flexible'\sfor\sconsistency. +D 2022-12-05T13:12:48.698 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 21a7c443f496f891e5362db75884a5c3c72058559bbfbcae707ec5bbefd80de9 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api e17dc504c135e0d9585282faa82bf8ef3c45465b086fb02fb5605e821efb5bbc F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d @@ -503,9 +503,9 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js 2c45a3c05badb11c37152d9d819156420a5c026ef9cf6fee97512b279b33bf44 -F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js 0dba5a99f795ce6bb88b621960cc3f5f3e6370ee6f16230a0b165136529fcf40 +F ext/wasm/api/sqlite3-api-glue.js e3422737c64bb6d871ddff7f7c85e637cc1597edd3bc49ef537578c57d6cfaa9 +F ext/wasm/api/sqlite3-api-oo1.js 793883953d4024e7b8c5ee1c7a6cb49c18ca53a1d235a203f93746f8907d32ba +F ext/wasm/api/sqlite3-api-prologue.js 6f5aee50867dae4e0885dd1bd1913f1f42f433def15ca9309dc5b3e83ab5c667 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html 29b1d87f7d51f70d61645719fee657f3787fe939bb695f27034c75404e8f1e6f F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js d03c8c50063ec82d38694db077fc75af390c8baa0f72eb52d3c4e4fde68ee95e +F ext/wasm/tester1.c-pp.js 419a2fd31b0230e0495ed38a56dbe1e2bbc93c5953b796de43e2a70901970b83 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a94552434a657376d5ce1831de05c1b15fb153020848cd825fb0df413c3baa70 -R 3e5f8074bc013b334ba01f938431dc3c +P 9d81d51d5a255b42f8416da850c992a9e4c8eebc940e0702a9262cfcaa6d7b2f +R 277425a2c74bda828b2402faf303a6b1 U stephan -Z 6b18e44631fce1838526a30c03df5312 +Z c0b7c94a232e1e96230a439e5bd96242 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5b3f6a2a2a..43725293de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d81d51d5a255b42f8416da850c992a9e4c8eebc940e0702a9262cfcaa6d7b2f \ No newline at end of file +15f8042fddaeabab43dd187c463d3ccc56758cbf19bf2ca4837d9087a4850c1a \ No newline at end of file From e0818715b7fdadfba5bb23472048987b339b9f15 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 13:33:42 +0000 Subject: [PATCH 160/282] Export collation-related APIs and strncmp()/strnicmp() to wasm. FossilOrigin-Name: c3c56d9b944fd0d806d8dad9f0c7be3d7a5441765310908872cc525d82ab6a33 --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 5 +++++ ext/wasm/api/sqlite3-api-glue.js | 4 ++-- ext/wasm/api/sqlite3-api-prologue.js | 11 ++++++++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index f5aabfe079..59cca49c8d 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -14,6 +14,7 @@ _sqlite3_changes _sqlite3_changes64 _sqlite3_clear_bindings _sqlite3_close_v2 +_sqlite3_collation_needed _sqlite3_column_blob _sqlite3_column_bytes _sqlite3_column_count @@ -27,6 +28,8 @@ _sqlite3_column_type _sqlite3_compileoption_get _sqlite3_compileoption_used _sqlite3_complete +_sqlite3_create_collation +_sqlite3_create_collation_v2 _sqlite3_create_function _sqlite3_create_function_v2 _sqlite3_create_module @@ -83,7 +86,9 @@ _sqlite3_sourceid _sqlite3_sql _sqlite3_step _sqlite3_strglob +_sqlite3_stricmp _sqlite3_strlike +_sqlite3_strnicmp _sqlite3_total_changes _sqlite3_total_changes64 _sqlite3_trace_v2 diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index ddfc1daf43..0516953f10 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -579,7 +579,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ "int", ["sqlite3*", "*", "int", "int", "**", "**"]); - /* Documented in the api object's initializer. */ + /* Documented in the capi object's initializer. */ capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ if(f.length!==arguments.length){ return __dbArgcMismatch(pDb,"sqlite3_prepare_v3",f.length); @@ -596,7 +596,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; - /* Documented in the api object's initializer. */ + /* Documented in the capi object's initializer. */ capi.sqlite3_prepare_v2 = function f(pDb, sql, sqlLen, ppStmt, pzTail){ return (f.length===arguments.length) ? capi.sqlite3_prepare_v3(pDb, sql, sqlLen, 0, ppStmt, pzTail) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 5f456edf27..5632569d72 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -900,6 +900,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_close_v2", "int", "sqlite3*"], ["sqlite3_changes", "int", "sqlite3*"], ["sqlite3_clear_bindings","int", "sqlite3_stmt*"], + ["sqlite3_collation_needed", "int", "sqlite3*", "*", "*"/*=>v(ppis)*/], ["sqlite3_column_blob","*", "sqlite3_stmt*", "int"], ["sqlite3_column_bytes","int", "sqlite3_stmt*", "int"], ["sqlite3_column_count", "int", "sqlite3_stmt*"], @@ -914,6 +915,12 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /* sqlite3_create_function(), sqlite3_create_function_v2(), and sqlite3_create_window_function() use hand-written bindings to simplify handling of their function-type arguments. */ + ["sqlite3_create_collation", "int", + "sqlite3*", "string", "int"/*SQLITE_UTF8 is the only legal value*/, + "*", "*"], + ["sqlite3_create_collation_v2", "int", + "sqlite3*", "string", "int"/*SQLITE_UTF8 is the only legal value*/, + "*", "*", "*"], ["sqlite3_data_count", "int", "sqlite3_stmt*"], ["sqlite3_db_filename", "string", "sqlite3*", "string"], ["sqlite3_db_handle", "sqlite3*", "sqlite3_stmt*"], @@ -970,7 +977,9 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_sql", "string", "sqlite3_stmt*"], ["sqlite3_step", "int", "sqlite3_stmt*"], ["sqlite3_strglob", "int", "string","string"], - ["sqlite3_strlike", "int", "string","string","int"], + ["sqlite3_stricmp", "int", "string", "string"], + ["sqlite3_strlike", "int", "string", "string","int"], + ["sqlite3_strnicmp", "int", "string", "string", "int"], ["sqlite3_trace_v2", "int", "sqlite3*", "int", "*", "*"], ["sqlite3_total_changes", "int", "sqlite3*"], /* Note sqlite3_uri_...() has very specific requirements diff --git a/manifest b/manifest index 3084b5ab0b..2b59fffc13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Export\ssqlite3_vtab_collation()\sto\swasm.\sRename\s'flexible-string'\sJS\sargument\sadapter\sto\s'string:flexible'\sfor\sconsistency. -D 2022-12-05T13:12:48.698 +C Export\scollation-related\sAPIs\sand\sstrncmp()/strnicmp()\sto\swasm. +D 2022-12-05T13:33:42.870 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api e17dc504c135e0d9585282faa82bf8ef3c45465b086fb02fb5605e821efb5bbc +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 65456c98b3433b6b34b1253885ccf9a59146a8fb1b5d41a8638e86b5d1f87d55 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d @@ -503,9 +503,9 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js e3422737c64bb6d871ddff7f7c85e637cc1597edd3bc49ef537578c57d6cfaa9 +F ext/wasm/api/sqlite3-api-glue.js c3a11e1d0e6fd381f68f9e76ad01f3616a6b809fbf9f5aa8e323955c128a6811 F ext/wasm/api/sqlite3-api-oo1.js 793883953d4024e7b8c5ee1c7a6cb49c18ca53a1d235a203f93746f8907d32ba -F ext/wasm/api/sqlite3-api-prologue.js 6f5aee50867dae4e0885dd1bd1913f1f42f433def15ca9309dc5b3e83ab5c667 +F ext/wasm/api/sqlite3-api-prologue.js 92dd2dd3efedfbe9f75d2663ed6af1164ace93bf64b20204cb12db13c08affec F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9d81d51d5a255b42f8416da850c992a9e4c8eebc940e0702a9262cfcaa6d7b2f -R 277425a2c74bda828b2402faf303a6b1 +P 15f8042fddaeabab43dd187c463d3ccc56758cbf19bf2ca4837d9087a4850c1a +R 017b8d9d1b435c1f731acc879040c83a U stephan -Z c0b7c94a232e1e96230a439e5bd96242 +Z 3190520e942052577f9152539aa749d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 43725293de..27930bfe58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15f8042fddaeabab43dd187c463d3ccc56758cbf19bf2ca4837d9087a4850c1a \ No newline at end of file +c3c56d9b944fd0d806d8dad9f0c7be3d7a5441765310908872cc525d82ab6a33 \ No newline at end of file From 27ef91b551add7170e57ff371892679b3b0fb209 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Dec 2022 14:12:14 +0000 Subject: [PATCH 161/282] Fix a problem in the memdb vfs xLock() function allowing clients to upgrade to EXCLUSIVE locks when other connections are holding SHARED. FossilOrigin-Name: 15f0be8a640e7bfa4130edd4650a745337bd96083b119a1553f9abf9ff066806 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/memdb.c | 13 +++++++++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9dc543142a..3bbac5f1c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sSQLITE_EXPERIMENTAL\stag\sfrom\ssqlite3_vtab_collation()\sand\sfix\sa\srelated\sdoc\stypo. -D 2022-12-05T13:07:06.892 +C Fix\sa\sproblem\sin\sthe\smemdb\svfs\sxLock()\sfunction\sallowing\sclients\sto\supgrade\sto\sEXCLUSIVE\slocks\swhen\sother\sconnections\sare\sholding\sSHARED. +D 2022-12-05T14:12:14.522 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -615,7 +615,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c 5a3dbd8ac8a6501152a4fc1fcae9b0900c2d7eb0589c4ec7456fdde15725a26c -F src/memdb.c a42248bc04dc1f80da3511618a78956ed77b65058b73e251bde39061204e9f97 +F src/memdb.c fa280078fb48c4bb7ef47e361cd958938a1a3c46a9a45f6622da6efcb57cc055 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 28fc32abdcb1c6af19f0926a3a7317e15930a0ba89645d5ffbf6ce99d8463273 -R fd4f685ed0d35736bb0b09c84b60e02d -U stephan -Z 52cef66f712e32342d30b9b660ac21ec +P 35d670b3593a46e13ded541ef477fa35dac6bcb9c31e6cf4b90bd7fa550a8ee2 +R ce00ca3fa8b85afb0cb81cb19f7a317f +U dan +Z 91e55fba552fdd00d9614ef94c1303dc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0d4304ce2..6a051aba6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35d670b3593a46e13ded541ef477fa35dac6bcb9c31e6cf4b90bd7fa550a8ee2 \ No newline at end of file +15f0be8a640e7bfa4130edd4650a745337bd96083b119a1553f9abf9ff066806 \ No newline at end of file diff --git a/src/memdb.c b/src/memdb.c index 0aecb71022..7485f51ffb 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -371,9 +371,22 @@ static int memdbLock(sqlite3_file *pFile, int eLock){ if( eLock==pThis->eLock ) return SQLITE_OK; memdbEnter(p); if( eLock>SQLITE_LOCK_SHARED ){ + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){ rc = SQLITE_READONLY; + }else if( eLock==SQLITE_LOCK_EXCLUSIVE ){ + /* Taking an EXCLUSIVE lock. Fail if we only have SHARED and any + ** other client has any kind of write-lock. Also fail if any other + ** client is holding read-lock. */ + if( pThis->eLock<=SQLITE_LOCK_SHARED && p->nWrLock ){ + rc = SQLITE_BUSY; + }else if( p->nRdLock>1 ){ + rc = SQLITE_BUSY; + } + p->nWrLock = 1; }else if( pThis->eLock<=SQLITE_LOCK_SHARED ){ + /* Upgrading to RESERVED or PENDING from SHARED. Fail if any other + ** client has a write-lock of any kind. */ if( p->nWrLock ){ rc = SQLITE_BUSY; }else{ From e1d25177c2acf7c3584c29ba9979427c4ac4060e Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 14:13:55 +0000 Subject: [PATCH 162/282] Expose sqlite3_get/set_auxdata() to wasm. Minor test app CSS tweaks. FossilOrigin-Name: 44659ad32a9fe6363badfc5dbb0bd51d6fb2ee1c8aa47b71e9cf3dbd631fde9e --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 2 ++ ext/wasm/api/sqlite3-api-prologue.js | 22 +++++++++++++-------- ext/wasm/common/emscripten.css | 2 +- ext/wasm/common/testing.css | 6 ++++++ ext/wasm/tester1-worker.html | 6 +----- ext/wasm/tester1.c-pp.html | 9 ++------- manifest | 22 ++++++++++----------- manifest.uuid | 2 +- 8 files changed, 38 insertions(+), 33 deletions(-) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index 59cca49c8d..523488cf08 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -52,6 +52,7 @@ _sqlite3_extended_result_codes _sqlite3_file_control _sqlite3_finalize _sqlite3_free +_sqlite3_get_auxdata _sqlite3_initialize _sqlite3_libversion _sqlite3_libversion_number @@ -81,6 +82,7 @@ _sqlite3_result_text _sqlite3_result_zeroblob _sqlite3_result_zeroblob64 _sqlite3_serialize +_sqlite3_set_auxdata _sqlite3_shutdown _sqlite3_sourceid _sqlite3_sql diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 5632569d72..a51e957f8f 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -942,6 +942,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_file_control", "int", "sqlite3*", "string", "int", "*"], ["sqlite3_finalize", "int", "sqlite3_stmt*"], ["sqlite3_free", undefined,"*"], + ["sqlite3_get_auxdata", "*", "sqlite3_context*", "int"], ["sqlite3_initialize", undefined], /*["sqlite3_interrupt", undefined, "sqlite3*" ^^^ we cannot actually currently support this because JS is @@ -959,19 +960,20 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( the range of supported argument types. */ ["sqlite3_realloc", "*","*","int"], ["sqlite3_reset", "int", "sqlite3_stmt*"], - ["sqlite3_result_blob",undefined, "sqlite3_context*", "*", "int", "*"], - ["sqlite3_result_double",undefined, "sqlite3_context*", "f64"], - ["sqlite3_result_error",undefined, "sqlite3_context*", "string", "int"], + ["sqlite3_result_blob", undefined, "sqlite3_context*", "*", "int", "*"], + ["sqlite3_result_double", undefined, "sqlite3_context*", "f64"], + ["sqlite3_result_error", undefined, "sqlite3_context*", "string", "int"], ["sqlite3_result_error_code", undefined, "sqlite3_context*", "int"], ["sqlite3_result_error_nomem", undefined, "sqlite3_context*"], ["sqlite3_result_error_toobig", undefined, "sqlite3_context*"], - ["sqlite3_result_int",undefined, "sqlite3_context*", "int"], - ["sqlite3_result_null",undefined, "sqlite3_context*"], - ["sqlite3_result_pointer",undefined, + ["sqlite3_result_int", undefined, "sqlite3_context*", "int"], + ["sqlite3_result_null", undefined, "sqlite3_context*"], + ["sqlite3_result_pointer", undefined, "sqlite3_context*", "*", "string:static", "*"], - ["sqlite3_result_text",undefined, "sqlite3_context*", "string", "int", "*"], + ["sqlite3_result_text", undefined, "sqlite3_context*", "string", "int", "*"], ["sqlite3_result_zeroblob", undefined, "sqlite3_context*", "int"], ["sqlite3_serialize","*", "sqlite3*", "string", "*", "int"], + ["sqlite3_set_auxdata", undefined, "sqlite3_context*", "int", "*", "*"/* => v(*) */], ["sqlite3_shutdown", undefined], ["sqlite3_sourceid", "string"], ["sqlite3_sql", "string", "sqlite3_stmt*"], @@ -1012,6 +1014,10 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( Functions which require BigInt (int64) support are separated from the others because we need to conditionally bind them or apply dummy impls, depending on the capabilities of the environment. + + Note that not all of these functions directly require int64 + but are only for use with APIs which require int64. For example, + the vtab-related functions. */ wasm.bindingSignatures.int64 = [ ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], @@ -1027,7 +1033,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_msize", "i64", "*"], ["sqlite3_overload_function", "int", ["sqlite3*","string","int"]], ["sqlite3_realloc64", "*","*", "i64"], - ["sqlite3_result_int64",undefined, "*", "i64"], + ["sqlite3_result_int64", undefined, "*", "i64"], ["sqlite3_result_zeroblob64", "int", "*", "i64"], ["sqlite3_total_changes64", "i64", ["sqlite3*"]], ["sqlite3_uri_int64", "i64", ["sqlite3_filename", "string", "i64"]], diff --git a/ext/wasm/common/emscripten.css b/ext/wasm/common/emscripten.css index 7e3dc811d0..d8f82c73b2 100644 --- a/ext/wasm/common/emscripten.css +++ b/ext/wasm/common/emscripten.css @@ -1,4 +1,4 @@ -/* emcscript-related styling, used during the module load/intialization processes... */ +/* emscripten-related styling, used during the module load/intialization processes... */ .emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; } div.emscripten { text-align: center; } div.emscripten_border { border: 1px solid black; } diff --git a/ext/wasm/common/testing.css b/ext/wasm/common/testing.css index 9438b330c9..fb44f1d612 100644 --- a/ext/wasm/common/testing.css +++ b/ext/wasm/common/testing.css @@ -61,3 +61,9 @@ span.labeled-input { flex-direction: column-reverse; } label[for] { cursor: pointer } + +h1 { + border-radius: 0.25em; + padding: 0.15em 0.25em; +} +h1:first-of-type {margin: 0 0 0.5em 0;} diff --git a/ext/wasm/tester1-worker.html b/ext/wasm/tester1-worker.html index eba3fdeb4e..a9c12d72c5 100644 --- a/ext/wasm/tester1-worker.html +++ b/ext/wasm/tester1-worker.html @@ -7,11 +7,7 @@ sqlite3 tester #1: Worker thread - +

    sqlite3 tester #1: Worker thread

    diff --git a/ext/wasm/tester1.c-pp.html b/ext/wasm/tester1.c-pp.html index b1b68e486e..535e58962d 100644 --- a/ext/wasm/tester1.c-pp.html +++ b/ext/wasm/tester1.c-pp.html @@ -13,14 +13,9 @@ ES6 Module in UI thread UI thread //#endif - + - -

    +

    See tester1-worker.html for the Worker-thread variant.
    diff --git a/manifest b/manifest index 2b59fffc13..8345aa2237 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Export\scollation-related\sAPIs\sand\sstrncmp()/strnicmp()\sto\swasm. -D 2022-12-05T13:33:42.870 +C Expose\ssqlite3_get/set_auxdata()\sto\swasm.\sMinor\stest\sapp\sCSS\stweaks. +D 2022-12-05T14:13:55.858 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 65456c98b3433b6b34b1253885ccf9a59146a8fb1b5d41a8638e86b5d1f87d55 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api ffa70413409e922ce0f761779787a1d9100b34b43c8e3106bb7ccf2786a41326 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d @@ -505,7 +505,7 @@ F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a9578430388 F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js c3a11e1d0e6fd381f68f9e76ad01f3616a6b809fbf9f5aa8e323955c128a6811 F ext/wasm/api/sqlite3-api-oo1.js 793883953d4024e7b8c5ee1c7a6cb49c18ca53a1d235a203f93746f8907d32ba -F ext/wasm/api/sqlite3-api-prologue.js 92dd2dd3efedfbe9f75d2663ed6af1164ace93bf64b20204cb12db13c08affec +F ext/wasm/api/sqlite3-api-prologue.js 815fef5ee93e1bb11ebec5a1d6a1b8ae2e47cfeb66dc5f6e93380ccce045f194 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 @@ -519,8 +519,8 @@ F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d F ext/wasm/batch-runner.js 49609e89aaac9989d6c1ad3fae268e4878e1ad7bc5fd3e5c2f44959660780b2e F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c0779 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b -F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/common/testing.css 35889709547d89a6109ff83b25c11bbc91d8dd43aab8722e428655ca98880a06 +F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 +F ext/wasm/common/testing.css 0ff15602a3ab2bad8aef2c3bd120c7ee3fd1c2054ad2ace7e214187ae68d926f F ext/wasm/common/whwasmutil.js 0de1e72494d52185d518892a3ac95d38b8e295d3699b64ddb36a3d46c11c8346 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 @@ -553,8 +553,8 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac -F ext/wasm/tester1-worker.html 29b1d87f7d51f70d61645719fee657f3787fe939bb695f27034c75404e8f1e6f -F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 +F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 +F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 F ext/wasm/tester1.c-pp.js 419a2fd31b0230e0495ed38a56dbe1e2bbc93c5953b796de43e2a70901970b83 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 15f8042fddaeabab43dd187c463d3ccc56758cbf19bf2ca4837d9087a4850c1a -R 017b8d9d1b435c1f731acc879040c83a +P c3c56d9b944fd0d806d8dad9f0c7be3d7a5441765310908872cc525d82ab6a33 +R 585ea2dfb36f1f8d84b5d643e3dc1f7a U stephan -Z 3190520e942052577f9152539aa749d6 +Z 583c0236badf6d39ab9de3ab68428681 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 27930bfe58..3aea11316b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3c56d9b944fd0d806d8dad9f0c7be3d7a5441765310908872cc525d82ab6a33 \ No newline at end of file +44659ad32a9fe6363badfc5dbb0bd51d6fb2ee1c8aa47b71e9cf3dbd631fde9e \ No newline at end of file From 49d402684b86e0f49264b5fbbe1d0ca2e7f64b93 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Dec 2022 14:20:54 +0000 Subject: [PATCH 163/282] Add test case that should have been part of previous commit. FossilOrigin-Name: dc7dd2d3e50e7cc474b22f1b5b219da32bcd7aa1ba56864d1dbcf0d3a6fa06f2 --- manifest | 11 ++++---- manifest.uuid | 2 +- test/memdb2.test | 66 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 test/memdb2.test diff --git a/manifest b/manifest index 3bbac5f1c0..66f7ed61f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\smemdb\svfs\sxLock()\sfunction\sallowing\sclients\sto\supgrade\sto\sEXCLUSIVE\slocks\swhen\sother\sconnections\sare\sholding\sSHARED. -D 2022-12-05T14:12:14.522 +C Add\stest\scase\sthat\sshould\shave\sbeen\spart\sof\sprevious\scommit. +D 2022-12-05T14:20:54.124 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1317,6 +1317,7 @@ F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 F test/memdb1.test 2c4e9cc10d21c6bf4e217d72b7f6b8ba9b2605971bb2c5e6df76018e189f98f5 +F test/memdb2.test d1dc6058ee59f78c7f46f85e069bb974a981920f8c499f0167690d16c0b079f6 F test/memjournal.test 70f3a00c7f84ee2978ad14e831231caa1e7f23915a2c54b4f775a021d5740c6c F test/memjournal2.test 6b9083cfaab9a3281ec545c3da2487999e8025fb7501bbae10f713f80c56454c F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 @@ -2065,8 +2066,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35d670b3593a46e13ded541ef477fa35dac6bcb9c31e6cf4b90bd7fa550a8ee2 -R ce00ca3fa8b85afb0cb81cb19f7a317f +P 15f0be8a640e7bfa4130edd4650a745337bd96083b119a1553f9abf9ff066806 +R 0d5d7f5190e8e855ad343ae2d8e33bd7 U dan -Z 91e55fba552fdd00d9614ef94c1303dc +Z 56bab95316a0c053d879db2cb26ee05d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6a051aba6e..6117ebbdad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15f0be8a640e7bfa4130edd4650a745337bd96083b119a1553f9abf9ff066806 \ No newline at end of file +dc7dd2d3e50e7cc474b22f1b5b219da32bcd7aa1ba56864d1dbcf0d3a6fa06f2 \ No newline at end of file diff --git a/test/memdb2.test b/test/memdb2.test new file mode 100644 index 0000000000..91843aa191 --- /dev/null +++ b/test/memdb2.test @@ -0,0 +1,66 @@ +# 2022-12-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is the "memdb" VFS +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix memdb1 +do_not_use_codec + +ifcapable !deserialize { + finish_test + return +} + +db close + +#------------------------------------------------------------------------- +# Test that when using a memdb database, it is not possible to upgrade +# to an EXCLUSIVE lock if some other client is holding SHARED. +# +sqlite3 db file:/test.db?vfs=memdb -uri 1 +sqlite3 db2 file:/test.db?vfs=memdb -uri 1 + +do_execsql_test 1.1 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); +} + +do_execsql_test -db db2 1.2 { + BEGIN; + SELECT * FROM t1; +} {1 2} + +do_execsql_test 1.3 { + BEGIN; + INSERT INTO t1 VALUES(3, 4); +} + +do_catchsql_test 1.4 { + COMMIT +} {1 {database is locked}} + +do_execsql_test -db db2 1.5 { + SELECT * FROM t1; + END; +} {1 2} + +do_execsql_test 1.6 { + COMMIT +} {} + +do_execsql_test -db db2 1.7 { + SELECT * FROM t1 +} {1 2 3 4} + +finish_test From cfb66014bc067ea38f03728972153753a258c348 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 14:32:35 +0000 Subject: [PATCH 164/282] Jaccwabyt (JS) doc updates. FossilOrigin-Name: a329a809b5da135a9c251e4d5f637d45d01d0248110ac05f2ad8f01d9df38c64 --- ext/wasm/jaccwabyt/jaccwabyt.js | 7 +++-- ext/wasm/jaccwabyt/jaccwabyt.md | 51 +++++++++++++++++++-------------- manifest | 14 ++++----- manifest.uuid | 2 +- 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index 7c8d42b331..960aae7936 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -598,10 +598,13 @@ self.Jaccwabyt = function StructBinderFactory(config){ new DataView(heap().buffer, this.pointer + descr.offset, descr.sizeof) )[f._.getters[sigGlyph]](0, isLittleEndian); if(dbg.getter) log("debug.getter:",xPropName,"result =",rc); - if(rc && isAutoPtrSig(descr.signature)){ + /* // Removed: it is legal for multiple StructType instances to + // proxy the same pointer, and instanceForPointer() cannot account + // for that. + if(rc && isAutoPtrSig(descr.signature)){ rc = StructType.instanceForPointer(rc) || rc; if(dbg.getter) log("debug.getter:",xPropName,"resolved =",rc); - } + }*/ return rc; }; if(descr.readOnly){ diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index 73f6bfa947..855a1d16a0 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -281,8 +281,8 @@ supported letters are: signature entry. - **`f`** = `float` (4 bytes) - **`d`** = `double` (8 bytes) -- **`c`** = `int8` (char - see notes below!) -- **`C`** = `int8` (unsigned char - see notes below!) +- **`c`** = `int8` (1 byte) char - see notes below! +- **`C`** = `uint8` (1 byte) unsigned char - see notes below! - **`p`** = `int32` (see notes below!) - **`P`** = Like `p` but with extra handling. Described below. - **`s`** = like `int32` but is a _hint_ that it's a pointer to a @@ -295,10 +295,10 @@ supported letters are: Noting that: -- All of these types are numeric. Attempting to set any struct-bound - property to a non-numeric value will trigger an exception except in - cases explicitly noted otherwise. -- "Char" types: WASM does not define an `int8` type, nor does it +- **All of these types are numeric**. Attempting to set any + struct-bound property to a non-numeric value will trigger an + exception except in cases explicitly noted otherwise. +- **"Char" types**: WASM does not define an `int8` type, nor does it distinguish between signed and unsigned. This API treats `c` as `int8` and `C` as `uint8` for purposes of getting and setting values when using the `DataView` class. It is _not_ recommended that client @@ -345,12 +345,15 @@ special use of unsigned numbers). A capital `P` changes the semantics of plain member pointers (but not, as of this writing, function pointer members) as follows: -- When a `P`-type member is **fetched** via `myStruct.x` and its value is - a non-0 integer, [`StructBinder.instanceForPointer()`][StructBinder] - is used to try to map that pointer to a struct instance. If a match - is found, the "get" operation returns that instance instead of the - integer. If no match is found, it behaves exactly as for `p`, returning - the integer value. +- When a `P`-type member is **fetched** via `myStruct.x` and its + value is a non-0 integer, + [`StructBinder.instanceForPointer()`][StructBinder] is used to try + to map that pointer to a struct instance. If a match is found, the + "get" operation returns that instance instead of the integer. If no + match is found, it behaves exactly as for `p`, returning the integer + value. Removed because it's legal and possible to for multiple + instances to proxy the same pointer and this infrastructure cannot + account for that. - When a `P`-type member is **set** via `myStruct.x=y`, if [`(y instanceof StructType)`][StructType] then the value of `y.pointer` is stored in `myStruct.x`. If `y` is neither a number nor @@ -403,7 +406,6 @@ instances which have not been manually disposed. The following usage pattern offers one way to easily ensure proper cleanup of struct instances: - > ```javascript const my = new MyStruct(); @@ -417,11 +419,6 @@ try { from the byte array. */ // Pass the struct to C code which takes a MyStruct pointer: aCFunction( my.pointer ); - // Type-safely check if a pointer returned from C is a MyStruct: - const x = MyStruct.instanceForPointer( anotherCFunction() ); - // If it is a MyStruct, x now refers to that object. Note, however, - // that this only works for instances created in JS, as the - // pointer mapping only exists in JS space. } finally { my.dispose(); } @@ -434,6 +431,15 @@ to use `try`/`finally` without a `catch`, and doing so is an ideal match for the memory management requirements of Jaccwaby-bound struct instances. +It is often useful to wrap an existing instance of a C-side struct +without taking over ownership of its memory. That can be achieved by +simply passing a pointer to the constructor. For example: + +```js +const m = new MyStruct( functionReturningASharedPtr() ); +// calling m.dispose() will _not_ free the wrapped C-side instance. +``` + Now that we have struct instances, there are a number of things we can do with them, as covered in the rest of this document. @@ -557,7 +563,10 @@ The Struct Binder has the following members: any of its "significant" configuration values may have undefined results. -- `instanceForPointer(pointer)` +- `instanceForPointer(pointer)` (DEPRECATED) + *Do not use this - it will be removed* because it is legal for + multiple StructType instances to proxy the same pointer, and + instanceForPointer() cannot account for that.\ Given a pointer value relative to `config.memory`, if that pointer resolves to a struct of _any type_ generated via the same Struct Binder, this returns the struct instance associated with it, or @@ -599,7 +608,7 @@ individual instances via `theInstance.constructor`.): [struct's constructor][StructCtors]. If true, the memory is owned by someone other than the object and must outlive the object. -- `instanceForPointer(pointer)` +- `instanceForPointer(pointer)` (DEPRECATED) Works identically to the [StructBinder][] method of the same name. - `isA(value)` @@ -752,7 +761,7 @@ These constructors have the following "static" members: with _all_ instances and clears the `instanceForPointer()` mappings. Returns `this`. -- `instanceForPointer(pointer)` +- `instanceForPointer(pointer)` (DEPRECATED) Given a pointer value (accessible via the `pointer` property of all struct instances) which ostensibly refers to an instance of this class, this returns the instance associated with it, or `undefined` diff --git a/manifest b/manifest index 8345aa2237..0a6cf576d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expose\ssqlite3_get/set_auxdata()\sto\swasm.\sMinor\stest\sapp\sCSS\stweaks. -D 2022-12-05T14:13:55.858 +C Jaccwabyt\s(JS)\sdoc\supdates. +D 2022-12-05T14:32:35.107 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -539,8 +539,8 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff -F ext/wasm/jaccwabyt/jaccwabyt.js 292d37ad3b5fcef420db7b449813c38f1656f490f43e214ea4895793158e7fce -F ext/wasm/jaccwabyt/jaccwabyt.md 50df3ccb7a773634000496a2ccb805dd25cf8cd963696a7deec3209a68c6093b +F ext/wasm/jaccwabyt/jaccwabyt.js 199f3a0a7513692dac8e843fed210d4e7da12b1e3f168aff6b796e941424e8da +F ext/wasm/jaccwabyt/jaccwabyt.md c0172c0795522a137a5591ccc63703e5fc0410d7d7677884edfa8d6a9ab093f4 F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c3c56d9b944fd0d806d8dad9f0c7be3d7a5441765310908872cc525d82ab6a33 -R 585ea2dfb36f1f8d84b5d643e3dc1f7a +P 44659ad32a9fe6363badfc5dbb0bd51d6fb2ee1c8aa47b71e9cf3dbd631fde9e +R b435a2d94f66e49b4a6802642705f112 U stephan -Z 583c0236badf6d39ab9de3ab68428681 +Z 58b6d11303c3251b731643b22a0a2877 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3aea11316b..a3e46a1ef5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -44659ad32a9fe6363badfc5dbb0bd51d6fb2ee1c8aa47b71e9cf3dbd631fde9e \ No newline at end of file +a329a809b5da135a9c251e4d5f637d45d01d0248110ac05f2ad8f01d9df38c64 \ No newline at end of file From 2582d418d3acf2932c438ad26726d8f1976463ae Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 5 Dec 2022 15:05:46 +0000 Subject: [PATCH 165/282] Remove two features of jaccwabyt which were fundamentally flawed, along with approx. 250 lines of unit tests which heavily relied on them. Thankfully, none of the sqlite3.js-level code used those bits. FossilOrigin-Name: a190abc307847174f36421eaa3f47ef349c6f84a2bb35857fa64f64bbe722708 --- ext/wasm/jaccwabyt/jaccwabyt.js | 65 ------- ext/wasm/jaccwabyt/jaccwabyt.md | 57 +------ ext/wasm/tester1.c-pp.js | 289 ++------------------------------ manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 22 insertions(+), 407 deletions(-) diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index 960aae7936..0bd7869bc6 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -200,30 +200,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ return ()=>toss(sPropName(structName,propName),"is read-only."); }; - /** - When C code passes a pointer of a bound struct to back into - a JS function via a function pointer struct member, it - arrives in JS as a number (pointer). - StructType.instanceForPointer(ptr) can be used to get the - instance associated with that pointer, and __ptrBacklinks - holds that mapping. WeakMap keys must be objects, so we - cannot use a weak map to map pointers to instances. We use - the StructType constructor as the WeakMap key, mapped to a - plain, prototype-less Object which maps the pointers to - struct instances. That arrangement gives us a - per-StructType type-safe way to resolve pointers. - */ - const __ptrBacklinks = new WeakMap(); - /** - Similar to __ptrBacklinks but is scoped at the StructBinder - level and holds pointer-to-object mappings for all struct - instances created by any struct from any StructFactory - which this specific StructBinder has created. The intention - of this is to help implement more transparent handling of - pointer-type property resolution. - */ - const __ptrBacklinksGlobal = Object.create(null); - /** In order to completely hide StructBinder-bound struct pointers from JS code, we store them in a scope-local @@ -265,8 +241,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ }); } delete obj.ondispose; - delete __ptrBacklinks.get(ctor)[m]; - delete __ptrBacklinksGlobal[m]; __instancePointerMap.delete(obj); if(ctor.debugFlags.__flags.dealloc){ log("debug.dealloc:",(obj[xPtrPropName]?"EXTERNAL":""), @@ -299,8 +273,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ } if(fill) heap().fill(0, m, m + ctor.structInfo.sizeof); __instancePointerMap.set(obj, m); - __ptrBacklinks.get(ctor)[m] = obj; - __ptrBacklinksGlobal[m] = obj; }catch(e){ __freeStruct(ctor, obj, m); throw e; @@ -349,16 +321,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ return emscriptenFormat ? f._(m.signature) : m.signature; }; - /** - Returns the instanceForPointer() impl for the given - StructType constructor. - */ - const __instanceBacklinkFactory = function(ctor){ - const b = Object.create(null); - __ptrBacklinks.set(ctor, b); - return (ptr)=>b[ptr]; - }; - const __ptrPropDescriptor = { configurable: false, enumerable: false, get: function(){return __instancePointerMap.get(this)}, @@ -533,7 +495,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ */ Object.defineProperties(StructType, { allocCString: rop(__allocCString), - instanceForPointer: rop((ptr)=>__ptrBacklinksGlobal[ptr]), isA: rop((v)=>v instanceof StructType), hasExternalPointer: rop((v)=>(v instanceof StructType) && !!v[xPtrPropName]), memberKey: __memberKeyProp @@ -598,13 +559,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ new DataView(heap().buffer, this.pointer + descr.offset, descr.sizeof) )[f._.getters[sigGlyph]](0, isLittleEndian); if(dbg.getter) log("debug.getter:",xPropName,"result =",rc); - /* // Removed: it is legal for multiple StructType instances to - // proxy the same pointer, and instanceForPointer() cannot account - // for that. - if(rc && isAutoPtrSig(descr.signature)){ - rc = StructType.instanceForPointer(rc) || rc; - if(dbg.getter) log("debug.getter:",xPropName,"resolved =",rc); - }*/ return rc; }; if(descr.readOnly){ @@ -694,27 +648,9 @@ self.Jaccwabyt = function StructBinderFactory(config){ }; Object.defineProperties(StructCtor,{ debugFlags: debugFlags, - disposeAll: rop(function(){ - const map = __ptrBacklinks.get(StructCtor); - Object.keys(map).forEach(function(ptr){ - const b = map[ptr]; - if(b) __freeStruct(StructCtor, b, ptr); - }); - __ptrBacklinks.set(StructCtor, Object.create(null)); - return StructCtor; - }), - instanceForPointer: rop(__instanceBacklinkFactory(StructCtor)), isA: rop((v)=>v instanceof StructCtor), memberKey: __memberKeyProp, memberKeys: __structMemberKeys, - resolveToInstance: rop(function(v, throwIfNot=false){ - if(!(v instanceof StructCtor)){ - v = Number.isSafeInteger(v) - ? StructCtor.instanceForPointer(v) : undefined; - } - if(!v && throwIfNot) toss("Value is-not-a",StructCtor.structName); - return v; - }), methodInfoForKey: rop(function(mKey){ }), structInfo: rop(structInfo), @@ -732,7 +668,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ ); return StructCtor; }; - StructBinder.instanceForPointer = StructType.instanceForPointer; StructBinder.StructType = StructType; StructBinder.config = config; StructBinder.allocCString = __allocCString; diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index 855a1d16a0..c6cf5c8a25 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -345,15 +345,6 @@ special use of unsigned numbers). A capital `P` changes the semantics of plain member pointers (but not, as of this writing, function pointer members) as follows: -- When a `P`-type member is **fetched** via `myStruct.x` and its - value is a non-0 integer, - [`StructBinder.instanceForPointer()`][StructBinder] is used to try - to map that pointer to a struct instance. If a match is found, the - "get" operation returns that instance instead of the integer. If no - match is found, it behaves exactly as for `p`, returning the integer - value. Removed because it's legal and possible to for multiple - instances to proxy the same pointer and this infrastructure cannot - account for that. - When a `P`-type member is **set** via `myStruct.x=y`, if [`(y instanceof StructType)`][StructType] then the value of `y.pointer` is stored in `myStruct.x`. If `y` is neither a number nor @@ -399,9 +390,7 @@ It is important to understand that creating a new instance allocates memory on the WASM heap. We must not simply rely on garbage collection to clean up the instances because doing so will not free up the WASM heap memory. The correct way to free up that memory is to use the -object's `dispose()` method. Alternately, there is a "nuclear option": -`MyBinder.disposeAll()` will free the memory allocated for _all_ -instances which have not been manually disposed. +object's `dispose()` method. The following usage pattern offers one way to easily ensure proper cleanup of struct instances: @@ -563,23 +552,6 @@ The Struct Binder has the following members: any of its "significant" configuration values may have undefined results. -- `instanceForPointer(pointer)` (DEPRECATED) - *Do not use this - it will be removed* because it is legal for - multiple StructType instances to proxy the same pointer, and - instanceForPointer() cannot account for that.\ - Given a pointer value relative to `config.memory`, if that pointer - resolves to a struct of _any type_ generated via the same Struct - Binder, this returns the struct instance associated with it, or - `undefined` if no struct object is mapped to that pointer. This - differs from the struct-type-specific member of the same name in - that this one is not "type-safe": it does not know the type of the - returned object (if any) and may return a struct of any - [StructType][] for which this Struct Binder has created a - constructor. It cannot return instances created via a different - [StructBinderFactory][] because each factory can hypothetically have - a different memory heap. - - API: Struct Type ------------------------------------------------------------ @@ -608,9 +580,6 @@ individual instances via `theInstance.constructor`.): [struct's constructor][StructCtors]. If true, the memory is owned by someone other than the object and must outlive the object. -- `instanceForPointer(pointer)` (DEPRECATED) - Works identically to the [StructBinder][] method of the same name. - - `isA(value)` Returns true if its argument is a StructType instance _from the same [StructBinder][]_ as this StructType. @@ -755,21 +724,6 @@ pointer can be taken over using something like These constructors have the following "static" members: -- `disposeAll()` - For each instance of this struct, the equivalent of its `dispose()` - method is called. This frees all WASM-allocated memory associated - with _all_ instances and clears the `instanceForPointer()` - mappings. Returns `this`. - -- `instanceForPointer(pointer)` (DEPRECATED) - Given a pointer value (accessible via the `pointer` property of all - struct instances) which ostensibly refers to an instance of this - class, this returns the instance associated with it, or `undefined` - if no object _of this specific struct type_ is mapped to that - pointer. When C-side code calls back into JS code and passes a - pointer to an object, this function can be used to type-safely - "cast" that pointer back to its original object. - - `isA(value)` Returns true if its argument was created by this constructor. @@ -779,15 +733,6 @@ These constructors have the following "static" members: - `memberKeys(string)` Works exactly as documented for [StructType][]. -- `resolveToInstance(value [,throwIfNot=false])` - Works like `instanceForPointer()` but accepts either an instance - of this struct type or a pointer which resolves to one. - It returns an instance of this struct type on success. - By default it returns a falsy value if its argument is not, - or does not resolve to, an instance of this struct type, - but if passed a truthy second argument then it will throw - instead. - - `structInfo` The structure description passed to [StructBinder][] when this constructor was generated. diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 2d1b634e55..5db00cee64 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -740,9 +740,6 @@ self.sqlite3InitModule = sqlite3InitModule; assert(undefined === K.prototype.lookupMember('nope',false)). assert(k1 instanceof StructType). assert(StructType.isA(k1)). - assert(K.resolveToInstance(k1.pointer)===k1). - mustThrowMatching(()=>K.resolveToInstance(null,true), /is-not-a my_struct/). - assert(k1 === StructType.instanceForPointer(k1.pointer)). mustThrowMatching(()=>k1.$ro = 1, /read-only/); Object.keys(MyStructDef.members).forEach(function(key){ key = K.memberKey(key); @@ -752,8 +749,7 @@ self.sqlite3InitModule = sqlite3InitModule; " from "+k1.memoryDump()); }); T.assert('number' === typeof k1.pointer). - mustThrowMatching(()=>k1.pointer = 1, /pointer/). - assert(K.instanceForPointer(k1.pointer) === k1); + mustThrowMatching(()=>k1.pointer = 1, /pointer/); k1.$p4 = 1; k1.$pP = 2; T.assert(1 === k1.$p4).assert(2 === k1.$pP); if(MyStructDef.members.$p8){ @@ -768,22 +764,13 @@ self.sqlite3InitModule = sqlite3InitModule; assert('number' === typeof k1.$cstr). assert('A C-string.' === k1.memberToJsString('cstr')); k1.$pP = k2; - T.assert(k1.$pP === k2); + T.assert(k1.$pP === k2.pointer); k1.$pP = null/*null is special-cased to 0.*/; T.assert(0===k1.$pP); let ptr = k1.pointer; k1.dispose(); T.assert(undefined === k1.pointer). - assert(undefined === K.instanceForPointer(ptr)). mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); - const k3 = new K(); - ptr = k3.pointer; - T.assert(k3 === K.instanceForPointer(ptr)); - K.disposeAll(); - T.assert(ptr). - assert(undefined === k2.pointer). - assert(undefined === k3.pointer). - assert(undefined === K.instanceForPointer(ptr)); }finally{ k1.dispose(); k2.dispose(); @@ -813,10 +800,8 @@ self.sqlite3InitModule = sqlite3InitModule; assert(wts instanceof WTStruct). assert(wts instanceof StructType). assert(StructType.isA(wts)). - assert(wts === StructType.instanceForPointer(wts.pointer)); - T.assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). - assert(0===wts.$ppV).assert(0===wts.$xFunc). - assert(WTStruct.instanceForPointer(wts.pointer) === wts); + assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). + assert(0===wts.$ppV).assert(0===wts.$xFunc); const testFunc = W.xGet('sqlite3_wasm_test_struct'/*name gets mangled in -O3 builds!*/); let counter = 0; @@ -825,7 +810,6 @@ self.sqlite3InitModule = sqlite3InitModule; /*log("This from a JS function called from C, "+ "which itself was called from JS. arg =",arg);*/ ++counter; - T.assert(WTStruct.instanceForPointer(arg) === wts); if(3===counter){ tossQuietly("Testing exception propagation."); } @@ -849,7 +833,7 @@ self.sqlite3InitModule = sqlite3InitModule; testFunc(wts.pointer); //log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) - .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)) + .assert(wts.$ppV === wts.pointer) .assert('string' === typeof wts.memberToJsString('cstr')) .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) .mustThrowMatching(()=>wts.memberToJsString('xFunc'), @@ -857,279 +841,30 @@ self.sqlite3InitModule = sqlite3InitModule; ; testFunc(wts.pointer); T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) - .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)); + .assert(wts.$ppV === wts.pointer); /** The 3rd call to wtsFunc throw from JS, which is called from C, which is called from JS. Let's ensure that that exception propagates back here... */ T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); W.uninstallFunction(wts.$xFunc); wts.$xFunc = 0; - if(autoResolvePtr){ - wts.$ppV = 0; - T.assert(!wts.$ppV); - //WTStruct.debugFlags(0x03); - wts.$ppV = wts; - T.assert(wts === wts.$ppV) - //WTStruct.debugFlags(0); - } + wts.$ppV = 0; + T.assert(!wts.$ppV); + //WTStruct.debugFlags(0x03); + wts.$ppV = wts; + T.assert(wts.pointer === wts.$ppV) wts.setMemberCString('cstr', "A C-string."); T.assert(Array.isArray(wts.ondispose)). assert(wts.ondispose[0] === wts.$cstr). assert('A C-string.' === wts.memberToJsString('cstr')); const ptr = wts.pointer; wts.dispose(); - T.assert(ptr).assert(undefined === wts.pointer). - assert(undefined === WTStruct.instanceForPointer(ptr)) + T.assert(ptr).assert(undefined === wts.pointer); }finally{ wts.dispose(); } }/*StructBinder*/) - //////////////////////////////////////////////////////////////////// - .t('sqlite3.StructBinder part 2', function(sqlite3){ - // https://www.sqlite.org/c3ref/vfs.html - // https://www.sqlite.org/c3ref/io_methods.html - const sqlite3_io_methods = capi.sqlite3_io_methods, - sqlite3_vfs = capi.sqlite3_vfs, - sqlite3_file = capi.sqlite3_file; - //log("struct sqlite3_file", sqlite3_file.memberKeys()); - //log("struct sqlite3_vfs", sqlite3_vfs.memberKeys()); - //log("struct sqlite3_io_methods", sqlite3_io_methods.memberKeys()); - const installMethod = function callee(tgt, name, func){ - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - callee.ondisposeRemoveFunc = function(){ - if(this.__ondispose){ - const who = this; - this.__ondispose.forEach( - (v)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - }else{/*wasm function wrapper property*/ - delete who[v]; - } - } - ); - delete this.__ondispose; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name), - memKey = tgt.memberKey(name); - //log("installMethod",tgt, name, sigN); - if(!tgt.__ondispose){ - T.assert(undefined === tgt.ondispose); - tgt.ondispose = [callee.ondisposeRemoveFunc]; - tgt.__ondispose = []; - } - const fProxy = callee.argcProxy(func, sigN); - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - /** - ACHTUNG: function pointer IDs are from a different pool than - allocation IDs, starting at 1 and incrementing in steps of 1, - so if we set tgt[memKey] to those values, we'd very likely - later misinterpret them as plain old pointer addresses unless - unless we use some silly heuristic like "all values <5k are - presumably function pointers," or actually perform a function - lookup on every pointer to first see if it's a function. That - would likely work just fine, but would be kludgy. - - It turns out that "all values less than X are functions" is - essentially how it works in wasm: a function pointer is - reported to the client as its index into the - __indirect_function_table. - - So... once jaccwabyt can be told how to access the - function table, it could consider all pointer values less - than that table's size to be functions. As "real" pointer - values start much, much higher than the function table size, - that would likely work reasonably well. e.g. the object - pointer address for sqlite3's default VFS is (in this local - setup) 65104, whereas the function table has fewer than 600 - entries. - */ - const wrapperKey = '$'+memKey; - tgt[wrapperKey] = fProxy; - tgt.__ondispose.push(pFunc, wrapperKey); - //log("tgt.__ondispose =",tgt.__ondispose); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; - - const installIOMethods = function instm(iom){ - (iom instanceof capi.sqlite3_io_methods) || toss("Invalid argument type."); - if(!instm._requireFileArg){ - instm._requireFileArg = function(arg,methodName){ - arg = capi.sqlite3_file.resolveToInstance(arg); - if(!arg){ - err("sqlite3_io_methods::xClose() was passed a non-sqlite3_file."); - } - return arg; - }; - instm._methods = { - // https://sqlite.org/c3ref/io_methods.html - xClose: /*i(P)*/function(f){ - /* int (*xClose)(sqlite3_file*) */ - log("xClose(",f,")"); - if(!(f = instm._requireFileArg(f,'xClose'))) return capi.SQLITE_MISUSE; - f.dispose(/*noting that f has externally-owned memory*/); - return 0; - }, - xRead: /*i(Ppij)*/function(f,dest,n,offset){ - /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ - log("xRead(",arguments,")"); - if(!(f = instm._requireFileArg(f))) return capi.SQLITE_MISUSE; - wasm.heap8().fill(0, dest + offset, n); - return 0; - }, - xWrite: /*i(Ppij)*/function(f,dest,n,offset){ - /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ - log("xWrite(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xWrite'))) return capi.SQLITE_MISUSE; - return 0; - }, - xTruncate: /*i(Pj)*/function(f){ - /* int (*xTruncate)(sqlite3_file*, sqlite3_int64 size) */ - log("xTruncate(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xTruncate'))) return capi.SQLITE_MISUSE; - return 0; - }, - xSync: /*i(Pi)*/function(f){ - /* int (*xSync)(sqlite3_file*, int flags) */ - log("xSync(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xSync'))) return capi.SQLITE_MISUSE; - return 0; - }, - xFileSize: /*i(Pp)*/function(f,pSz){ - /* int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize) */ - log("xFileSize(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xFileSize'))) return capi.SQLITE_MISUSE; - wasm.setMemValue(pSz, 0/*file size*/); - return 0; - }, - xLock: /*i(Pi)*/function(f){ - /* int (*xLock)(sqlite3_file*, int) */ - log("xLock(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xLock'))) return capi.SQLITE_MISUSE; - return 0; - }, - xUnlock: /*i(Pi)*/function(f){ - /* int (*xUnlock)(sqlite3_file*, int) */ - log("xUnlock(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xUnlock'))) return capi.SQLITE_MISUSE; - return 0; - }, - xCheckReservedLock: /*i(Pp)*/function(){ - /* int (*xCheckReservedLock)(sqlite3_file*, int *pResOut) */ - log("xCheckReservedLock(",arguments,")"); - return 0; - }, - xFileControl: /*i(Pip)*/function(){ - /* int (*xFileControl)(sqlite3_file*, int op, void *pArg) */ - log("xFileControl(",arguments,")"); - return capi.SQLITE_NOTFOUND; - }, - xSectorSize: /*i(P)*/function(){ - /* int (*xSectorSize)(sqlite3_file*) */ - log("xSectorSize(",arguments,")"); - return 0/*???*/; - }, - xDeviceCharacteristics:/*i(P)*/function(){ - /* int (*xDeviceCharacteristics)(sqlite3_file*) */ - log("xDeviceCharacteristics(",arguments,")"); - return 0; - } - }; - }/*static init*/ - iom.$iVersion = 1; - Object.keys(instm._methods).forEach( - (k)=>installMethod(iom, k, instm._methods[k]) - ); - }/*installIOMethods()*/; - - const iom = new sqlite3_io_methods, sfile = new sqlite3_file; - const err = console.error.bind(console); - try { - const IOM = sqlite3_io_methods, S3F = sqlite3_file; - //log("iom proto",iom,iom.constructor.prototype); - //log("sfile",sfile,sfile.constructor.prototype); - T.assert(0===sfile.$pMethods).assert(iom.pointer > 0); - //log("iom",iom); - sfile.$pMethods = iom.pointer; - T.assert(iom.pointer === sfile.$pMethods) - .assert(IOM.resolveToInstance(iom)) - .assert(undefined ===IOM.resolveToInstance(sfile)) - .mustThrow(()=>IOM.resolveToInstance(0,true)) - .assert(S3F.resolveToInstance(sfile.pointer)) - .assert(undefined===S3F.resolveToInstance(iom)) - .assert(iom===IOM.resolveToInstance(sfile.$pMethods)); - T.assert(0===iom.$iVersion); - installIOMethods(iom); - T.assert(1===iom.$iVersion); - //log("iom.__ondispose",iom.__ondispose); - T.assert(Array.isArray(iom.__ondispose)).assert(iom.__ondispose.length>10); - }finally{ - iom.dispose(); - T.assert(undefined === iom.__ondispose); - } - - const dVfs = new sqlite3_vfs(capi.sqlite3_vfs_find(null)); - try { - const SB = sqlite3.StructBinder; - T.assert(dVfs instanceof SB.StructType) - .assert(dVfs.pointer) - .assert('sqlite3_vfs' === dVfs.structName) - .assert(!!dVfs.structInfo) - .assert(SB.StructType.hasExternalPointer(dVfs)) - .assert(dVfs.$iVersion>0) - .assert('number'===typeof dVfs.$zName) - .assert('number'===typeof dVfs.$xSleep) - .assert(wasm.functionEntry(dVfs.$xOpen)) - .assert(dVfs.memberIsString('zName')) - .assert(dVfs.memberIsString('$zName')) - .assert(!dVfs.memberIsString('pAppData')) - .mustThrowMatching(()=>dVfs.memberToJsString('xSleep'), - /Invalid member type signature for C-string/) - .mustThrowMatching(()=>dVfs.memberSignature('nope'), /nope is not a mapped/) - .assert('string' === typeof dVfs.memberToJsString('zName')) - .assert(dVfs.memberToJsString('zName')===dVfs.memberToJsString('$zName')) - ; - //log("Default VFS: @",dVfs.pointer); - Object.keys(sqlite3_vfs.structInfo.members).forEach(function(mname){ - const mk = sqlite3_vfs.memberKey(mname), mbr = sqlite3_vfs.structInfo.members[mname], - addr = dVfs[mk], prefix = 'defaultVfs.'+mname; - if(1===mbr.signature.length){ - let sep = '?', val = undefined; - switch(mbr.signature[0]){ - // TODO: move this into an accessor, e.g. getPreferredValue(member) - case 'i': case 'j': case 'f': case 'd': sep = '='; val = dVfs[mk]; break - case 'p': case 'P': sep = '@'; val = dVfs[mk]; break; - case 's': sep = '='; - val = dVfs.memberToJsString(mname); - break; - } - //log(prefix, sep, val); - }else{ - //log(prefix," = funcptr @",addr, wasm.functionEntry(addr)); - } - }); - }finally{ - dVfs.dispose(); - T.assert(undefined===dVfs.pointer); - } - }/*StructBinder part 2*/) - //////////////////////////////////////////////////////////////////// .t('sqlite3.wasm.pstack', function(sqlite3){ const P = wasm.pstack; diff --git a/manifest b/manifest index 0a6cf576d7..a3f0d928c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Jaccwabyt\s(JS)\sdoc\supdates. -D 2022-12-05T14:32:35.107 +C Remove\stwo\sfeatures\sof\sjaccwabyt\swhich\swere\sfundamentally\sflawed,\salong\swith\sapprox.\s250\slines\sof\sunit\stests\swhich\sheavily\srelied\son\sthem.\sThankfully,\snone\sof\sthe\ssqlite3.js-level\scode\sused\sthose\sbits. +D 2022-12-05T15:05:46.306 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -539,8 +539,8 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff -F ext/wasm/jaccwabyt/jaccwabyt.js 199f3a0a7513692dac8e843fed210d4e7da12b1e3f168aff6b796e941424e8da -F ext/wasm/jaccwabyt/jaccwabyt.md c0172c0795522a137a5591ccc63703e5fc0410d7d7677884edfa8d6a9ab093f4 +F ext/wasm/jaccwabyt/jaccwabyt.js b7261221133cda8d363f16ddbac8e5b671fd51ce962fc34dc10e738a293b696d +F ext/wasm/jaccwabyt/jaccwabyt.md 72742a3205f1477de68086e7e4a854ed5f7d08dfcd6db54cdbea4dba4866f159 F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 -F ext/wasm/tester1.c-pp.js 419a2fd31b0230e0495ed38a56dbe1e2bbc93c5953b796de43e2a70901970b83 +F ext/wasm/tester1.c-pp.js 8ed17c0e1f271e536cb7ccd86d3992785fc8bf2f94f9c2e0088ca670601ee087 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 44659ad32a9fe6363badfc5dbb0bd51d6fb2ee1c8aa47b71e9cf3dbd631fde9e -R b435a2d94f66e49b4a6802642705f112 +P a329a809b5da135a9c251e4d5f637d45d01d0248110ac05f2ad8f01d9df38c64 +R 3d4ed37777b015bac598ce3a6734f5aa U stephan -Z 58b6d11303c3251b731643b22a0a2877 +Z 002b2dd6406bdc41efd7e7f9f5d7918f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a3e46a1ef5..14e600f31e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a329a809b5da135a9c251e4d5f637d45d01d0248110ac05f2ad8f01d9df38c64 \ No newline at end of file +a190abc307847174f36421eaa3f47ef349c6f84a2bb35857fa64f64bbe722708 \ No newline at end of file From a3d0c158a0e5942a2cfbfa05b1c1a629ed230ed0 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Dec 2022 18:19:56 +0000 Subject: [PATCH 166/282] Add loops and rows counters to "USE TEMP B-TREE FOR ORDER BY" records. Also fix the sqliteHwtime() function so that it returns a 64-bit value. FossilOrigin-Name: 41a0e05e8c0fca3b803fe4bd017a157c172b2ca518356a2a4d4ed4f12d01a1e3 --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/expr.c | 1 + src/hwtime.h | 6 +++--- src/select.c | 10 +++++----- src/vdbe.c | 8 +++++++- src/vdbe.h | 2 ++ src/vdbeapi.c | 2 +- src/vdbeaux.c | 19 +++++++++++++++++++ test/scanstatus2.test | 7 +++---- 10 files changed, 55 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 7ca1c989db..881e5d52a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sSQLITE_SCANSTAT_NCYCLE\sso\sthat\sit\sreports\son\svirtual\stables. -D 2022-12-03T21:24:26.253 +C Add\sloops\sand\srows\scounters\sto\s"USE\sTEMP\sB-TREE\sFOR\sORDER\sBY"\srecords.\sAlso\sfix\sthe\ssqliteHwtime()\sfunction\sso\sthat\sit\sreturns\sa\s64-bit\svalue. +D 2022-12-05T18:19:56.530 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -595,14 +595,14 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c a56a7ad1163a9888d46cd5820be2e65354fb1aa04ed6909f7c3e5831e0ee2c29 F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 02a24db96257d0d0dfc82401a6720ae3156cd20b60b84db9523aaf87884a3254 +F src/expr.c d0ce060008452653e9c7bcdce8b1e3ff078fbbd5f9f80b530b3ab99bb70a8078 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d F src/global.c e06ff8e0acd85aec13563c9ecb44fbbf38232ccf73594998fd880b92d619594b F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 -F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 +F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 1b11a2e33ee52db93c02fddac67e39d00161d61b69fac2675b82f2aa68c1b61c F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 @@ -645,7 +645,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c f6674cdcb839f118cf931b0c49e94d5fd37007a616239f62882d02df51cacbe7 +F src/select.c 321f29e431fbb71e594cc7026391a827a0270237597d2e2401244e7602dea8bd F src/shell.c.in 8cfe0c7dbd3ac8f9eca40fdad85d90f22362a89ab9ad8d64826fd0ad5e408d13 F src/sqlite.h.in 3ba99a3c2b414b9ad36d06e2f273ab6077f5f74e4d79630aee047394b92f7c7f F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -717,11 +717,11 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 313f3154e2b85a447326f5dd15de8d31a4df6ab0c3579bd58f426ff634ec9050 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 5f5cc749c999d65e2115bb3ec555f7ac0ecfbda7f47526272d53a032e9d083b1 -F src/vdbe.h 77b3fb0abe5c5edaca849816584f0fd019c0f91e80410ef2d65f57fcf290ddd8 +F src/vdbe.c e744259050ec9bbcd47acf9a03a66fe3305d7429c823995de11ed524ca06d16f +F src/vdbe.h 6d921884cf8ec6a53efba99f8b68e32e955367631743e29039840e781aaf547c F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 -F src/vdbeapi.c 35b4ca406f9a4bfd23fb7016b6d36280a5c1d68372b69271d6fbfcceead4b8ab -F src/vdbeaux.c 7dcaff5933e4fbe5983be435b7cd2951d5f3ebd0f2acaf70fed4271dbfb57981 +F src/vdbeapi.c 56a27f44784dc0dbf4cdc0f7f3e45cdf6d2b2b17ee95b3a5d43165427135c2ea +F src/vdbeaux.c 42e3632fa1fea7e4f9cf5a59c4b1e1c5e07931b796bc670a096eed68fe5ade11 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -1457,7 +1457,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 7dbcfd6adc6a8df6abc59f83d6da5a27e1bce0b2f6fa55147c8176d7c44e0450 -F test/scanstatus2.test 799129ea953ba1235d4d5e7f59375cdff79f9185142773ed5d9e1c2c14500833 +F test/scanstatus2.test 503c8529111fbac777b225b846f3a31a020e80950d40763b65953fae46e9ceb9 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2066,8 +2066,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 365011ae8b7e3fcaa2c4ea9601231a3ef2223e60d7a53ec33013109dca22ad58 -R d4e70f090a6234401994809de206bf5b +P 622d8eb3724bee617b55d6fb71f1a2d683db6858065adced6bf3ce9525bcd6b5 +R 26a74ba4d9b4917db1af945f6ebd0886 U dan -Z 77f61cda4725a7748ad22a19d7f4eb4e +Z 549ef47ca8c045470c46bb8ad599992d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b1c968e110..6c7d1a6e12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -622d8eb3724bee617b55d6fb71f1a2d683db6858065adced6bf3ce9525bcd6b5 \ No newline at end of file +41a0e05e8c0fca3b803fe4bd017a157c172b2ca518356a2a4d4ed4f12d01a1e3 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1528877e23..592e9dd25c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3314,6 +3314,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ */ ExplainQueryPlan2(addrExplain, (pParse, 1, "%sSCALAR SUBQUERY %d", addrOnce?"":"CORRELATED ", pSel->selId)); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, -1); nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); pParse->nMem += nReg; diff --git a/src/hwtime.h b/src/hwtime.h index 037c55a977..d27204a69c 100644 --- a/src/hwtime.h +++ b/src/hwtime.h @@ -48,9 +48,9 @@ #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; } #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) diff --git a/src/select.c b/src/select.c index 7d010c6244..34ba3f7a4c 100644 --- a/src/select.c +++ b/src/select.c @@ -66,7 +66,7 @@ struct SortCtx { #endif struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - int addrPushStart; /* First instruction to push data into sorter */ + int addrPush; /* First instruction to push data into sorter */ int addrPushEnd; /* Last instruction that pushes data into sorter */ #endif }; @@ -726,7 +726,7 @@ static void pushOntoSorter( assert( nData==1 || regData==regOrigData || regOrigData==0 ); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - pSort->addrPushStart = sqlite3VdbeCurrentAddr(v); + pSort->addrPush = sqlite3VdbeCurrentAddr(v); #endif if( nPrefixReg ){ @@ -1665,9 +1665,9 @@ static void generateSortTail( ExplainQueryPlan2(addrExplain, (pParse, 0, "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") ); - sqlite3VdbeScanStatusRange( - v, addrExplain, pSort->addrPushStart, pSort->addrPushEnd - ); + sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); + assert( addrBreak<0 ); if( pSort->labelBkOut ){ diff --git a/src/vdbe.c b/src/vdbe.c index 81642eb929..1bf0ceb297 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -8735,6 +8735,7 @@ default: { /* This is really OP_Noop, OP_Explain */ assert( pnCycle ); if( pnCycle ){ *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + assert( *pnCycle < (((u64)1) << 48) ); pnCycle = 0; } #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) @@ -8820,9 +8821,14 @@ vdbe_return: #if defined(VDBE_PROFILE) if( pnCycle ){ *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + assert( *pnCycle < (((u64)1) << 48) ); + pnCycle = 0; } #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( pnCycle ) *pnCycle += sqlite3Hwtime(); + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; + } #endif #ifndef SQLITE_OMIT_PROGRESS_CALLBACK diff --git a/src/vdbe.h b/src/vdbe.h index 8d0c8123f3..a199247434 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -388,9 +388,11 @@ int sqlite3VdbeBytecodeVtabInit(sqlite3*); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); void sqlite3VdbeScanStatusRange(Vdbe*, int, int, int); +void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int); #else # define sqlite3VdbeScanStatus(a,b,c,d,e,f) # define sqlite3VdbeScanStatusRange(a,b,c,d) +# define sqlite3VdbeScanStatusCounters(a,b,c,d) #endif #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 2b9b1087f2..2e9a39578c 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2142,7 +2142,7 @@ int sqlite3_stmt_scanstatus_v2( }else{ for(idx=0; idxnScan; idx++){ pScan = &p->aScan[idx]; - if( pScan->addrLoop ){ + if( pScan->zName ){ iScan--; if( iScan<0 ) break; } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 74d3866c48..3078237a9d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1152,6 +1152,25 @@ void sqlite3VdbeScanStatusRange( } } } + +void sqlite3VdbeScanStatusCounters( + Vdbe *p, + int addrExplain, + int addrLoop, + int addrVisit +){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + pScan->addrLoop = addrLoop; + pScan->addrVisit = addrVisit; + } +} #endif diff --git a/test/scanstatus2.test b/test/scanstatus2.test index a7dfabc4ac..784ed290d6 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -135,7 +135,7 @@ QUERY (nCycle=nnn) ----SCAN t2 --SCAN v2 --SCAN t1 ---USE TEMP B-TREE FOR ORDER BY +--USE TEMP B-TREE FOR ORDER BY (nCycle=nnn) } #------------------------------------------------------------------------- @@ -147,12 +147,11 @@ do_execsql_test 2.0 { INSERT INTO ft VALUES('ghi'); } -explain_i { - SELECT * FROM ft('def') - } do_graph_test 2.1 { SELECT * FROM ft('def') } { +QUERY (nCycle=nnn) +--SCAN ft VIRTUAL TABLE INDEX 0:M1 (nCycle=nnn) } finish_test From 07c8e08889b840574bb28f7ffd0e19fc76ba5ea5 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Dec 2022 18:52:12 +0000 Subject: [PATCH 167/282] Update comments in sqlite.h.in to account for sqlite3_stmt_scanstatus_v2(). FossilOrigin-Name: 009462f2344b1f468cf9440343a47fec68d783a2bfb4fa6168bb227ec910b918 --- manifest | 16 ++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 63 ++++++++++++++++++++++++++++++++++--------------- src/vdbeapi.c | 1 - src/vdbeaux.c | 12 ++++++++++ 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 137a5c9ea7..d0d1acb7a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges. -D 2022-12-05T18:26:37.073 +C Update\scomments\sin\ssqlite.h.in\sto\saccount\sfor\ssqlite3_stmt_scanstatus_v2(). +D 2022-12-05T18:52:12.491 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -647,7 +647,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 321f29e431fbb71e594cc7026391a827a0270237597d2e2401244e7602dea8bd F src/shell.c.in e7fc8db64df14d1893a86dce488770efdbb1007e44ec8cd5549769533bb419f2 -F src/sqlite.h.in e72678d6b1b2e9b006db786d5d9ac534ccf58fff6612557290b6276d0d3a5a84 +F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f F src/sqliteInt.h 0c9934acd88e0fa14f0d4743233b4cd7bd7bbc84099ba3cad50d8e69676ce2b9 @@ -720,8 +720,8 @@ F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c e744259050ec9bbcd47acf9a03a66fe3305d7429c823995de11ed524ca06d16f F src/vdbe.h 6d921884cf8ec6a53efba99f8b68e32e955367631743e29039840e781aaf547c F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 -F src/vdbeapi.c 56a27f44784dc0dbf4cdc0f7f3e45cdf6d2b2b17ee95b3a5d43165427135c2ea -F src/vdbeaux.c 42e3632fa1fea7e4f9cf5a59c4b1e1c5e07931b796bc670a096eed68fe5ade11 +F src/vdbeapi.c 959cef4c1e5cb6589e67b3295fe9d3e45cbd109ae6618d57b74c900dbd6be0cd +F src/vdbeaux.c 0b81f317c86ed9f4ba822af66a52777fed6e8180edc79d4fc62ffe75c8e52d80 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 41a0e05e8c0fca3b803fe4bd017a157c172b2ca518356a2a4d4ed4f12d01a1e3 dc7dd2d3e50e7cc474b22f1b5b219da32bcd7aa1ba56864d1dbcf0d3a6fa06f2 -R 656aefb1ecbcda39e45961b382db5e0d +P 1a72777b1279f74f212fb2f675a4594a238e5d28f048879d7f5ad5287673c3c4 +R 8c721203ec6b199973a0c855b39497c4 U dan -Z eafa1697653e28cd3e4cfd55976da0a1 +Z 6aa6d3a2a8294beaab19425ddd9ee52d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 71602cd212..34762ab775 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a72777b1279f74f212fb2f675a4594a238e5d28f048879d7f5ad5287673c3c4 \ No newline at end of file +009462f2344b1f468cf9440343a47fec68d783a2bfb4fa6168bb227ec910b918 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 13eb2e233c..91e1e07f87 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -9886,6 +9886,10 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** managed by the prepared statement S and will be automatically freed when ** S is finalized. ** +** Not all values are available for all query elements. When a value is +** not available, the output variable is set to -1 if the value is numeric, +** or to NULL if it is a string (SQLITE_SCANSTAT_NAME). +** **
    ** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    **
    ^The [sqlite3_int64] variable pointed to by the V parameter will be @@ -9913,13 +9917,25 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] ** description for the X-th loop. ** -** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECTID
    **
    ^The "int" variable pointed to by the V parameter will be set to the -** "select-id" for the X-th loop. The select-id identifies which query or -** subquery the loop is part of. The main query has a select-id of zero. -** The select-id is the same value as is output in the first column -** of an [EXPLAIN QUERY PLAN] query. +** id for the X-th query plan element. The id value is unique within the +** statement. The select-id is the same value as is output in the first +** column of an [EXPLAIN QUERY PLAN] query. **
    +** +** [[SQLITE_SCANSTAT_PARENTID]]
    SQLITE_SCANSTAT_PARENTID
    +**
    The "int" variable pointed to by the V parameter will be set to the +** the id of the parent of the current query element, if applicable, or +** to zero if the query element has no parent. This is the same value as +** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_NCYCLE]]
    SQLITE_SCANSTAT_NCYCLE
    +**
    The sqlite3_int64 output value is set to the number of cycles, +** according to the processor time-stamp counter, that elapsed while the +** query element was being processed. This value is not available for +** all query elements - if it is unavailable the output variable is +** set to -1. */ #define SQLITE_SCANSTAT_NLOOP 0 #define SQLITE_SCANSTAT_NVISIT 1 @@ -9934,7 +9950,7 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** CAPI3REF: Prepared Statement Scan Status ** METHOD: sqlite3_stmt ** -** This interface returns information about the predicted and measured +** These interfaces return information about the predicted and measured ** performance for pStmt. Advanced applications can use this ** interface to compare the predicted and the measured performance and ** issue warnings and/or rerun [ANALYZE] if discrepancies are found. @@ -9945,19 +9961,25 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** ** The "iScanStatusOp" parameter determines which status information to return. ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior -** of this interface is undefined. -** ^The requested measurement is written into a variable pointed to by -** the "pOut" parameter. -** Parameter "idx" identifies the specific loop to retrieve statistics for. -** Loops are numbered starting from zero. ^If idx is out of range - less than -** zero or greater than or equal to the total number of loops used to implement -** the statement - a non-zero value is returned and the variable that pOut -** points to is unchanged. +** of this interface is undefined. ^The requested measurement is written into +** a variable pointed to by the "pOut" parameter. ** -** ^Statistics might not be available for all loops in all statements. ^In cases -** where there exist loops with no available statistics, this function behaves -** as if the loop did not exist - it returns non-zero and leave the variable -** that pOut points to unchanged. +** The "flags" parameter must be passed a mask of flags. At present only +** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** is specified, then status information is available for all elements +** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements +** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of +** the EXPLAIN QUERY PLAN output) are available. Invoking API +** sqlite3_stmt_scanstatus() is equivalent to calling +** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. +** +** Parameter "idx" identifies the specific query element to retrieve statistics +** for. Query elements are numbered starting from zero. A value of -1 may be +** to query for statistics regarding the entire query. ^If idx is out of range +** - less than -1 or greater than or equal to the total number of query +** elements used to implement the statement - a non-zero value is returned and +** the variable that pOut points to is unchanged. ** ** See also: [sqlite3_stmt_scanstatus_reset()] */ @@ -9967,7 +9989,6 @@ int sqlite3_stmt_scanstatus( int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ void *pOut /* Result written here */ ); - int sqlite3_stmt_scanstatus_v2( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ int idx, /* Index of loop to report on */ @@ -9976,6 +9997,10 @@ int sqlite3_stmt_scanstatus_v2( void *pOut /* Result written here */ ); +/* +** CAPI3REF: Prepared Statement Scan Status +** KEYWORDS: {scan status flags} +*/ #define SQLITE_SCANSTAT_COMPLEX 0x0001 /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 2e9a39578c..0644cffa57 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2258,7 +2258,6 @@ int sqlite3_stmt_scanstatus( return sqlite3_stmt_scanstatus_v2(pStmt, iScan, iScanStatusOp, 0, pOut); } - /* ** Zero all counters associated with the sqlite3_stmt_scanstatus() data. */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 3078237a9d..e22b741196 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1128,6 +1128,13 @@ void sqlite3VdbeScanStatus( } } +/* +** Add the range of instructions from addrStart to addrEnd (inclusive) to +** the set of those corresponding to the sqlite3_stmt_scanstatus() counters +** associated with the OP_Explain instruction at addrExplain. The +** sum of the sqlite3Hwtime() values for each of these instructions +** will be returned for SQLITE_SCANSTAT_NCYCLE requests. +*/ void sqlite3VdbeScanStatusRange( Vdbe *p, int addrExplain, @@ -1153,6 +1160,11 @@ void sqlite3VdbeScanStatusRange( } } +/* +** Set the addresses for the SQLITE_SCANSTAT_NLOOP and SQLITE_SCANSTAT_NROW +** counters for the query element associated with the OP_Explain at +** addrExplain. +*/ void sqlite3VdbeScanStatusCounters( Vdbe *p, int addrExplain, From 9ab1598e52d70a6b7ae823804144af33a80bbbc0 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 6 Dec 2022 05:09:51 +0000 Subject: [PATCH 168/282] Add optional feature: A CLI continuation prompt which reflects open lexemes and parens, similarly to PG shell. FossilOrigin-Name: dac2ddc287db7a68d0cd49b785060f62290868fbb1aa2ee09e54d3b1acfbf55f --- manifest | 17 ++++---- manifest.uuid | 2 +- src/shell.c.in | 112 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 117 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 66f7ed61f6..8059688414 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scase\sthat\sshould\shave\sbeen\spart\sof\sprevious\scommit. -D 2022-12-05T14:20:54.124 +C Add\soptional\sfeature:\sA\sCLI\scontinuation\sprompt\swhich\sreflects\sopen\slexemes\sand\sparens,\ssimilarly\sto\sPG\sshell. +D 2022-12-06T05:09:51.137 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 -F src/shell.c.in f6ab148f150dc0c8460be74a61566d37c65d43311e84963cc1a58df3fc277511 +F src/shell.c.in 47c15d0a5d3e8b172389117be0cbb5296529df49f979ce98ec5e5f967b38bf34 F src/sqlite.h.in 14d1273e84a8a4d7cbfc044592c4f97ea04ecf59b2a684f3c7c1b2ab9e48ae0e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2066,8 +2066,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 15f0be8a640e7bfa4130edd4650a745337bd96083b119a1553f9abf9ff066806 -R 0d5d7f5190e8e855ad343ae2d8e33bd7 -U dan -Z 56bab95316a0c053d879db2cb26ee05d +P dc7dd2d3e50e7cc474b22f1b5b219da32bcd7aa1ba56864d1dbcf0d3a6fa06f2 +R 241e4221ed9c630dbcac27e953db6022 +T *branch * dynamic_prompt +T *sym-dynamic_prompt * +T -sym-trunk * +U larrybr +Z 409cc86f921d8ebfc04f748d4dd15026 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6117ebbdad..010e305bbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc7dd2d3e50e7cc474b22f1b5b219da32bcd7aa1ba56864d1dbcf0d3a6fa06f2 \ No newline at end of file +dac2ddc287db7a68d0cd49b785060f62290868fbb1aa2ee09e54d3b1acfbf55f \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 0f8108e05f..f94496ba28 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -467,8 +467,93 @@ static char *Argv0; ** Prompt strings. Initialized in main. Settable with ** .prompt main continue */ -static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ -static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ +#define PROMPT_LEN_MAX 20 +/* First line prompt. default: "sqlite> " */ +static char mainPrompt[PROMPT_LEN_MAX]; +/* Continuation prompt. default: " ...> " */ +static char continuePrompt[PROMPT_LEN_MAX]; + +/* +** Optionally disable dynamic continuation prompt. +** Unless disabled, the continuation prompt shows open SQL lexemes if any, +** or open parentheses level if non-zero, or continuation prompt as set. +** This facility interacts with the scanner and process_input() where the +** below 5 macros are used. +*/ +#ifdef SQLITE_OMIT_DYNAPROMPT +# define CONTINUATION_PROMPT continuePrompt +# define CONTINUE_PROMPT_RESET(p) +# define CONTINUE_PROMPT_AWAITS(p,s) +# define CONTINUE_PROMPT_AWAITC(p,c) +# define CONTINUE_PAREN_INCR(p,n) +# define CONTINUE_PROMPT_STATE 0 +# define SCAN_TRACKER_REFTYPE void* +#else +# define CONTINUATION_PROMPT dynamicContinuePrompt() +# define CONTINUE_PROMPT_RESET(p) \ + if(p) (setLexemeOpen(p,0,0), trackParenLevel(p,0)) +# define CONTINUE_PROMPT_AWAITS(p,s) \ + if(p && stdin_is_interactive) setLexemeOpen(p, s, 0) +# define CONTINUE_PROMPT_AWAITC(p,c) \ + if(p && stdin_is_interactive) setLexemeOpen(p, 0, c) +# define CONTINUE_PAREN_INCR(p,n) \ + if(p && stdin_is_interactive) (trackParenLevel(p,n)) +# define SCAN_TRACKER_REFTYPE struct DynaPrompt * +# define CONTINUE_PROMPT_STATE ((SCAN_TRACKER_REFTYPE)&dynPrompt) + +static struct DynaPrompt { + char dynamicPrompt[PROMPT_LEN_MAX]; + char acAwait[2]; + int inParenLevel; + char *zScannerAwaits; +} dynPrompt = { {0}, {0}, 0, 0 }; + +/* Record parenthesis nesting level change, or force level to 0. */ +static void trackParenLevel(struct DynaPrompt *p, int ni){ + p->inParenLevel += ni; + if( ni==0 ) p->inParenLevel = 0; + p->zScannerAwaits = 0; +} + +/* Record that a lexeme is opened, or closed with args==0. */ +static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ + if( s!=0 || c==0 ){ + p->zScannerAwaits = s; + p->acAwait[0] = 0; + }else{ + p->acAwait[0] = c; + p->zScannerAwaits = p->acAwait; + } +} + +/* Upon demand, derive the continuation prompt to display. */ +static char *dynamicContinuePrompt(void){ + if( continuePrompt[0]==0 + || (dynPrompt.zScannerAwaits==0 && dynPrompt.inParenLevel == 0) ){ + return continuePrompt; + }else{ + if( dynPrompt.zScannerAwaits ){ + int ncp = strlen(continuePrompt), ndp = strlen(dynPrompt.zScannerAwaits); + if( ndp > ncp-3 ) return continuePrompt; + strncpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits, ndp); + while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; + strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + }else{ + if( dynPrompt.inParenLevel>9 ){ + strncpy(dynPrompt.dynamicPrompt, "(..", 3); + }else if( dynPrompt.inParenLevel<0 ){ + strncpy(dynPrompt.dynamicPrompt, ")x!", 3); + }else{ + strncpy(dynPrompt.dynamicPrompt, "(x.", 3); + dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); + } + strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); + } + } + return dynPrompt.dynamicPrompt; +} +#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ /* ** Render output like fprintf(). Except, if the output is going to the @@ -729,7 +814,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ if( in!=0 ){ zResult = local_getline(zPrior, in); }else{ - zPrompt = isContinuation ? continuePrompt : mainPrompt; + zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; #if SHELL_USE_LOCAL_GETLINE printf("%s", zPrompt); fflush(stdout); @@ -10907,7 +10992,8 @@ typedef enum { ** The scan is resumable for subsequent lines when prior ** return values are passed as the 2nd argument. */ -static QuickScanState quickscan(char *zLine, QuickScanState qss){ +static QuickScanState quickscan(char *zLine, QuickScanState qss, + SCAN_TRACKER_REFTYPE pst){ char cin; char cWait = (char)qss; /* intentional narrowing loss */ if( cWait==0 ){ @@ -10931,6 +11017,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ if( *zLine=='*' ){ ++zLine; cWait = '*'; + CONTINUE_PROMPT_AWAITS(pst, "/*"); qss = QSS_SETV(qss, cWait); goto TermScan; } @@ -10941,7 +11028,14 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ case '`': case '\'': case '"': cWait = cin; qss = QSS_HasDark | cWait; + CONTINUE_PROMPT_AWAITC(pst, cin); goto TermScan; + case '(': + CONTINUE_PAREN_INCR(pst, 1); + break; + case ')': + CONTINUE_PAREN_INCR(pst, -1); + break; default: break; } @@ -10957,6 +11051,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ continue; ++zLine; cWait = 0; + CONTINUE_PROMPT_AWAITC(pst, 0); qss = QSS_SETV(qss, 0); goto PlainScan; case '`': case '\'': case '"': @@ -10967,6 +11062,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ /* fall thru */ case ']': cWait = 0; + CONTINUE_PROMPT_AWAITC(pst, 0); qss = QSS_SETV(qss, 0); goto PlainScan; default: assert(0); @@ -10990,7 +11086,7 @@ static int line_is_command_terminator(char *zLine){ zLine += 2; /* SQL Server */ else return 0; - return quickscan(zLine, QSS_Start)==QSS_Start; + return quickscan(zLine, QSS_Start, 0)==QSS_Start; } /* @@ -11130,6 +11226,7 @@ static int process_input(ShellState *p){ } ++p->inputNesting; p->lineno = 0; + CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); zLine = one_input_line(p->in, zLine, nSql>0); @@ -11148,7 +11245,7 @@ static int process_input(ShellState *p){ && line_is_complete(zSql, nSql) ){ memcpy(zLine,";",2); } - qss = quickscan(zLine, qss); + qss = quickscan(zLine, qss, CONTINUE_PROMPT_STATE); if( QSS_PLAINWHITE(qss) && nSql==0 ){ /* Just swallow single-line whitespace */ echo_group_input(p, zLine); @@ -11156,6 +11253,7 @@ static int process_input(ShellState *p){ continue; } if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){ + CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); echo_group_input(p, zLine); if( zLine[0]=='.' ){ rc = do_meta_command(zLine, p); @@ -11191,6 +11289,7 @@ static int process_input(ShellState *p){ if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); + CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); nSql = 0; if( p->outCount ){ output_reset(p); @@ -11210,6 +11309,7 @@ static int process_input(ShellState *p){ /* This may be incomplete. Let the SQL parser deal with that. */ echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); + CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); } free(zSql); free(zLine); From 6b271abc9853e77a19bb208893c6e2a3924e9f50 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 6 Dec 2022 06:09:03 +0000 Subject: [PATCH 169/282] Add a demonstration sqlite3_vtab/module implemented in JS, based on ext/misc/templatevtab.c. Add oo1.selectArrays() and selectObjects(). FossilOrigin-Name: 60482c97e02bc4cafefef281be0cf0bc8c5c53232162829c137f3f7a80cdc534 --- ext/wasm/GNUmakefile | 2 +- ext/wasm/api/README.md | 9 +- ext/wasm/api/sqlite3-api-oo1.js | 29 ++ ext/wasm/api/sqlite3-api-prologue.js | 1 - ext/wasm/api/sqlite3-v-helper.js | 392 +++++++++++++++++++++++++++ ext/wasm/api/sqlite3-vfs-helper.js | 221 --------------- ext/wasm/jaccwabyt/jaccwabyt.js | 1 + ext/wasm/jaccwabyt/jaccwabyt.md | 3 +- ext/wasm/tester1.c-pp.js | 197 +++++++++++++- manifest | 26 +- manifest.uuid | 2 +- 11 files changed, 637 insertions(+), 246 deletions(-) create mode 100644 ext/wasm/api/sqlite3-v-helper.js delete mode 100644 ext/wasm/api/sqlite3-vfs-helper.js diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 968d4f440c..1b18437682 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -289,7 +289,7 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js sqlite3-api.jses += $(sqlite3-api-build-version.js) sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.js +sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index f9955a8957..ce34b26b8a 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -78,11 +78,10 @@ browser client: a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- **`sqlite3-vfs-helper.js`**\ - This internal-use-only file installs `sqlite3.VfsHelper` for use by - `sqlite3-*.js` files which create `sqlite3_vfs` implementations. - `sqlite3.VfsHelper` gets removed from the the `sqlite3` object after - the library is finished initializing. +- **`sqlite3-v-helper.js`**\ + Installs `sqlite3.VfsHelper` and `sqlite3.VtabHelper` for use by + downstream code which creates `sqlite3_vfs` and `sqlite3_module` + implementations. - **`sqlite3-vfs-opfs.c-pp.js`**\ is an sqlite3 VFS implementation which supports Google Chrome's Origin-Private FileSystem (OPFS) as a storage layer to provide diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index b377efc242..6253c659fc 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -473,6 +473,15 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return rc; }; + /** + Internal impl of the DB.selectArrays() and + selectObjects() methods. + */ + const __selectAll = + (db, sql, bind, rowMode)=>db.exec({ + sql, bind, rowMode, returnValue: 'resultRows' + }); + /** Expects to be given a DB instance or an `sqlite3*` pointer (may be null) and an sqlite3 API result code. If the result code is @@ -1098,6 +1107,26 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return __selectFirstRow(this, sql, bind, {}); }, + /** + Runs the given SQL and returns an array of all results, with + each row represented as an array, as per the 'array' `rowMode` + option to `exec()`. An empty result set resolves + to an empty array. The second argument, if any, is treated as + the 'bind' option to a call to exec(). + */ + selectArrays: function(sql,bind){ + return __selectAll(this, sql, bind, 'array'); + }, + + /** + Works identically to selectArrays() except that each value + in the returned array is an object, as per the 'object' `rowMode` + option to `exec()`. + */ + selectObjects: function(sql,bind){ + return __selectAll(this, sql, bind, 'object'); + }, + /** Returns the number of currently-opened Stmt handles for this db handle, or 0 if this DB instance is closed. diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index a51e957f8f..558fcda1e0 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1642,7 +1642,6 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( some initializers. Retain them when running in test mode so that we can add tests for them. */ delete sqlite3.util; - delete sqlite3.VfsHelper; delete sqlite3.StructBinder; } return sqlite3; diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js new file mode 100644 index 0000000000..84272266e8 --- /dev/null +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -0,0 +1,392 @@ +/* +** 2022-11-30 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +*/ + +/** + This file installs sqlite3.VfsHelper, and object which exists to + assist in the creation of JavaScript implementations of sqlite3_vfs, + along with its virtual table counterpart, sqlite3.VtabHelper. +*/ +'use strict'; +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss; + const vh = Object.create(null), vt = Object.create(null); + + sqlite3.VfsHelper = vh; + sqlite3.VtabHelper = vt; + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error, e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc. + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + + If tgt.ondispose is set before this is called then it _must_ + be an array, to which this function will append entries. + + ACHTUNG: because we cannot generically know how to transform JS + exceptions into result codes, the installed functions do no + automatic catching of exceptions. It is critical, to avoid + undefined behavior in the C layer, that methods mapped via + this function do not throw. The exception, as it were, to that + rule is... + + If applyArgcCheck is true then each method gets wrapped in a + proxy which asserts that it is passed the expected number of + arguments, throwing if the argument count does not match + expectations. That is only intended for dev-time usage for sanity + checking, and will leave the C environment in an undefined + state. For non-dev-time use, it is a given that the C API will + never call one of the generated function wrappers with the wrong + argument count. + */ + vh.installMethod = vt.installMethod = function callee( + tgt, name, func, applyArgcCheck = callee.installMethodArgcCheck + ){ + if(!(tgt instanceof sqlite3.StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + } + if(!callee.argcProxy){ + callee.argcProxy = function(tgt, funcName, func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch for", + tgt.structInfo.name+"::"+funcName + +": Native signature is:",sig); + } + return func.apply(this, args); + } + }; + /* An ondispose() callback for use with + sqlite3.StructBinder-created types. */ + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + const fProxy = applyArgcCheck + /** This middle-man proxy is only for use during development, to + confirm that we always pass the proper number of + arguments. We know that the C-level code will always use the + correct argument count. */ + ? callee.argcProxy(tgt, memKey, func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + }/*installMethod*/; + vh.installMethod.installMethodArgcCheck = false; + + /** + Installs methods into the given StructType-type instance. Each + entry in the given methods object must map to a known member of + the given StructType, else an exception will be triggered. See + installMethod() for more details, including the semantics of the + 3rd argument. + + As an exception to the above, if any two or more methods in the + 2nd argument are the exact same function, installMethod() is + _not_ called for the 2nd and subsequent instances, and instead + those instances get assigned the same method pointer which is + created for the first instance. This optimization is primarily to + accommodate special handling of sqlite3_module::xConnect and + xCreate methods. + + On success, returns this object. Throws on error. + */ + vh.installMethods = vt.installMethods = function( + structType, methods, applyArgcCheck = vh.installMethod.installMethodArgcCheck + ){ + const seen = new Map /* map of */; + for(const k of Object.keys(methods)){ + const m = methods[k]; + const prior = seen.get(m); + if(prior){ + const mkey = structType.memberKey(k); + structType[mkey] = structType[structType.memberKey(prior)]; + }else{ + vh.installMethod(structType, k, m, applyArgcCheck); + seen.set(m, k); + } + } + return this; + }; + + /** + Uses sqlite3_vfs_register() to register the + sqlite3.capi.sqlite3_vfs-type vfs, which must have already been + filled out properly. If the 2nd argument is truthy, the VFS is + registered as the default VFS, else it is not. + + On success, returns this object. Throws on error. + */ + vh.registerVfs = function(vfs, asDefault=false){ + if(!(vfs instanceof sqlite3.capi.sqlite3_vfs)){ + toss("Expecting a sqlite3_vfs-type argument."); + } + const rc = capi.sqlite3_vfs_register(vfs.pointer, asDefault ? 1 : 0); + if(rc){ + toss("sqlite3_vfs_register(",vfs,") failed with rc",rc); + } + if(vfs.pointer !== capi.sqlite3_vfs_find(vfs.$zName)){ + toss("BUG: sqlite3_vfs_find(vfs.$zName) failed for just-installed VFS", + vfs); + } + return this; + }; + + /** + A wrapper for installMethods() or registerVfs() to reduce + installation of a VFS and/or its I/O methods to a single + call. + + Accepts an object which contains the properties "io" and/or + "vfs", each of which is itself an object with following properties: + + - `struct`: an sqlite3.StructType-type struct. This must be a + populated (except for the methods) object of type + sqlite3_io_methods (for the "io" entry) or sqlite3_vfs (for the + "vfs" entry). + + - `methods`: an object mapping sqlite3_io_methods method names + (e.g. 'xClose') to JS implementations of those methods. The JS + implementations must be call-compatible with their native + counterparts. + + For each of those object, this function passes its (`struct`, + `methods`, (optional) `applyArgcCheck`) properties to + this.installMethods(). + + If the `vfs` entry is set then: + + - Its `struct` property is passed to this.registerVfs(). The + `vfs` entry may optionally have an `asDefault` property, which + gets passed as the 2nd argument to registerVfs(). + + - If `struct.$zName` is falsy and the entry has a string-type + `name` property, `struct.$zName` is set to the C-string form of + that `name` value before registerVfs() is called. + + On success returns this object. Throws on error. + */ + vh.installVfs = function(opt){ + let count = 0; + const propList = ['io','vfs']; + for(const key of propList){ + const o = opt[key]; + if(o){ + ++count; + this.installMethods(o.struct, o.methods, !!o.applyArgcCheck); + if('vfs'===key){ + if(!o.struct.$zName && 'string'===typeof o.name){ + o.struct.$zName = wasm.allocCString(o.name); + /* Note that we leak that C-string. */ + } + this.registerVfs(o.struct, !!o.asDefault); + } + } + } + if(!count) toss("Misuse: installVfs() options object requires at least", + "one of:", propList); + return this; + }; + + /** + Expects to be passed the (argc,argv) arguments of + sqlite3_module::xFilter(), or an equivalent API. This function + transforms the arguments (an array of (sqlite3_value*)) into a JS + array of equivalent JS values. It uses the same type conversions + as sqlite3_create_function_v2() and friends. Throws on error, + e.g. if it cannot figure out a sensible data conversion. + */ + vt.sqlite3ValuesToJs = capi.sqlite3_create_function_v2.udfConvertArgs; + + /** + Factory function for xyz2js() impls. + */ + const __v2jsFactory = function(structType){ + return function(ptr,remove=false){ + if(0===arguments.length) ptr = new structType; + if(ptr instanceof structType){ + //T.assert(!this.has(ptr.pointer)); + this.set(ptr.pointer, ptr); + return ptr; + }else if(!wasm.isPtr(ptr)){ + sqlite3.SQLite3Error.toss("Invalid argument to v2jsFactory"); + } + let rc = this.get(ptr); + if(remove) this.delete(ptr); + /*arguable else if(!rc){ + rc = new structType(ptr); + this.set(ptr, rc); + }*/ + return rc; + }.bind(new Map); + }; + /** + EXPERIMENTAL. DO NOT USE IN CLIENT CODE. + + Has 3 distinct uses: + + - vtab2js() instantiates a new capi.sqlite3_vtab instance, maps + its pointer for later by-pointer lookup, and returns that + object. This is intended to be called from + sqlite3_module::xConnect() or xCreate() implementations. + + - vtab2js(pVtab) accepts a WASM pointer to a C-level + (sqlite3_vtab*) instance and returns the capi.sqlite3_vtab + object created by the first form of this function, or undefined + if that form has not been used. This is intended to be called + from sqlite3_module methods which take a (sqlite3_vtab*) pointer + _except_ for xDisconnect(), in which case use... + + - vtab2js(pVtab,true) as for the previous form, but removes the + pointer-to-object mapping before returning. The caller must + call dispose() on the returned object. This is intended to be + called from sqlite3_module::xDisconnect() implementations or + in error handling of a failed xCreate() or xConnect(). + */ + vt.vtab2js = __v2jsFactory(capi.sqlite3_vtab); + + /** + EXPERIMENTAL. DO NOT USE IN CLIENT CODE. + + Works identically to vtab2js() except that it deals with + sqlite3_cursor objects and pointers instead of sqlite3_vtab. + + - vcur2js() is intended to be called from sqlite3_module::xOpen() + + - vcur2js(pCursor) is intended to be called from all sqlite3_module + methods which take a (sqlite3_vtab_cursor*) _except_ for + xClose(), in which case use... + + - vcur2js(pCursor, true) will remove the m apping of pCursor to a + capi.sqlite3_vtab_cursor object and return that object. The + caller must call dispose() on the returned object. This is + intended to be called form xClose() or in error handling of a + failed xOpen(). + */ + vt.vcur2js = __v2jsFactory(capi.sqlite3_vtab_cursor); + + /** + Given an error object, this function returns + sqlite3.capi.SQLITE_NOMEM if (e instanceof + sqlite3.WasmAllocError), else it returns its + second argument. Its intended usage is in the methods + of a sqlite3_vfs or sqlite3_module: + + ``` + try{ + let rc = ... + return rc; + }catch(e){ + return sqlite3.VtabHelper.exceptionToRc(e, sqlite3.capi.SQLITE_XYZ); + // where SQLITE_XYZ is some call-appropriate result code. + } + ``` + */ + /**vh.exceptionToRc = vt.exceptionToRc = + (e, defaultRc=capi.SQLITE_ERROR)=>( + (e instanceof sqlite3.WasmAllocError) + ? capi.SQLITE_NOMEM + : defaultRc + );*/ + + /** + Given an sqlite3_module method name and error object, this + function returns sqlite3.capi.SQLITE_NOMEM if (e instanceof + sqlite3.WasmAllocError), else it returns its second argument. Its + intended usage is in the methods of a sqlite3_vfs or + sqlite3_module: + + ``` + try{ + let rc = ... + return rc; + }catch(e){ + return sqlite3.VtabHelper.xMethodError( + 'xColumn', e, sqlite3.capi.SQLITE_XYZ); + // where SQLITE_XYZ is some call-appropriate result code + // defaulting to SQLITE_ERROR. + } + ``` + + If xMethodError.errorReporter is a function, it is called in + order to report the error, else the error is not reported. + If that function throws, that exception is ignored. + */ + vt.xMethodError = function f(methodName, err, defaultRc=capi.SQLITE_ERROR){ + if(f.errorReporter instanceof Function){ + try{f.errorReporter("sqlite3_module::"+methodName+"(): "+err.message);} + catch(e){/*ignored*/} + } + return (err instanceof sqlite3.WasmAllocError) + ? capi.SQLITE_NOMEM + : defaultRc; + }; + vt.xMethodError.errorReporter = 1 ? console.error.bind(console) : false; + + /** + "The problem" with this is that it introduces an outer function with + a different arity than the passed-in method callback. That means we + cannot do argc validation on these. Additionally, some methods (namely + xConnect) may have call-specific error handling. It would be a shame to + hard-coded that per-method support in this function. + */ + /** vt.methodCatcher = function(methodName, method, defaultErrRc=capi.SQLITE_ERROR){ + return function(...args){ + try { method(...args); } + }catch(e){ return vt.xMethodError(methodName, e, defaultRc) } + }; + */ + + /** + A helper for sqlite3_vtab::xRow() implementations. It must be + passed that function's 2nd argument and the value for that + pointer. Returns the same as wasm.setMemValue() and will throw + if the 1st or 2nd arguments are invalid for that function. + */ + vt.setRowId = (ppRowid64, value)=>wasm.setMemValue(ppRowid64, value, 'i64'); +}/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/api/sqlite3-vfs-helper.js b/ext/wasm/api/sqlite3-vfs-helper.js deleted file mode 100644 index f9d3c18c7b..0000000000 --- a/ext/wasm/api/sqlite3-vfs-helper.js +++ /dev/null @@ -1,221 +0,0 @@ -/* -** 2022-11-30 -** -** The author disclaims copyright to this source code. In place of a -** legal notice, here is a blessing: -** -** * May you do good and not evil. -** * May you find forgiveness for yourself and forgive others. -** * May you share freely, never taking more than you give. -*/ - -/** - This file installs sqlite.VfsHelper, an object which exists - to assist in the creation of JavaScript implementations of - sqlite3_vfs. It is NOT part of the public API, and is an - internal implemenation detail for use in this project's - own development of VFSes. It may be exposed to clients - at some point, provided there is value in doing so. -*/ -'use strict'; -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss; - const vh = Object.create(null); - - /** - Does nothing more than holds a permanent reference to each - argument. This is useful in some cases to ensure that, e.g., a - custom sqlite3_io_methods instance does not get - garbage-collected. - - Returns this object. - */ - vh.holdReference = function(...args){ - for(const v of args) this.refs.add(v); - return vh; - }.bind({refs: new Set}); - - /** - Installs a StructBinder-bound function pointer member of the - given name and function in the given StructType target object. - It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws - on the slightest hint of error, e.g. tgt is-not-a StructType, - name does not map to a struct-bound member, etc. - - If applyArgcCheck is true then each method gets wrapped in a - proxy which asserts that it is passed the expected number of - arguments, throwing if the argument count does not match - expectations. That is only recommended for dev-time usage for - sanity checking. Once a VFS implementation is known to be - working, it is a given that the C API will never call it with the - wrong argument count. - - Returns a proxy for this function which is bound to tgt and takes - 2 args (name,func). That function returns the same thing, - permitting calls to be chained. - - If called with only 1 arg, it has no side effects but returns a - func with the same signature as described above. - - If tgt.ondispose is set before this is called then it _must_ - be an array, to which this function will append entries. - */ - vh.installMethod = function callee(tgt, name, func, - applyArgcCheck=callee.installMethodArgcCheck){ - if(!(tgt instanceof sqlite3.StructBinder.StructType)){ - toss("Usage error: target object is-not-a StructType."); - } - if(1===arguments.length){ - return (n,f)=>callee(tgt, n, f, applyArgcCheck); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - /* An ondispose() callback for use with - sqlite3.StructBinder-created types. */ - callee.removeFuncList = function(){ - if(this.ondispose.__removeFuncList){ - this.ondispose.__removeFuncList.forEach( - (v,ndx)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - } - /* else it's a descriptive label for the next number in - the list. */ - } - ); - delete this.ondispose.__removeFuncList; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name); - if(sigN.length<2){ - toss("Member",name," is not a function pointer. Signature =",sigN); - } - const memKey = tgt.memberKey(name); - const fProxy = applyArgcCheck - /** This middle-man proxy is only for use during development, to - confirm that we always pass the proper number of - arguments. We know that the C-level code will always use the - correct argument count. */ - ? callee.argcProxy(func, sigN) - : func; - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose) tgt.ondispose = []; - if(!tgt.ondispose.__removeFuncList){ - tgt.ondispose.push('ondispose.__removeFuncList handler', - callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; - } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); - return (n,f)=>callee(tgt, n, f, applyArgcCheck); - }/*installMethod*/; - vh.installMethod.installMethodArgcCheck = false; - - /** - Installs methods into the given StructType-type object. Each - entry in the given methods object must map to a known member of - the given StructType, else an exception will be triggered. - See installMethod() for more details, including the semantics - of the 3rd argument. - - On success, passes its first argument to holdRefence() and - returns this object. Throws on error. - */ - vh.installMethods = function(structType, methods, - applyArgcCheck=vh.installMethod.installMethodArgcCheck){ - for(const k of Object.keys(methods)){ - vh.installMethod(structType, k, methods[k], applyArgcCheck); - } - return vh.holdReference(structType); - }; - - /** - Uses sqlite3_vfs_register() to register the - sqlite3.capi.sqlite3_vfs-type vfs, which must have already been - filled out properly. If the 2nd argument is truthy, the VFS is - registered as the default VFS, else it is not. - - On success, passes its first argument to this.holdReference() and - returns this object. Throws on error. - */ - vh.registerVfs = function(vfs, asDefault=false){ - if(!(vfs instanceof sqlite3.capi.sqlite3_vfs)){ - toss("Expecting a sqlite3_vfs-type argument."); - } - const rc = capi.sqlite3_vfs_register(vfs.pointer, asDefault ? 1 : 0); - if(rc){ - toss("sqlite3_vfs_register(",vfs,") failed with rc",rc); - } - if(vfs.pointer !== capi.sqlite3_vfs_find(vfs.$zName)){ - toss("BUG: sqlite3_vfs_find(vfs.$zName) failed for just-installed VFS", - vfs); - } - return vh.holdReference(vfs); - }; - - /** - A wrapper for installMethods() or registerVfs() to reduce - installation of a VFS and/or its I/O methods to a single - call. - - Accepts an object which contains the properties "io" and/or - "vfs", each of which is itself an object with following properties: - - - `struct`: an sqlite3.StructType-type struct. This must be a - populated (except for the methods) object of type - sqlite3_io_methods (for the "io" entry) or sqlite3_vfs (for the - "vfs" entry). - - - `methods`: an object mapping sqlite3_io_methods method names - (e.g. 'xClose') to JS implementations of those methods. - - For each of those object, this function passes its (`struct`, - `methods`, (optional) `applyArgcCheck`) properties to - this.installMethods(). - - If the `vfs` entry is set then: - - - Its `struct` property is passed to this.registerVfs(). The - `vfs` entry may optionally have an `asDefault` property, which - gets passed as the 2nd argument to registerVfs(). - - - If `struct.$zName` is falsy and the entry has a string-type - `name` property, `struct.$zName` is set to the C-string form of - that `name` value before registerVfs() is called. - - On success returns this object. Throws on error. - */ - vh.installVfs = function(opt){ - let count = 0; - const propList = ['io','vfs']; - for(const key of propList){ - const o = opt[key]; - if(o){ - ++count; - this.installMethods(o.struct, o.methods, !!o.applyArgcCheck); - if('vfs'===key){ - if(!o.struct.$zName && 'string'===typeof o.name){ - o.struct.$zName = wasm.allocCString(o.name); - /* Note that we leak that C-string. */ - } - this.registerVfs(o.struct, !!o.asDefault); - } - } - } - if(!count) toss("Misuse: installVfs() options object requires at least", - "one of:", propList); - return this; - }; - - sqlite3.VfsHelper = vh; -}/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index 0bd7869bc6..1fa6baf154 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -231,6 +231,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ obj.ondispose.forEach(function(x){ try{ if(x instanceof Function) x.call(obj); + else if(x instanceof StructType) x.dispose(); else if('number' === typeof x) dealloc(x); // else ignore. Strings are permitted to annotate entries // to assist in debugging. diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index c6cf5c8a25..17ada1edf0 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -426,7 +426,8 @@ simply passing a pointer to the constructor. For example: ```js const m = new MyStruct( functionReturningASharedPtr() ); -// calling m.dispose() will _not_ free the wrapped C-side instance. +// calling m.dispose() will _not_ free the wrapped C-side instance +// but will trigger any ondispose handler. ``` Now that we have struct instances, there are a number of things we diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 5db00cee64..6fcc895329 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -419,6 +419,7 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////// T.g('C/WASM Utilities') .t('sqlite3.wasm namespace', function(sqlite3){ + // TODO: break this into smaller individual test functions. const w = wasm; const chr = (x)=>x.charCodeAt(0); //log("heap getters..."); @@ -974,7 +975,7 @@ self.sqlite3InitModule = sqlite3InitModule; .t('Create db', function(sqlite3){ const dbFile = '/tester1.db'; wasm.sqlite3_wasm_vfs_unlink(0, dbFile); - const db = this.db = new sqlite3.oo1.DB(dbFile); + const db = this.db = new sqlite3.oo1.DB(dbFile, 0 ? 'ct' : 'c'); T.assert(Number.isInteger(db.pointer)) .mustThrowMatching(()=>db.pointer=1, /read-only/) .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) @@ -1197,6 +1198,29 @@ self.sqlite3InitModule = sqlite3InitModule; rc = db.selectArray('select a, b from t where b=-1'); T.assert(undefined === rc); }) + //////////////////////////////////////////////////////////////////////// + .t('selectArrays/Objects()', function(sqlite3){ + const db = this.db; + const sql = 'select a, b from t where a=? or b=? order by a'; + let rc = db.selectArrays(sql, [1, 4]); + T.assert(Array.isArray(rc)) + .assert(2===rc.length) + .assert(2===rc[0].length) + .assert(1===rc[0][0]) + .assert(2===rc[0][1]) + .assert(3===rc[1][0]) + .assert(4===rc[1][1]) + rc = db.selectArrays(sql, [99,99]); + T.assert(Array.isArray(rc)).assert(0===rc.length); + rc = db.selectObjects(sql, [1,4]); + T.assert(Array.isArray(rc)) + .assert(2===rc.length) + .assert('object' === typeof rc[1]) + .assert(1===rc[0].a) + .assert(2===rc[0].b) + .assert(3===rc[1].a) + .assert(4===rc[1].b); + }) //////////////////////////////////////////////////////////////////////// .t({ @@ -1503,6 +1527,173 @@ self.sqlite3InitModule = sqlite3InitModule; T.mustThrow(()=>db.exec("select * from foo.bar")); }) + //////////////////////////////////////////////////////////////////////// + .t({ + name: 'Custom virtual tables', + predicate: ()=>wasm.bigIntEnabled, + test: function(sqlite3){ + warn("The vtab/module JS bindings are experimental and subject to change."); + const vth = sqlite3.VtabHelper; + const tmplCols = Object.assign(Object.create(null),{ + A: 0, B: 1 + }); + /** + The vtab demonstrated here is a JS-ification of + ext/misc/templatevtab.c. + */ + const tmplMethods = { + xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ + try{ + const rc = capi.sqlite3_declare_vtab( + pDb, "CREATE TABLE ignored(a,b)" + ); + if(0===rc){ + const t = vth.vtab2js(); + wasm.setPtrValue(ppVtab, t.pointer); + T.assert(t === vth.vtab2js(wasm.getPtrValue(ppVtab))); + } + return rc; + }catch(e){ + if(!(e instanceof sqlite3.WasmAllocError)){ + wasm.setPtrValue(pzErr, wasm.allocCString(e.message)); + } + return vth.xMethodError('xConnect',e); + } + }, + xDisconnect: function(pVtab){ + try { + const t = vth.vtab2js(pVtab, true); + t.dispose(); + return 0; + }catch(e){ + return vth.xMethodError('xDisconnect',e); + } + }, + xOpen: function(pVtab, ppCursor){ + try{ + const t = vth.vtab2js(pVtab), c = vth.vcur2js(); + T.assert(t instanceof capi.sqlite3_vtab); + T.assert(c instanceof capi.sqlite3_vtab_cursor); + wasm.setPtrValue(ppCursor, c.pointer); + c._rowId = 0; + return 0; + }catch(e){ + return vth.xMethodError('xOpen',e); + } + }, + xClose: function(pCursor){ + try{ + const c = vth.vcur2js(pCursor,true); + T.assert(c instanceof capi.sqlite3_vtab_cursor) + .assert(!vth.vcur2js(pCursor)); + c.dispose(); + return 0; + }catch(e){ + return vth.xMethodError('xClose',e); + } + }, + xNext: function(pCursor){ + try{ + const c = vth.vcur2js(pCursor); + ++c._rowId; + return 0; + }catch(e){ + return vth.xMethodError('xNext',e); + } + + }, + xColumn: function(pCursor, pCtx, iCol){ + try{ + const c = vth.vcur2js(pCursor); + switch(iCol){ + case tmplCols.A: + capi.sqlite3_result_int(pCtx, 1000 + c._rowId); + break; + case tmplCols.B: + capi.sqlite3_result_int(pCtx, 2000 + c._rowId); + break; + default: sqlite3.SQLite3Error.toss("Invalid column id",iCol); + } + return 0; + }catch(e){ + return vth.xMethodError('xColumn',e); + } + }, + xRowid: function(pCursor, ppRowid64){ + try{ + const c = vth.vcur2js(pCursor); + vth.setRowId(ppRowid64, c._rowId); + return 0; + }catch(e){ + return vth.xMethodError('xRowid',e); + } + }, + xEof: function(pCursor){ + const c = vth.vcur2js(pCursor); + return c._rowId>=10; + }, + xFilter: function(pCursor, idxNum, idxCStr, + argc, argv/* [sqlite3_value* ...] */){ + try{ + const c = vth.vcur2js(pCursor); + c._rowId = 0; + return 0; + }catch(e){ + return vth.xMethodError('xFilter',e); + } + }, + xBestIndex: function(pVtab, pIdxInfo){ + try{ + const t = vth.vtab2js(pVtab); + const pii = new capi.sqlite3_index_info(pIdxInfo); + pii.$estimatedRows = 10; + pii.$estimatedCost = 10.0; + pii.dispose(); + return 0; + }catch(e){ + return vth.xMethodError('xBestIndex',e); + } + } + }; + /** + Problem to resolve: the vtab API places relevance on + whether xCreate and xConnect are exactly the same function + (same pointer address). Two JS-side references to the same + method will end up, without acrobatics to counter it, being + compiled as two different WASM-side bindings, i.e. two + different pointers. + + In order to account for this, VtabHelper.installMethods() + checks for duplicate function entries and maps them to the + same WASM-compiled instance + */ + if(1){ + tmplMethods.xCreate = tmplMethods.xConnect; + } + + const tmplMod = new sqlite3.capi.sqlite3_module(); + tmplMod.ondispose = []; + tmplMod.$iVersion = 0; + vth.installMethods(tmplMod, tmplMethods, true); + if(tmplMethods.xCreate){ + T.assert(tmplMod.$xCreate === tmplMod.$xConnect, + "installMethods() must avoid re-compiling identical functions"); + tmplMod.$xCreate = 0; + } + let rc = capi.sqlite3_create_module( + this.db, "testvtab", tmplMod, 0 + ); + this.db.checkRc(rc); + + const list = this.db.selectArrays( + "SELECT a,b FROM testvtab order by a" + ); + T.assert(10===list.length) + .assert(1000===list[0][0]) + .assert(2009===list[list.length-1][1]) + } + })/*vtab sanity checks*/ + //////////////////////////////////////////////////////////////////// .t({ name: 'C-side WASM tests (if compiled in)', @@ -1590,8 +1781,8 @@ self.sqlite3InitModule = sqlite3InitModule; }/* jaccwabyt-specific tests */) .t('Close db', function(){ - T.assert(this.db).assert(Number.isInteger(this.db.pointer)); - wasm.exports.sqlite3_wasm_db_reset(this.db.pointer); + T.assert(this.db).assert(wasm.isPtr(this.db.pointer)); + wasm.sqlite3_wasm_db_reset(this.db); this.db.close(); T.assert(!this.db.pointer); }) diff --git a/manifest b/manifest index a3f0d928c3..3a917c3196 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\stwo\sfeatures\sof\sjaccwabyt\swhich\swere\sfundamentally\sflawed,\salong\swith\sapprox.\s250\slines\sof\sunit\stests\swhich\sheavily\srelied\son\sthem.\sThankfully,\snone\sof\sthe\ssqlite3.js-level\scode\sused\sthose\sbits. -D 2022-12-05T15:05:46.306 +C Add\sa\sdemonstration\ssqlite3_vtab/module\simplemented\sin\sJS,\sbased\son\sext/misc/templatevtab.c.\sAdd\soo1.selectArrays()\sand\sselectObjects(). +D 2022-12-06T06:09:03.466 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,12 +491,12 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 +F ext/wasm/GNUmakefile 54c0db93a5493f625c0a993c12aee5d83951440eee03b2aecfc8aeb998182998 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api ffa70413409e922ce0f761779787a1d9100b34b43c8e3106bb7ccf2786a41326 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 20a256f4aaae80035d2bb1c9e3e0a125570313a8d137d427471d7be10edde87a +F ext/wasm/api/README.md 17fb1e10335cc87e366dec496c5b17b061f3f75cdf216e825258de34d97a3e53 F ext/wasm/api/extern-post-js.c-pp.js 8923f76c3d2213159e12d641dc750523ead5c848185dc4996fae5cc12397f88d F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 @@ -504,12 +504,12 @@ F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js c3a11e1d0e6fd381f68f9e76ad01f3616a6b809fbf9f5aa8e323955c128a6811 -F ext/wasm/api/sqlite3-api-oo1.js 793883953d4024e7b8c5ee1c7a6cb49c18ca53a1d235a203f93746f8907d32ba -F ext/wasm/api/sqlite3-api-prologue.js 815fef5ee93e1bb11ebec5a1d6a1b8ae2e47cfeb66dc5f6e93380ccce045f194 +F ext/wasm/api/sqlite3-api-oo1.js b970787aaf0bdd3a3df59cf66aeb84d0decaaa0529aed7eaf45121087181245f +F ext/wasm/api/sqlite3-api-prologue.js 31cffd8ce212fad8d316a08decd864b0f614c5fce3686153707dffe40f8279e8 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 -F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 +F ext/wasm/api/sqlite3-v-helper.js 4451763a0cd85734f0afe18b48918cb3c88ca99cef399b7c5f12119281e7b6a8 w ext/wasm/api/sqlite3-vfs-helper.js F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 723522a6c2a2463884a83fa1cc7ae5770deaaf0856a1058cc1023b2bfa1c898b @@ -539,8 +539,8 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff -F ext/wasm/jaccwabyt/jaccwabyt.js b7261221133cda8d363f16ddbac8e5b671fd51ce962fc34dc10e738a293b696d -F ext/wasm/jaccwabyt/jaccwabyt.md 72742a3205f1477de68086e7e4a854ed5f7d08dfcd6db54cdbea4dba4866f159 +F ext/wasm/jaccwabyt/jaccwabyt.js f4fc93375e9c40ef60f56cbecca1b4dc8bf4f53fab2c3abc860ed34890c5d32d +F ext/wasm/jaccwabyt/jaccwabyt.md 4bf62f7519857cdd674594aba7436cc4fae177eefbfaabc00740e16d9a828bee F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 -F ext/wasm/tester1.c-pp.js 8ed17c0e1f271e536cb7ccd86d3992785fc8bf2f94f9c2e0088ca670601ee087 +F ext/wasm/tester1.c-pp.js d96a77dbf0d8af11e3f3adb013bee2bfb5bf9410e3f5eade528d70104451dd10 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a329a809b5da135a9c251e4d5f637d45d01d0248110ac05f2ad8f01d9df38c64 -R 3d4ed37777b015bac598ce3a6734f5aa +P a190abc307847174f36421eaa3f47ef349c6f84a2bb35857fa64f64bbe722708 +R 5c37541d673cc882e16d46aec14f6ca8 U stephan -Z 002b2dd6406bdc41efd7e7f9f5d7918f +Z d89d1c25adf51ede4f60b7c8a7ea4184 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 14e600f31e..17c068d5db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a190abc307847174f36421eaa3f47ef349c6f84a2bb35857fa64f64bbe722708 \ No newline at end of file +60482c97e02bc4cafefef281be0cf0bc8c5c53232162829c137f3f7a80cdc534 \ No newline at end of file From 2a665edbd4816e0684389ecfdc63a5d5db9479a5 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 6 Dec 2022 06:16:11 +0000 Subject: [PATCH 170/282] Minor test tweaks. FossilOrigin-Name: f902f3b2c79d6c699ead1efeb1426e1e0f4ac709afdff88be1de62f34f3d5ccc --- ext/wasm/tester1.c-pp.js | 23 ++++++++++++----------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 6fcc895329..2866d34f59 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1656,16 +1656,16 @@ self.sqlite3InitModule = sqlite3InitModule; } }; /** - Problem to resolve: the vtab API places relevance on - whether xCreate and xConnect are exactly the same function - (same pointer address). Two JS-side references to the same - method will end up, without acrobatics to counter it, being - compiled as two different WASM-side bindings, i.e. two - different pointers. + The vtab API places relevance on whether xCreate and + xConnect are exactly the same function (same pointer + address). Two JS-side references to the same method will + end up, without acrobatics to counter it, being compiled as + two different WASM-side bindings, i.e. two different + pointers. In order to account for this, VtabHelper.installMethods() checks for duplicate function entries and maps them to the - same WASM-compiled instance + same WASM-compiled instance. */ if(1){ tmplMethods.xCreate = tmplMethods.xConnect; @@ -1676,8 +1676,9 @@ self.sqlite3InitModule = sqlite3InitModule; tmplMod.$iVersion = 0; vth.installMethods(tmplMod, tmplMethods, true); if(tmplMethods.xCreate){ - T.assert(tmplMod.$xCreate === tmplMod.$xConnect, - "installMethods() must avoid re-compiling identical functions"); + T.assert(tmplMod.$xCreate) + .assert(tmplMod.$xCreate === tmplMod.$xConnect, + "installMethods() must avoid re-compiling identical functions"); tmplMod.$xCreate = 0; } let rc = capi.sqlite3_create_module( @@ -1696,8 +1697,8 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////// .t({ - name: 'C-side WASM tests (if compiled in)', - predicate: haveWasmCTests, + name: 'C-side WASM tests', + predicate: ()=>(haveWasmCTests() || "Not compiled in."), test: function(){ const w = wasm, db = this.db; const stack = w.scopedAllocPush(); diff --git a/manifest b/manifest index 3a917c3196..b4b8f16bb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sdemonstration\ssqlite3_vtab/module\simplemented\sin\sJS,\sbased\son\sext/misc/templatevtab.c.\sAdd\soo1.selectArrays()\sand\sselectObjects(). -D 2022-12-06T06:09:03.466 +C Minor\stest\stweaks. +D 2022-12-06T06:16:11.079 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -509,7 +509,7 @@ F ext/wasm/api/sqlite3-api-prologue.js 31cffd8ce212fad8d316a08decd864b0f614c5fce F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 -F ext/wasm/api/sqlite3-v-helper.js 4451763a0cd85734f0afe18b48918cb3c88ca99cef399b7c5f12119281e7b6a8 w ext/wasm/api/sqlite3-vfs-helper.js +F ext/wasm/api/sqlite3-v-helper.js 4451763a0cd85734f0afe18b48918cb3c88ca99cef399b7c5f12119281e7b6a8 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 723522a6c2a2463884a83fa1cc7ae5770deaaf0856a1058cc1023b2bfa1c898b @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 -F ext/wasm/tester1.c-pp.js d96a77dbf0d8af11e3f3adb013bee2bfb5bf9410e3f5eade528d70104451dd10 +F ext/wasm/tester1.c-pp.js 411a120a4e57b5f43c3f7448d72c48b4fedb7b4b62efe0c617dd412c192c2810 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a190abc307847174f36421eaa3f47ef349c6f84a2bb35857fa64f64bbe722708 -R 5c37541d673cc882e16d46aec14f6ca8 +P 60482c97e02bc4cafefef281be0cf0bc8c5c53232162829c137f3f7a80cdc534 +R 72355a527ceb0556cf62fff969df2dd7 U stephan -Z d89d1c25adf51ede4f60b7c8a7ea4184 +Z 77f81bcdf15e49ab52ab2cf2ea76255f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 17c068d5db..7d693291c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60482c97e02bc4cafefef281be0cf0bc8c5c53232162829c137f3f7a80cdc534 \ No newline at end of file +f902f3b2c79d6c699ead1efeb1426e1e0f4ac709afdff88be1de62f34f3d5ccc \ No newline at end of file From b849832a79271c8743ddcd455d6bfcaca88433d0 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 6 Dec 2022 08:21:23 +0000 Subject: [PATCH 171/282] Minor internal JS code/docs cleanups. FossilOrigin-Name: 21331bdd36a91b07a687ffadce392dcf2ccd0fd824b35d9dd027d4289a40fc96 --- ext/wasm/api/sqlite3-api-glue.js | 4 ++-- ext/wasm/common/whwasmutil.js | 17 +++++++++++++---- ext/wasm/jaccwabyt/jaccwabyt.js | 14 +++++++++----- ext/wasm/jaccwabyt/jaccwabyt.md | 18 +++++++++--------- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 43 insertions(+), 30 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 0516953f10..89c8044be1 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -31,9 +31,9 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ heap: 0 ? wasm.memory : wasm.heap8u, alloc: wasm.alloc, dealloc: wasm.dealloc, - functionTable: wasm.functionTable, bigIntEnabled: wasm.bigIntEnabled, - memberPrefix: '$' + memberPrefix: /* Never change this: this prefix is baked into any + amount of code and client-facing docs. */ '$' }); delete self.Jaccwabyt; diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 9bce82f1a9..ea344c6ea3 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -943,10 +943,19 @@ self.WhWasmUtilInstaller = function(target){ const __allocCStr = function(jstr, returnWithLength, allocator, funcName){ __affirmAlloc(target, funcName); if('string'!==typeof jstr) return null; - const n = target.jstrlen(jstr), - ptr = allocator(n+1); - target.jstrcpy(jstr, target.heap8u(), ptr, n+1, true); - return returnWithLength ? [ptr, n] : ptr; + if(0){/* older impl, possibly more widely compatible? */ + const n = target.jstrlen(jstr), + ptr = allocator(n+1); + target.jstrcpy(jstr, target.heap8u(), ptr, n+1, true); + return returnWithLength ? [ptr, n] : ptr; + }else{/* newer, (probably) faster and (certainly) simpler impl */ + const u = cache.utf8Encoder.encode(jstr), + ptr = allocator(u.length+1), + heap = heapWrappers().HEAP8U; + heap.set(u, ptr); + heap[ptr + u.length] = 0; + return returnWithLength ? [ptr, u.length] : ptr; + } }; /** diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index 1fa6baf154..d5225f6e4d 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -61,7 +61,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ BigInt = self['BigInt'], BigInt64Array = self['BigInt64Array'], /* Undocumented (on purpose) config options: */ - functionTable = config.functionTable/*EXPERIMENTAL, undocumented*/, ptrSizeof = config.ptrSizeof || 4, ptrIR = config.ptrIR || 'i32' ; @@ -258,7 +257,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ iterable: false, value: v}}; /** Allocates obj's memory buffer based on the size defined in - DEF.sizeof. */ + ctor.structInfo.sizeof. */ const __allocStruct = function(ctor, obj, m){ let fill = !m; if(m) Object.defineProperty(obj, xPtrPropName, rop(m)); @@ -295,7 +294,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ if tossIfNotFound is true, else returns undefined if not found. The given name may be either the name of the structInfo.members key (faster) or the key as modified by the - memberPrefix/memberSuffix settings. + memberPrefix and memberSuffix settings. */ const __lookupMember = function(structInfo, memberName, tossIfNotFound=true){ let m = structInfo.members[memberName]; @@ -423,8 +422,9 @@ self.Jaccwabyt = function StructBinderFactory(config){ const mem = alloc(u.length+1); if(!mem) toss("Allocation error while duplicating string:",str); const h = heap(); - let i = 0; - for( ; i < u.length; ++i ) h[mem + i] = u[i]; + //let i = 0; + //for( ; i < u.length; ++i ) h[mem + i] = u[i]; + h.set(u, mem); h[mem + u.length] = 0; //log("allocCString @",mem," =",u); return mem; @@ -436,6 +436,10 @@ self.Jaccwabyt = function StructBinderFactory(config){ to free any prior memory, if appropriate. The newly-allocated string is added to obj.ondispose so will be freed when the object is disposed. + + The given name may be either the name of the structInfo.members + key (faster) or the key as modified by the memberPrefix and + memberSuffix settings. */ const __setMemberCString = function(obj, memberName, str){ const m = __lookupMember(obj.structInfo, memberName, true); diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index 17ada1edf0..49e2d3b8ff 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -205,7 +205,6 @@ simply look like: The StructBinder factory function returns a function which can then be used to create bindings for our structs. - Step 2: Create a Struct Description ------------------------------------------------------------ @@ -606,14 +605,15 @@ legally be called on concrete struct instances unless noted otherwise: - If it is a function, it is called with the struct object as its `this`. That method must not throw - if it does, the exception will be ignored. - - If it is an array, it may contain functions, pointers, and/or JS - strings. If an entry is a function, it is called as described - above. If it's a number, it's assumed to be a pointer and is - passed to the `dealloc()` function configured for the parent - [StructBinder][]. If it's a JS string, it's assumed to be a - helpful description of the next entry in the list and is simply - ignored. Strings are supported primarily for use as debugging - information. + - If it is an array, it may contain functions, pointers, other + [StructType] instances, and/or JS strings. If an entry is a + function, it is called as described above. If it's a number, it's + assumed to be a pointer and is passed to the `dealloc()` function + configured for the parent [StructBinder][]. If it's a + [StructType][] instance then its `dispose()` method is called. If + it's a JS string, it's assumed to be a helpful description of the + next entry in the list and is simply ignored. Strings are + supported primarily for use as debugging information. - Some struct APIs will manipulate the `ondispose` member, creating it as an array or converting it from a function to array as needed. diff --git a/manifest b/manifest index 14bb688eee..92dde177a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sinto\swasm-vtab\sbranch. -D 2022-12-06T06:21:02.190 +C Minor\sinternal\sJS\scode/docs\scleanups. +D 2022-12-06T08:21:23.652 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,7 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js c3a11e1d0e6fd381f68f9e76ad01f3616a6b809fbf9f5aa8e323955c128a6811 +F ext/wasm/api/sqlite3-api-glue.js 057c9ed555ee68ee795dc61e8d923862a0ddeb3d4cbad89866515cf5a1b77343 F ext/wasm/api/sqlite3-api-oo1.js b970787aaf0bdd3a3df59cf66aeb84d0decaaa0529aed7eaf45121087181245f F ext/wasm/api/sqlite3-api-prologue.js 31cffd8ce212fad8d316a08decd864b0f614c5fce3686153707dffe40f8279e8 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f @@ -521,7 +521,7 @@ F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c07 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css 0ff15602a3ab2bad8aef2c3bd120c7ee3fd1c2054ad2ace7e214187ae68d926f -F ext/wasm/common/whwasmutil.js 0de1e72494d52185d518892a3ac95d38b8e295d3699b64ddb36a3d46c11c8346 +F ext/wasm/common/whwasmutil.js cedbdb2162db129fd95951025572e087066d5457adc27ffd8083610a26306fc9 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 @@ -539,8 +539,8 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff -F ext/wasm/jaccwabyt/jaccwabyt.js f4fc93375e9c40ef60f56cbecca1b4dc8bf4f53fab2c3abc860ed34890c5d32d -F ext/wasm/jaccwabyt/jaccwabyt.md 4bf62f7519857cdd674594aba7436cc4fae177eefbfaabc00740e16d9a828bee +F ext/wasm/jaccwabyt/jaccwabyt.js 35c7eaa61ba4b875cd49da5a06a35d9935fd19121de65dd5f467cbe376359782 +F ext/wasm/jaccwabyt/jaccwabyt.md 888aff20e486abb6c955d432f9a3af271e2cdd2cd99a92c6f87077116ca57091 F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f902f3b2c79d6c699ead1efeb1426e1e0f4ac709afdff88be1de62f34f3d5ccc f41f18b1c3c950565ee3c237aebb51cfc3241813c6425813a8217e07aa0153a6 -R 8bcaf9d3b9f238bc8e19a8bbe1bb65e6 +P d106edb956bbe22bddc2c6e5bcdf4fb8f797794c725c2e359e3fa4b31d881843 +R 9cc98a74b7c97dbae209498a8d29cfcb U stephan -Z 78f5f82441cc43a02424eee016e46cb3 +Z 5e0b6a6419517a5a90c958328ab9c938 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1b960ebdc5..7afb1aff37 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d106edb956bbe22bddc2c6e5bcdf4fb8f797794c725c2e359e3fa4b31d881843 \ No newline at end of file +21331bdd36a91b07a687ffadce392dcf2ccd0fd824b35d9dd027d4289a40fc96 \ No newline at end of file From 671386c6376b00363ef24fe8732d9e8de1078a7d Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 6 Dec 2022 08:39:17 +0000 Subject: [PATCH 172/282] Add wasm.cArgvToJs() to support sqlite3_module::xConnect(). FossilOrigin-Name: c3ebdccf94d5e63c229bf91056c08052d78732e663334070ef3b0ef6fb4bfb8f --- ext/wasm/common/whwasmutil.js | 21 +++++++++++++++++++++ ext/wasm/tester1.c-pp.js | 5 +++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index ea344c6ea3..571486912f 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -1131,6 +1131,27 @@ self.WhWasmUtilInstaller = function(target){ */ target.allocMainArgv = (list)=>__allocMainArgv(false, list); + /** + Expects to be given a C-style string array and its length. It + returns a JS array of strings and/or nulls: any entry in the + pArgv array which is NULL results in a null entry in the result + array. If argc is 0 then an empty array is returned. + + Results are undefined if any entry in the first argc entries of + pArgv are neither 0 (NULL) nor legal UTF-format C strings. + + To be clear, the expected C-style arguments to be passed to this + function are `(int, char **)` (optionally const-qualified). + */ + target.cArgvToJs = (argc, pArgv)=>{ + const list = []; + for(let i = 0; i < argc; ++i){ + const arg = target.getPtrValue(pArgv + (target.ptrSizeof * i)); + list.push( arg ? target.cstringToJs(arg) : null ); + } + return list; + }; + /** Wraps function call func() in a scopedAllocPush() and scopedAllocPop() block, such that all calls to scopedAlloc() and diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 2866d34f59..8ba88db18f 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1544,6 +1544,11 @@ self.sqlite3InitModule = sqlite3InitModule; const tmplMethods = { xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ try{ + const args = wasm.cArgvToJs(argc, argv); + T.assert(args.length>=3) + .assert(args[0] === 'testvtab') + .assert(args[1] === 'main') + .assert(args[2] === 'testvtab'); const rc = capi.sqlite3_declare_vtab( pDb, "CREATE TABLE ignored(a,b)" ); diff --git a/manifest b/manifest index 92dde177a1..d416956d91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\sJS\scode/docs\scleanups. -D 2022-12-06T08:21:23.652 +C Add\swasm.cArgvToJs()\sto\ssupport\ssqlite3_module::xConnect(). +D 2022-12-06T08:39:17.647 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c07 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css 0ff15602a3ab2bad8aef2c3bd120c7ee3fd1c2054ad2ace7e214187ae68d926f -F ext/wasm/common/whwasmutil.js cedbdb2162db129fd95951025572e087066d5457adc27ffd8083610a26306fc9 +F ext/wasm/common/whwasmutil.js 8be1a3a3ebca51ab5d02fc89f6d45f974b99f0d99262227094e8af4b7ade3234 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 -F ext/wasm/tester1.c-pp.js 411a120a4e57b5f43c3f7448d72c48b4fedb7b4b62efe0c617dd412c192c2810 +F ext/wasm/tester1.c-pp.js 51d396bc0a4e6abc700def5529ce290ea84ba2018741fc7e701c5162b0f1e5fb F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d106edb956bbe22bddc2c6e5bcdf4fb8f797794c725c2e359e3fa4b31d881843 -R 9cc98a74b7c97dbae209498a8d29cfcb +P 21331bdd36a91b07a687ffadce392dcf2ccd0fd824b35d9dd027d4289a40fc96 +R f99279163c9ea24b92a1c120dee81c3c U stephan -Z 5e0b6a6419517a5a90c958328ab9c938 +Z 5af57eae4e4ffe17648ac57b69cb7a17 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7afb1aff37..9e31dbe0cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21331bdd36a91b07a687ffadce392dcf2ccd0fd824b35d9dd027d4289a40fc96 \ No newline at end of file +c3ebdccf94d5e63c229bf91056c08052d78732e663334070ef3b0ef6fb4bfb8f \ No newline at end of file From 75435f8b2ddff951d48cbe5271f3549b0b9f3b5a Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 6 Dec 2022 08:46:39 +0000 Subject: [PATCH 173/282] Rename wasm.cstringToJs() to wasm.cstrToJs() for consistency with other wasm.cstr... APIs. FossilOrigin-Name: cbf483ea0ba3e6dc08ad7ed654380f818544b4c3cedfdb8aa848a83298268ceb --- ext/wasm/api/sqlite3-api-glue.js | 14 +++++++------- ext/wasm/api/sqlite3-api-oo1.js | 6 +++--- ext/wasm/api/sqlite3-api-prologue.js | 6 +++--- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 8 ++++---- ext/wasm/common/whwasmutil.js | 14 +++++++------- ext/wasm/tester1.c-pp.js | 6 +++--- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- 8 files changed, 39 insertions(+), 39 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 89c8044be1..14c0d58a42 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -193,8 +193,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try { let aVals = [], aNames = [], i = 0, offset = 0; for( ; i < nCols; offset += (wasm.ptrSizeof * ++i) ){ - aVals.push( wasm.cstringToJs(wasm.getPtrValue(pColVals + offset)) ); - aNames.push( wasm.cstringToJs(wasm.getPtrValue(pColNames + offset)) ); + aVals.push( wasm.cstrToJs(wasm.getPtrValue(pColVals + offset)) ); + aNames.push( wasm.cstrToJs(wasm.getPtrValue(pColNames + offset)) ); } rc = callback(pVoid, nCols, aVals, aNames) | 0; /* The first 2 args of the callback are useless for JS but @@ -610,7 +610,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ toss("Maintenance required: increase sqlite3_wasm_enum_json()'s", "static buffer size!"); } - wasm.ctype = JSON.parse(wasm.cstringToJs(cJson)); + wasm.ctype = JSON.parse(wasm.cstrToJs(cJson)); //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); const defineGroups = ['access', 'authorizer', 'blobFinalizers', 'dataTypes', @@ -702,7 +702,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try { const zXKey = kvvfsMakeKey(zClass,zKey); if(!zXKey) return -3/*OOM*/; - const jKey = wasm.cstringToJs(zXKey); + const jKey = wasm.cstrToJs(zXKey); const jV = kvvfsStorage(zClass).getItem(jKey); if(!jV) return -1; const nV = jV.length /* Note that we are relying 100% on v being @@ -731,8 +731,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try { const zXKey = kvvfsMakeKey(zClass,zKey); if(!zXKey) return 1/*OOM*/; - const jKey = wasm.cstringToJs(zXKey); - kvvfsStorage(zClass).setItem(jKey, wasm.cstringToJs(zData)); + const jKey = wasm.cstrToJs(zXKey); + kvvfsStorage(zClass).setItem(jKey, wasm.cstrToJs(zData)); return 0; }catch(e){ console.error("kvstorageWrite()",e); @@ -746,7 +746,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try { const zXKey = kvvfsMakeKey(zClass,zKey); if(!zXKey) return 1/*OOM*/; - kvvfsStorage(zClass).removeItem(wasm.cstringToJs(zXKey)); + kvvfsStorage(zClass).removeItem(wasm.cstrToJs(zXKey)); return 0; }catch(e){ console.error("kvstorageDelete()",e); diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 6253c659fc..4ec61af07c 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -73,7 +73,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(capi.SQLITE_TRACE_STMT===t){ // x == SQL, p == sqlite3_stmt* console.log("SQL TRACE #"+(++this.counter), - wasm.cstringToJs(x)); + wasm.cstrToJs(x)); } }.bind({counter: 0})); @@ -139,7 +139,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ console.error("Invalid DB ctor args",opt,arguments); toss3("Invalid arguments for DB constructor."); } - let fnJs = ('number'===typeof fn) ? wasm.cstringToJs(fn) : fn; + let fnJs = ('number'===typeof fn) ? wasm.cstrToJs(fn) : fn; const vfsCheck = ctor._name2vfs[fnJs]; if(vfsCheck){ vfsName = vfsCheck.vfs; @@ -600,7 +600,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ); if(pVfs){ const v = new capi.sqlite3_vfs(pVfs); - try{ rc = wasm.cstringToJs(v.$zName) } + try{ rc = wasm.cstrToJs(v.$zName) } finally { v.dispose() } } return rc; diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 558fcda1e0..bccae8b199 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -348,13 +348,13 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** If v is-a Array, its join("") result is returned. If isSQLableTypedArray(v) is true then typedArrayToString(v) is - returned. If it looks like a WASM pointer, wasm.cstringToJs(v) is + returned. If it looks like a WASM pointer, wasm.cstrToJs(v) is returned. Else v is returned as-is. */ const flexibleString = function(v){ if(isSQLableTypedArray(v)) return typedArrayToString(v); else if(Array.isArray(v)) return v.join(""); - else if(wasm.isPtr(v)) v = wasm.cstringToJs(v); + else if(wasm.isPtr(v)) v = wasm.cstrToJs(v); return v; }; @@ -1339,7 +1339,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( let pVfs = capi.sqlite3_vfs_find(0); while(pVfs){ const oVfs = new capi.sqlite3_vfs(pVfs); - rc.push(wasm.cstringToJs(oVfs.$zName)); + rc.push(wasm.cstrToJs(oVfs.$zName)); pVfs = oVfs.$pNext; oVfs.dispose(); } diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index f5a1eb6cc2..99d79641f4 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -803,7 +803,7 @@ const installOpfsVfs = function callee(options){ const vfsSyncWrappers = { xAccess: function(pVfs,zName,flags,pOut){ mTimeStart('xAccess'); - const rc = opRun('xAccess', wasm.cstringToJs(zName)); + const rc = opRun('xAccess', wasm.cstrToJs(zName)); wasm.setMemValue( pOut, (rc ? 0 : 1), 'i32' ); mTimeEnd(); return 0; @@ -823,7 +823,7 @@ const installOpfsVfs = function callee(options){ }, xDelete: function(pVfs, zName, doSyncDir){ mTimeStart('xDelete'); - opRun('xDelete', wasm.cstringToJs(zName), doSyncDir, false); + opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); /* We're ignoring errors because we cannot yet differentiate between harmless and non-harmless failures. */ mTimeEnd(); @@ -855,7 +855,7 @@ const installOpfsVfs = function callee(options){ C-string here. */ opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP; } - zName = wasm.cstringToJs(zName); + zName = wasm.cstrToJs(zName); } const fh = Object.create(null); fh.fid = pFile; @@ -1231,7 +1231,7 @@ const installOpfsVfs = function callee(options){ const readBuf = wasm.scopedAlloc(16); rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); wasm.setMemValue(readBuf+6,0); - let jRead = wasm.cstringToJs(readBuf); + let jRead = wasm.cstrToJs(readBuf); log("xRead() got:",jRead); if("sanity"!==jRead) toss("Unexpected xRead() value."); if(vfsSyncWrappers.xSleep){ diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 571486912f..0da1841b6e 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -756,7 +756,7 @@ self.WhWasmUtilInstaller = function(target){ JS-format string representing its contents. As a special case, if ptr is falsy or not a pointer, `null` is returned. */ - target.cstringToJs = function(ptr){ + target.cstrToJs = function(ptr){ const n = target.cstrlen(ptr); return n ? __utf8Decode(heapWrappers().HEAP8U, ptr, ptr+n) : (null===n ? n : ""); }; @@ -1147,7 +1147,7 @@ self.WhWasmUtilInstaller = function(target){ const list = []; for(let i = 0; i < argc; ++i){ const arg = target.getPtrValue(pArgv + (target.ptrSizeof * i)); - list.push( arg ? target.cstringToJs(arg) : null ); + list.push( arg ? target.cstrToJs(arg) : null ); } return list; }; @@ -1315,14 +1315,14 @@ self.WhWasmUtilInstaller = function(target){ if('string'===typeof v) return target.scopedAllocCString(v); return v ? xcv.arg[ptrIR](v) : null; }; - xcv.result.string = xcv.result.utf8 = (i)=>target.cstringToJs(i); + xcv.result.string = xcv.result.utf8 = (i)=>target.cstrToJs(i); xcv.result['string:dealloc'] = xcv.result['utf8:dealloc'] = (i)=>{ - try { return i ? target.cstringToJs(i) : null } + try { return i ? target.cstrToJs(i) : null } finally{ target.dealloc(i) } }; - xcv.result.json = (i)=>JSON.parse(target.cstringToJs(i)); + xcv.result.json = (i)=>JSON.parse(target.cstrToJs(i)); xcv.result['json:dealloc'] = (i)=>{ - try{ return i ? JSON.parse(target.cstringToJs(i)) : null } + try{ return i ? JSON.parse(target.cstrToJs(i)) : null } finally{ target.dealloc(i) } } xcv.result['void'] = (v)=>undefined; @@ -1460,7 +1460,7 @@ self.WhWasmUtilInstaller = function(target){ ```js target.xWrap.resultAdapter('string:my_free',(i)=>{ - try { return i ? target.cstringToJs(i) : null } + try { return i ? target.cstrToJs(i) : null } finally{ target.exports.my_free(i) } }; ``` diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 8ba88db18f..c32c4e5e84 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -547,13 +547,13 @@ self.sqlite3InitModule = sqlite3InitModule; let cpy = w.scopedAlloc(n+10); let rc = w.cstrncpy(cpy, cStr, n+10); T.assert(n+1 === rc). - assert("hello" === w.cstringToJs(cpy)). + assert("hello" === w.cstrToJs(cpy)). assert(chr('o') === w.getMemValue(cpy+n-1)). assert(0 === w.getMemValue(cpy+n)); let cStr2 = w.scopedAllocCString("HI!!!"); rc = w.cstrncpy(cpy, cStr2, 3); T.assert(3===rc). - assert("HI!lo" === w.cstringToJs(cpy)). + assert("HI!lo" === w.cstrToJs(cpy)). assert(chr('!') === w.getMemValue(cpy+2)). assert(chr('l') === w.getMemValue(cpy+3)); }finally{ @@ -675,7 +675,7 @@ self.sqlite3InitModule = sqlite3InitModule; // 'string:flexible' argAdapter() sanity checks... w.scopedAllocCall(()=>{ const argAd = w.xWrap.argAdapter('string:flexible'); - const cj = (v)=>w.cstringToJs(argAd(v)); + const cj = (v)=>w.cstrToJs(argAd(v)); T.assert('Hi' === cj('Hi')) .assert('hi' === cj(['h','i'])) .assert('HI' === cj(new Uint8Array([72, 73]))); diff --git a/manifest b/manifest index d416956d91..cd9692c393 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\swasm.cArgvToJs()\sto\ssupport\ssqlite3_module::xConnect(). -D 2022-12-06T08:39:17.647 +C Rename\swasm.cstringToJs()\sto\swasm.cstrToJs()\sfor\sconsistency\swith\sother\swasm.cstr...\sAPIs. +D 2022-12-06T08:46:39.353 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,14 +503,14 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js 057c9ed555ee68ee795dc61e8d923862a0ddeb3d4cbad89866515cf5a1b77343 -F ext/wasm/api/sqlite3-api-oo1.js b970787aaf0bdd3a3df59cf66aeb84d0decaaa0529aed7eaf45121087181245f -F ext/wasm/api/sqlite3-api-prologue.js 31cffd8ce212fad8d316a08decd864b0f614c5fce3686153707dffe40f8279e8 +F ext/wasm/api/sqlite3-api-glue.js 8fa55af37c9880f94a803f32591dc0304750cc2f750048daf41fe942757bee64 +F ext/wasm/api/sqlite3-api-oo1.js 416e6398721a4cbb80ddfa3d7b303216790f1d344efdbbc36239d39abc66aa27 +F ext/wasm/api/sqlite3-api-prologue.js a7596c30392d9ca8f5b7c14feb4e6788107d1159fd5f90eb26708d653d36c9bc F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 F ext/wasm/api/sqlite3-v-helper.js 4451763a0cd85734f0afe18b48918cb3c88ca99cef399b7c5f12119281e7b6a8 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e9656232ae4f5ef2d2fdd1711e89d40ba81c8f0faed0aadb22004341da5c8184 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 723522a6c2a2463884a83fa1cc7ae5770deaaf0856a1058cc1023b2bfa1c898b F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -521,7 +521,7 @@ F ext/wasm/c-pp.c 92285f7bce67ed7b7020b40fde8ed0982c442b63dc33df9dfd4b658d4a6c07 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css 0ff15602a3ab2bad8aef2c3bd120c7ee3fd1c2054ad2ace7e214187ae68d926f -F ext/wasm/common/whwasmutil.js 8be1a3a3ebca51ab5d02fc89f6d45f974b99f0d99262227094e8af4b7ade3234 +F ext/wasm/common/whwasmutil.js f0a742270b490748b9fdb0974287429b036698609b40eee81f13fe13e64358a7 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 -F ext/wasm/tester1.c-pp.js 51d396bc0a4e6abc700def5529ce290ea84ba2018741fc7e701c5162b0f1e5fb +F ext/wasm/tester1.c-pp.js 81c40395be4a7e97a3ca0ff62a4e02204239b1db6f24fe15b3c96a17fb41f29b F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21331bdd36a91b07a687ffadce392dcf2ccd0fd824b35d9dd027d4289a40fc96 -R f99279163c9ea24b92a1c120dee81c3c +P c3ebdccf94d5e63c229bf91056c08052d78732e663334070ef3b0ef6fb4bfb8f +R 89bdf08b4a75fdd12936e8e156f340f6 U stephan -Z 5af57eae4e4ffe17648ac57b69cb7a17 +Z 095ae4e504a3bccb462600500450b6bf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9e31dbe0cc..143d7cba7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3ebdccf94d5e63c229bf91056c08052d78732e663334070ef3b0ef6fb4bfb8f \ No newline at end of file +cbf483ea0ba3e6dc08ad7ed654380f818544b4c3cedfdb8aa848a83298268ceb \ No newline at end of file From d254db53e57cbe18858f38f64b74c9c054f2aa5c Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 6 Dec 2022 09:49:04 +0000 Subject: [PATCH 174/282] Remove deprecated symbol sqlite3.opfs.OpfsDb, which was renamed to sqlite3.oo1.OpfsDb on 2022-11-29. FossilOrigin-Name: 0ce51bed65d5e430364f74bf959fb76c42ac5eec0769490231d8c8110a1f388c --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 7 ++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 99d79641f4..49ebfbd80b 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -1156,11 +1156,8 @@ const installOpfsVfs = function callee(options){ opt.vfs = opfsVfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; - sqlite3.oo1.OpfsDb = - opfsUtil.OpfsDb /* sqlite3.opfs.OpfsDb => deprecated name - - will be phased out Real Soon */ = - OpfsDb; OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsDb = OpfsDb; sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, function(oo1Db, sqlite3){ @@ -1185,7 +1182,7 @@ const installOpfsVfs = function callee(options){ ], 0, 0, 0); } ); - } + }/*extend sqlite3.oo1*/ const sanityCheck = function(){ const scope = wasm.scopedAllocPush(); diff --git a/manifest b/manifest index cd9692c393..7c57dc1324 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\swasm.cstringToJs()\sto\swasm.cstrToJs()\sfor\sconsistency\swith\sother\swasm.cstr...\sAPIs. -D 2022-12-06T08:46:39.353 +C Remove\sdeprecated\ssymbol\ssqlite3.opfs.OpfsDb,\swhich\swas\srenamed\sto\ssqlite3.oo1.OpfsDb\son\s2022-11-29. +D 2022-12-06T09:49:04.292 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1 F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 F ext/wasm/api/sqlite3-v-helper.js 4451763a0cd85734f0afe18b48918cb3c88ca99cef399b7c5f12119281e7b6a8 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e9656232ae4f5ef2d2fdd1711e89d40ba81c8f0faed0aadb22004341da5c8184 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 8ec510fee735c646fb18a3b99f0ca5ca461f9e066c43cdc404d7144f12ae6ed6 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 723522a6c2a2463884a83fa1cc7ae5770deaaf0856a1058cc1023b2bfa1c898b F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c3ebdccf94d5e63c229bf91056c08052d78732e663334070ef3b0ef6fb4bfb8f -R 89bdf08b4a75fdd12936e8e156f340f6 +P cbf483ea0ba3e6dc08ad7ed654380f818544b4c3cedfdb8aa848a83298268ceb +R 16a04cd1971a4487c1631d3a37d3ce6e U stephan -Z 095ae4e504a3bccb462600500450b6bf +Z 455bc03c9fd6d42a81f698c07992ea6a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 143d7cba7b..2a7e690681 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cbf483ea0ba3e6dc08ad7ed654380f818544b4c3cedfdb8aa848a83298268ceb \ No newline at end of file +0ce51bed65d5e430364f74bf959fb76c42ac5eec0769490231d8c8110a1f388c \ No newline at end of file From 241cde98b89255f41ec13b35a33b3dfdaa9cf580 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 6 Dec 2022 11:21:46 +0000 Subject: [PATCH 175/282] JS vtables: add infrastructure related to accessing and modifying sqlite3_index_info. FossilOrigin-Name: 0d77c348039926c24e0fb50a7dc7e4b62895cd201c021f8e29832917e4b8b09f --- ext/wasm/api/sqlite3-v-helper.js | 72 ++++++++++++++++++++++++++------ ext/wasm/tester1.c-pp.js | 67 ++++++++++++++++++++++------- manifest | 14 +++---- manifest.uuid | 2 +- 4 files changed, 121 insertions(+), 34 deletions(-) diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index 84272266e8..a23070b24c 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -22,6 +22,53 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.VfsHelper = vh; sqlite3.VtabHelper = vt; + const sii = capi.sqlite3_index_info; + /** + If n is >=0 and less than this.$nConstraint, this function + returns either a WASM pointer to the 0-based nth entry of + this.$aConstraint (if passed a truthy 2nd argument) or an + sqlite3_index_info.sqlite3_index_constraint object wrapping that + address (if passed a falsy value or no 2nd argument). Returns a + falsy value if n is out of range. + */ + sii.prototype.nthConstraint = function(n, asPtr=false){ + if(n<0 || n>=this.$nConstraint) return false; + const ptr = this.$aConstraint + ( + sii.sqlite3_index_constraint.structInfo.sizeof * n + ); + return asPtr ? ptr : new sii.sqlite3_index_constraint(ptr); + }; + + /** + Works identically to nthConstraint() but returns state from + this.$aConstraintUsage, so returns an + sqlite3_index_info.sqlite3_index_constraint_usage instance + if passed no 2nd argument or a falsy 2nd argument. + */ + sii.prototype.nthConstraintUsage = function(n, asPtr=false){ + if(n<0 || n>=this.$nConstraint) return false; + const ptr = this.$aConstraintUsage + ( + sii.sqlite3_index_constraint_usage.structInfo.sizeof * n + ); + return asPtr ? ptr : new sii.sqlite3_index_constraint_usage(ptr); + }; + + /** + If n is >=0 and less than this.$nOrderBy, this function + returns either a WASM pointer to the 0-based nth entry of + this.$aOrderBy (if passed a truthy 2nd argument) or an + sqlite3_index_info.sqlite3_index_orderby object wrapping that + address (if passed a falsy value or no 2nd argument). Returns a + falsy value if n is out of range. + */ + sii.prototype.nthOrderBy = function(n, asPtr=false){ + if(n<0 || n>=this.$nOrderBy) return false; + const ptr = this.$aOrderBy + ( + sii.sqlite3_index_orderby.structInfo.sizeof * n + ); + return asPtr ? ptr : new sii.sqlite3_index_orderby(ptr); + }; + /** Installs a StructBinder-bound function pointer member of the given name and function in the given StructType target object. @@ -109,6 +156,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); tgt[memKey] = pFunc; if(!tgt.ondispose) tgt.ondispose = []; + else if(!Array.isArray(tgt.ondispose)) tgt.ondispose = [tgt.ondispose]; if(!tgt.ondispose.__removeFuncList){ tgt.ondispose.push('ondispose.__removeFuncList handler', callee.removeFuncList); @@ -244,9 +292,9 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ vt.sqlite3ValuesToJs = capi.sqlite3_create_function_v2.udfConvertArgs; /** - Factory function for xyz2js() impls. + Factory function for wrapXyz() impls. */ - const __v2jsFactory = function(structType){ + const __xWrapFactory = function(structType){ return function(ptr,remove=false){ if(0===arguments.length) ptr = new structType; if(ptr instanceof structType){ @@ -254,7 +302,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this.set(ptr.pointer, ptr); return ptr; }else if(!wasm.isPtr(ptr)){ - sqlite3.SQLite3Error.toss("Invalid argument to v2jsFactory"); + sqlite3.SQLite3Error.toss("Invalid argument to xWrapFactory"); } let rc = this.get(ptr); if(remove) this.delete(ptr); @@ -270,45 +318,45 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Has 3 distinct uses: - - vtab2js() instantiates a new capi.sqlite3_vtab instance, maps + - wrapVtab() instantiates a new capi.sqlite3_vtab instance, maps its pointer for later by-pointer lookup, and returns that object. This is intended to be called from sqlite3_module::xConnect() or xCreate() implementations. - - vtab2js(pVtab) accepts a WASM pointer to a C-level + - wrapVtab(pVtab) accepts a WASM pointer to a C-level (sqlite3_vtab*) instance and returns the capi.sqlite3_vtab object created by the first form of this function, or undefined if that form has not been used. This is intended to be called from sqlite3_module methods which take a (sqlite3_vtab*) pointer _except_ for xDisconnect(), in which case use... - - vtab2js(pVtab,true) as for the previous form, but removes the + - wrapVtab(pVtab,true) as for the previous form, but removes the pointer-to-object mapping before returning. The caller must call dispose() on the returned object. This is intended to be called from sqlite3_module::xDisconnect() implementations or in error handling of a failed xCreate() or xConnect(). */ - vt.vtab2js = __v2jsFactory(capi.sqlite3_vtab); + vt.xWrapVtab = __xWrapFactory(capi.sqlite3_vtab); /** EXPERIMENTAL. DO NOT USE IN CLIENT CODE. - Works identically to vtab2js() except that it deals with + Works identically to wrapVtab() except that it deals with sqlite3_cursor objects and pointers instead of sqlite3_vtab. - - vcur2js() is intended to be called from sqlite3_module::xOpen() + - wrapCursor() is intended to be called from sqlite3_module::xOpen() - - vcur2js(pCursor) is intended to be called from all sqlite3_module + - wrapCursor(pCursor) is intended to be called from all sqlite3_module methods which take a (sqlite3_vtab_cursor*) _except_ for xClose(), in which case use... - - vcur2js(pCursor, true) will remove the m apping of pCursor to a + - wrapCursor(pCursor, true) will remove the m apping of pCursor to a capi.sqlite3_vtab_cursor object and return that object. The caller must call dispose() on the returned object. This is intended to be called form xClose() or in error handling of a failed xOpen(). */ - vt.vcur2js = __v2jsFactory(capi.sqlite3_vtab_cursor); + vt.xWrapCursor = __xWrapFactory(capi.sqlite3_vtab_cursor); /** Given an error object, this function returns diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index c32c4e5e84..7013f3cea8 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1553,9 +1553,9 @@ self.sqlite3InitModule = sqlite3InitModule; pDb, "CREATE TABLE ignored(a,b)" ); if(0===rc){ - const t = vth.vtab2js(); + const t = vth.xWrapVtab(); wasm.setPtrValue(ppVtab, t.pointer); - T.assert(t === vth.vtab2js(wasm.getPtrValue(ppVtab))); + T.assert(t === vth.xWrapVtab(wasm.getPtrValue(ppVtab))); } return rc; }catch(e){ @@ -1567,7 +1567,7 @@ self.sqlite3InitModule = sqlite3InitModule; }, xDisconnect: function(pVtab){ try { - const t = vth.vtab2js(pVtab, true); + const t = vth.xWrapVtab(pVtab, true); t.dispose(); return 0; }catch(e){ @@ -1576,7 +1576,7 @@ self.sqlite3InitModule = sqlite3InitModule; }, xOpen: function(pVtab, ppCursor){ try{ - const t = vth.vtab2js(pVtab), c = vth.vcur2js(); + const t = vth.xWrapVtab(pVtab), c = vth.xWrapCursor(); T.assert(t instanceof capi.sqlite3_vtab); T.assert(c instanceof capi.sqlite3_vtab_cursor); wasm.setPtrValue(ppCursor, c.pointer); @@ -1588,9 +1588,9 @@ self.sqlite3InitModule = sqlite3InitModule; }, xClose: function(pCursor){ try{ - const c = vth.vcur2js(pCursor,true); + const c = vth.xWrapCursor(pCursor,true); T.assert(c instanceof capi.sqlite3_vtab_cursor) - .assert(!vth.vcur2js(pCursor)); + .assert(!vth.xWrapCursor(pCursor)); c.dispose(); return 0; }catch(e){ @@ -1599,7 +1599,7 @@ self.sqlite3InitModule = sqlite3InitModule; }, xNext: function(pCursor){ try{ - const c = vth.vcur2js(pCursor); + const c = vth.xWrapCursor(pCursor); ++c._rowId; return 0; }catch(e){ @@ -1609,7 +1609,7 @@ self.sqlite3InitModule = sqlite3InitModule; }, xColumn: function(pCursor, pCtx, iCol){ try{ - const c = vth.vcur2js(pCursor); + const c = vth.xWrapCursor(pCursor); switch(iCol){ case tmplCols.A: capi.sqlite3_result_int(pCtx, 1000 + c._rowId); @@ -1626,7 +1626,7 @@ self.sqlite3InitModule = sqlite3InitModule; }, xRowid: function(pCursor, ppRowid64){ try{ - const c = vth.vcur2js(pCursor); + const c = vth.xWrapCursor(pCursor); vth.setRowId(ppRowid64, c._rowId); return 0; }catch(e){ @@ -1634,14 +1634,17 @@ self.sqlite3InitModule = sqlite3InitModule; } }, xEof: function(pCursor){ - const c = vth.vcur2js(pCursor); + const c = vth.xWrapCursor(pCursor); return c._rowId>=10; }, xFilter: function(pCursor, idxNum, idxCStr, argc, argv/* [sqlite3_value* ...] */){ try{ - const c = vth.vcur2js(pCursor); + const c = vth.xWrapCursor(pCursor); c._rowId = 0; + const list = vth.sqlite3ValuesToJs(argc, argv); + T.assert(argc === list.length); + //log(argc,"xFilter value(s):",list); return 0; }catch(e){ return vth.xMethodError('xFilter',e); @@ -1649,10 +1652,44 @@ self.sqlite3InitModule = sqlite3InitModule; }, xBestIndex: function(pVtab, pIdxInfo){ try{ - const t = vth.vtab2js(pVtab); - const pii = new capi.sqlite3_index_info(pIdxInfo); + //const t = vth.xWrapVtab(pVtab); + const sii = capi.sqlite3_index_info; + const pii = new sii(pIdxInfo); pii.$estimatedRows = 10; pii.$estimatedCost = 10.0; + //log("xBestIndex $nConstraint =",pii.$nConstraint); + if(pii.$nConstraint>0){ + // Validate nthConstraint() and nthConstraintUsage() + const max = pii.$nConstraint; + for(let i=0; i < max; ++i ){ + let v = pii.nthConstraint(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthConstraint(i); + T.assert(v instanceof sii.sqlite3_index_constraint) + .assert(v.pointer >= pii.$aConstraint); + v.dispose(); + v = pii.nthConstraintUsage(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthConstraintUsage(i); + T.assert(v instanceof sii.sqlite3_index_constraint_usage) + .assert(v.pointer >= pii.$aConstraintUsage); + v.$argvIndex = i;//just to get some values into xFilter + v.dispose(); + } + } + //log("xBestIndex $nOrderBy =",pii.$nOrderBy); + if(pii.$nOrderBy>0){ + // Validate nthOrderBy() + const max = pii.$nOrderBy; + for(let i=0; i < max; ++i ){ + let v = pii.nthOrderBy(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthOrderBy(i); + T.assert(v instanceof sii.sqlite3_index_orderby) + .assert(v.pointer >= pii.$aOrderBy); + v.dispose(); + } + } pii.dispose(); return 0; }catch(e){ @@ -1692,7 +1729,9 @@ self.sqlite3InitModule = sqlite3InitModule; this.db.checkRc(rc); const list = this.db.selectArrays( - "SELECT a,b FROM testvtab order by a" + "SELECT a,b FROM testvtab where a<9999 and b>1 order by a, b" + /* Query is shaped so that it will ensure that some constraints + end up in xBestIndex(). */ ); T.assert(10===list.length) .assert(1000===list[0][0]) diff --git a/manifest b/manifest index 7c57dc1324..3bfa27aa70 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sdeprecated\ssymbol\ssqlite3.opfs.OpfsDb,\swhich\swas\srenamed\sto\ssqlite3.oo1.OpfsDb\son\s2022-11-29. -D 2022-12-06T09:49:04.292 +C JS\svtables:\sadd\sinfrastructure\srelated\sto\saccessing\sand\smodifying\ssqlite3_index_info. +D 2022-12-06T11:21:46.303 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -509,7 +509,7 @@ F ext/wasm/api/sqlite3-api-prologue.js a7596c30392d9ca8f5b7c14feb4e6788107d1159f F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 -F ext/wasm/api/sqlite3-v-helper.js 4451763a0cd85734f0afe18b48918cb3c88ca99cef399b7c5f12119281e7b6a8 +F ext/wasm/api/sqlite3-v-helper.js 6b408ee4e926cb0f7fe41a63a1205049283af301fe3f5de3c038845ccf9106df F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 8ec510fee735c646fb18a3b99f0ca5ca461f9e066c43cdc404d7144f12ae6ed6 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 723522a6c2a2463884a83fa1cc7ae5770deaaf0856a1058cc1023b2bfa1c898b @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 -F ext/wasm/tester1.c-pp.js 81c40395be4a7e97a3ca0ff62a4e02204239b1db6f24fe15b3c96a17fb41f29b +F ext/wasm/tester1.c-pp.js c5679da7895377df03e6075fc0e9dff8b5f570bd4edb63f714154d3030279fce F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cbf483ea0ba3e6dc08ad7ed654380f818544b4c3cedfdb8aa848a83298268ceb -R 16a04cd1971a4487c1631d3a37d3ce6e +P 0ce51bed65d5e430364f74bf959fb76c42ac5eec0769490231d8c8110a1f388c +R a1b37a98bcb406acaf386e5a65bbf6e3 U stephan -Z 455bc03c9fd6d42a81f698c07992ea6a +Z 3e0a3245949f854c4eed760afd40e3ce # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2a7e690681..a5ae05d6f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ce51bed65d5e430364f74bf959fb76c42ac5eec0769490231d8c8110a1f388c \ No newline at end of file +0d77c348039926c24e0fb50a7dc7e4b62895cd201c021f8e29832917e4b8b09f \ No newline at end of file From 109f173b6f9154c2686ba022f558b874c6b5dcc5 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 Dec 2022 11:41:05 +0000 Subject: [PATCH 176/282] Support an SQLITE_SCANSTAT_NCYCLE statistic for "CO-ROUTINE" elements. FossilOrigin-Name: d3f6a207fd1b3f53688fa1ff35eb71e1c5bcc4115c0c35bb187d22ccf7100b71 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 13 +++++++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index eb01d8ca73..ec923e1cf9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\soptional\sfeature:\sA\sCLI\scontinuation\sprompt\swhich\sreflects\sopen\slexemes\sand\sparens,\ssimilarly\sto\sPG\sshell. -D 2022-12-06T05:31:20.264 +C Support\san\sSQLITE_SCANSTAT_NCYCLE\sstatistic\sfor\s"CO-ROUTINE"\selements. +D 2022-12-06T11:41:05.297 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -645,7 +645,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 321f29e431fbb71e594cc7026391a827a0270237597d2e2401244e7602dea8bd +F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 F src/shell.c.in f4ef4c6bca35bcab91041c2e10ac9b53440c37d5751563713cc7feaa8c1ac880 F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4893b4e3eafc7c9c22b24717f90a585862203f987cf108b079ce6e946093e675 dac2ddc287db7a68d0cd49b785060f62290868fbb1aa2ee09e54d3b1acfbf55f -R 4e6f769112d568da8b23927a4847535b -U larrybr -Z 19f7409ace4582d7233d6e9f94f36495 +P f41f18b1c3c950565ee3c237aebb51cfc3241813c6425813a8217e07aa0153a6 +R 81af46208bd94570e4ca8392d104cd34 +U dan +Z cc39d40fca54132e62b35ae953e9525e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 02dae66a86..43afcfd03e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f41f18b1c3c950565ee3c237aebb51cfc3241813c6425813a8217e07aa0153a6 \ No newline at end of file +d3f6a207fd1b3f53688fa1ff35eb71e1c5bcc4115c0c35bb187d22ccf7100b71 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 34ba3f7a4c..886ac336f2 100644 --- a/src/select.c +++ b/src/select.c @@ -7260,13 +7260,16 @@ int sqlite3Select( ** set on each invocation. */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; +#endif pItem->regReturn = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); VdbeComment((v, "%!S", pItem)); pItem->addrFillSub = addrTop; sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); - ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); + ExplainQueryPlan2(addrExplain, (pParse, 1, "CO-ROUTINE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; @@ -7274,6 +7277,10 @@ int sqlite3Select( sqlite3VdbeEndCoroutine(v, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); + /* For SQLITE_SCANSTAT_NCYCLE, all instructions from the + ** OP_InitCoroutine coded above until this point are attributed to + ** the CO-ROUTINE query element. */ + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain-1, -1); }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ /* This is a CTE for which materialization code has already been ** generated. Invoke the subroutine to compute the materialization, @@ -7328,7 +7335,6 @@ int sqlite3Select( if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); - sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ @@ -7338,6 +7344,9 @@ int sqlite3Select( pCteUse->iCur = pItem->iCursor; pCteUse->nRowEst = pSub->nSelectRow; } + /* For SQLITE_SCANSTAT_NCYCLE, all instructions from the + ** OP_Explain to here are attibuted to the MATERIALIZE element. */ + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); } if( db->mallocFailed ) goto select_end; pParse->nHeight -= sqlite3SelectExprHeight(p); From b1026aeda9a6fb2ea411a37d0db9ea516e417bd2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 6 Dec 2022 13:12:33 +0000 Subject: [PATCH 177/282] In the unix backend, when implementing the defenses against small file descriptors, delete a file just created if it was opened with O_EXCL|O_CREAT so that it can be created again the next time through the loop. Fix for the problem described by [forum:/forumpost/699af709ab3a8ccf|forum post 699af709ab3a8ccf]. FossilOrigin-Name: c0cfe0582add87981826d124a0763482f51fae4b105b5a970dd56919f1d04d60 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ec923e1cf9..c7b2bd3bb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Support\san\sSQLITE_SCANSTAT_NCYCLE\sstatistic\sfor\s"CO-ROUTINE"\selements. -D 2022-12-06T11:41:05.297 +C In\sthe\sunix\sbackend,\swhen\simplementing\sthe\sdefenses\sagainst\ssmall\sfile\ndescriptors,\sdelete\sa\sfile\sjust\screated\sif\sit\swas\sopened\swith\nO_EXCL|O_CREAT\sso\sthat\sit\scan\sbe\screated\sagain\sthe\snext\stime\sthrough\sthe\nloop.\s\sFix\sfor\sthe\sproblem\sdescribed\sby\n[forum:/forumpost/699af709ab3a8ccf|forum\spost\s699af709ab3a8ccf]. +D 2022-12-06T13:12:33.146 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -629,7 +629,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 73f89ab97ecdb3216857d2acc8395103f89164eaadac87cce4e9e16445c89541 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 08191111a7040b8d5a6fff946f9fc9a11a0f83bac727c0415dfc5d030e1bc41f +F src/os_unix.c 60787ed9f2492a9f2f0ca5cca2d80cdbfd6e8f45a1b1560790ff39c9dfc680c0 F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c d3122cf67f327f1e2df12d06236a3473a8099542071e257067552f42917f172d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f41f18b1c3c950565ee3c237aebb51cfc3241813c6425813a8217e07aa0153a6 -R 81af46208bd94570e4ca8392d104cd34 -U dan -Z cc39d40fca54132e62b35ae953e9525e +P d3f6a207fd1b3f53688fa1ff35eb71e1c5bcc4115c0c35bb187d22ccf7100b71 +R a4b71026bbbedf9b7ee6a91d6a043bd8 +U drh +Z d5e50db849cd86cf21ff12c0ca4a2cbc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 43afcfd03e..132ad427f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d3f6a207fd1b3f53688fa1ff35eb71e1c5bcc4115c0c35bb187d22ccf7100b71 \ No newline at end of file +c0cfe0582add87981826d124a0763482f51fae4b105b5a970dd56919f1d04d60 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index ddb6f0c07c..d248703e9b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -686,6 +686,9 @@ static int robust_open(const char *z, int f, mode_t m){ break; } if( fd>=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break; + if( (f & (O_EXCL|O_CREAT))==(O_EXCL|O_CREAT) ){ + (void)osUnlink(z); + } osClose(fd); sqlite3_log(SQLITE_WARNING, "attempt to open \"%s\" as file descriptor %d", z, fd); From d84f185f9796e82b48a6c18e12b5dce05186ffd1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 6 Dec 2022 15:11:13 +0000 Subject: [PATCH 178/282] Fix compiler warnings in the new dynamic continuation prompt logic of the CLI. FossilOrigin-Name: 0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c7b2bd3bb1..38783ac47e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sunix\sbackend,\swhen\simplementing\sthe\sdefenses\sagainst\ssmall\sfile\ndescriptors,\sdelete\sa\sfile\sjust\screated\sif\sit\swas\sopened\swith\nO_EXCL|O_CREAT\sso\sthat\sit\scan\sbe\screated\sagain\sthe\snext\stime\sthrough\sthe\nloop.\s\sFix\sfor\sthe\sproblem\sdescribed\sby\n[forum:/forumpost/699af709ab3a8ccf|forum\spost\s699af709ab3a8ccf]. -D 2022-12-06T13:12:33.146 +C Fix\scompiler\swarnings\sin\sthe\snew\sdynamic\scontinuation\sprompt\slogic\sof\nthe\sCLI. +D 2022-12-06T15:11:13.637 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 -F src/shell.c.in f4ef4c6bca35bcab91041c2e10ac9b53440c37d5751563713cc7feaa8c1ac880 +F src/shell.c.in f37273af2d5cd2299091b3c1e260254ebd7cd16d7602426983b36009290e8577 F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d3f6a207fd1b3f53688fa1ff35eb71e1c5bcc4115c0c35bb187d22ccf7100b71 -R a4b71026bbbedf9b7ee6a91d6a043bd8 +P c0cfe0582add87981826d124a0763482f51fae4b105b5a970dd56919f1d04d60 +R 19f4ce887fdd7ce982932018e71e1a9d U drh -Z d5e50db849cd86cf21ff12c0ca4a2cbc +Z 087fdbe4ee41c07ef7676876e727f795 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 132ad427f2..16c4d729e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0cfe0582add87981826d124a0763482f51fae4b105b5a970dd56919f1d04d60 \ No newline at end of file +0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 78e681c022..85e708551d 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -499,7 +499,7 @@ static char continuePrompt[PROMPT_LEN_MAX]; # define CONTINUE_PAREN_INCR(p,n) \ if(p && stdin_is_interactive) (trackParenLevel(p,n)) # define SCAN_TRACKER_REFTYPE struct DynaPrompt * -# define CONTINUE_PROMPT_STATE ((SCAN_TRACKER_REFTYPE)&dynPrompt) +# define CONTINUE_PROMPT_STATE (pDynPrompt) static struct DynaPrompt { char dynamicPrompt[PROMPT_LEN_MAX]; @@ -507,6 +507,7 @@ static struct DynaPrompt { int inParenLevel; char *zScannerAwaits; } dynPrompt = { {0}, {0}, 0, 0 }; +static struct DynaPrompt *pDynPrompt = &dynPrompt; /* Record parenthesis nesting level change, or force level to 0. */ static void trackParenLevel(struct DynaPrompt *p, int ni){ From ad9ff1d5f2db2c90271a07fc6e3c6a3d650094b3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 6 Dec 2022 15:24:05 +0000 Subject: [PATCH 179/282] Simplified experimental changes to promote the use of co-routines. Less cruft than the coroutines-exp1 branch, but still does not work. Checked in as a work-in-progress. FossilOrigin-Name: e2318a30bf6ad2aeb4c4cb29661bc24ff78eb51bf07decc4b8da1ecac0015ef0 --- manifest | 21 ++++++++------ manifest.uuid | 2 +- src/select.c | 74 ++++++++++++++++++++++++++++++++----------------- src/sqliteInt.h | 1 + src/update.c | 1 + src/vdbe.h | 1 + 6 files changed, 65 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 38783ac47e..2fab02d772 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\sthe\snew\sdynamic\scontinuation\sprompt\slogic\sof\nthe\sCLI. -D 2022-12-06T15:11:13.637 +C Simplified\sexperimental\schanges\sto\spromote\sthe\suse\sof\sco-routines.\s\sLess\ncruft\sthan\sthe\scoroutines-exp1\sbranch,\sbut\sstill\sdoes\snot\swork.\s\sChecked\sin\nas\sa\swork-in-progress. +D 2022-12-06T15:24:05.905 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -645,12 +645,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 +F src/select.c cbb0ce95039273f436cdad9308251ab3affd69482fd9927495e1c37875afeeb1 F src/shell.c.in f37273af2d5cd2299091b3c1e260254ebd7cd16d7602426983b36009290e8577 F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 0c9934acd88e0fa14f0d4743233b4cd7bd7bbc84099ba3cad50d8e69676ce2b9 +F src/sqliteInt.h 5a91a427ab821dd3db556411ec0bf0f02bea796ce9927183979c2a1954ae4630 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -712,13 +712,13 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 -F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa +F src/update.c 3cf1cb45674177e09338bdbe5454fb62207c7f2eb8cea288e74286467e901959 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 313f3154e2b85a447326f5dd15de8d31a4df6ab0c3579bd58f426ff634ec9050 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c e744259050ec9bbcd47acf9a03a66fe3305d7429c823995de11ed524ca06d16f -F src/vdbe.h 6d921884cf8ec6a53efba99f8b68e32e955367631743e29039840e781aaf547c +F src/vdbe.h c93d4bc242fc9109558aa6b3ce8acef7effe100a42c148dc63239c488ad6c060 F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 F src/vdbeapi.c 959cef4c1e5cb6589e67b3295fe9d3e45cbd109ae6618d57b74c900dbd6be0cd F src/vdbeaux.c 0b81f317c86ed9f4ba822af66a52777fed6e8180edc79d4fc62ffe75c8e52d80 @@ -2067,8 +2067,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c0cfe0582add87981826d124a0763482f51fae4b105b5a970dd56919f1d04d60 -R 19f4ce887fdd7ce982932018e71e1a9d +P 0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 +R da3604bd27058bf7971e3104c707bf9f +T *branch * coroutines-exp2 +T *sym-coroutines-exp2 * +T -sym-trunk * U drh -Z 087fdbe4ee41c07ef7676876e727f795 +Z 83efea07fbf822c79733971b25cfb62c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 16c4d729e4..638c179901 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 \ No newline at end of file +e2318a30bf6ad2aeb4c4cb29661bc24ff78eb51bf07decc4b8da1ecac0015ef0 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 886ac336f2..8f3e5bb9aa 100644 --- a/src/select.c +++ b/src/select.c @@ -6854,6 +6854,52 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ return 0; } +/* +** Return TRUE (non-zero) if the i-th entry in the pTabList SrcList can +** be implemented as a co-routine. The i-th entry is guaranteed to be +** a subquery. +** +** The subquery is implemented as a co-routine if all of the following are +** true: +** +** (1) Either of the following are true: +** (1a) The subquery must be the outer loop because it is either +** (i) the only term in the FROM clause, or because (ii) it +** is the left-most term and a CROSS JOIN or similar requires +** it to be the outer loop. subquery and there is nothing +** (1b) There is nothing that would prevent the subquery from +** being an outer loop and the SQLITE_PREPARE_SAFESQL flag +** is not set. +** (2) The subquery is not a CTE that should be materialized +** (3) The subquery is not part of a left operand for a RIGHT JOIN +** (4) The SQLITE_Coroutine optimization disable flag is not set +*/ +static int fromClauseTermCanBeCoroutine( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* FROM clause */ + int i /* Which term of the FROM clause */ +){ + SrcItem *pItem = &pTabList->a[i]; + if( pItem->fg.isCte && pItem->u2.pCteUse->eM10d==M10d_Yes ) return 0;/* (2) */ + if( pTabList->a[0].fg.jointype & JT_LTORJ ) return 0; /* (3) */ + if( OptimizationDisabled(pParse->db, SQLITE_Coroutines) ) return 0; /* (4) */ + if( i==0 ){ + if( pTabList->nSrc==1 ) return 1; /* (1a-i) */ + if( pTabList->a[1].fg.jointype & JT_CROSS ) return 1; /* (1a-ii) */ + if( pParse->prepFlags & SQLITE_PREPARE_SAFEOPT ) return 0; + return 1; + } + if( pParse->prepFlags & SQLITE_PREPARE_SAFEOPT ) return 0; + while( 1 /*exit-by-break*/ ){ + if( pItem->fg.jointype & (JT_OUTER|JT_CROSS) ) return 0; + if( i==0 ) break; + i--; + pItem--; + if( pItem->pSelect!=0 ) return 0; + } + return 1; +} + /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7241,35 +7287,19 @@ int sqlite3Select( pParse->zAuthContext = pItem->zName; /* Generate code to implement the subquery - ** - ** The subquery is implemented as a co-routine if all of the following are - ** true: - ** - ** (1) the subquery is guaranteed to be the outer loop (so that - ** it does not need to be computed more than once), and - ** (2) the subquery is not a CTE that should be materialized - ** (3) the subquery is not part of a left operand for a RIGHT JOIN */ - if( i==0 - && (pTabList->nSrc==1 - || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0) /* (1) */ - && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) /* (2) */ - && (pTabList->a[0].fg.jointype & JT_LTORJ)==0 /* (3) */ - ){ + if( fromClauseTermCanBeCoroutine(pParse, pTabList, i) ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - int addrExplain; -#endif pItem->regReturn = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); VdbeComment((v, "%!S", pItem)); pItem->addrFillSub = addrTop; sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); - ExplainQueryPlan2(addrExplain, (pParse, 1, "CO-ROUTINE %!S", pItem)); + ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; @@ -7277,10 +7307,6 @@ int sqlite3Select( sqlite3VdbeEndCoroutine(v, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); - /* For SQLITE_SCANSTAT_NCYCLE, all instructions from the - ** OP_InitCoroutine coded above until this point are attributed to - ** the CO-ROUTINE query element. */ - sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain-1, -1); }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ /* This is a CTE for which materialization code has already been ** generated. Invoke the subroutine to compute the materialization, @@ -7335,6 +7361,7 @@ int sqlite3Select( if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ @@ -7344,9 +7371,6 @@ int sqlite3Select( pCteUse->iCur = pItem->iCursor; pCteUse->nRowEst = pSub->nSelectRow; } - /* For SQLITE_SCANSTAT_NCYCLE, all instructions from the - ** OP_Explain to here are attibuted to the MATERIALIZE element. */ - sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); } if( db->mallocFailed ) goto select_end; pParse->nHeight -= sqlite3SelectExprHeight(p); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5a19058bd1..393237358a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1844,6 +1844,7 @@ struct sqlite3 { #define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ +#define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* diff --git a/src/update.c b/src/update.c index 809b306539..0612b2708d 100644 --- a/src/update.c +++ b/src/update.c @@ -268,6 +268,7 @@ static void updateFromSelect( if( pSelect ) pSelect->selFlags |= SF_OrderByReqd; sqlite3SelectDestInit(&dest, eDest, iEph); dest.iSDParm2 = (pPk ? pPk->nKeyCol : -1); + pParse->prepFlags |= SQLITE_PREPARE_SAFEOPT; sqlite3Select(pParse, pSelect, &dest); sqlite3SelectDelete(db, pSelect); } diff --git a/src/vdbe.h b/src/vdbe.h index a199247434..467c999343 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -165,6 +165,7 @@ typedef struct VdbeOpList VdbeOpList; ** Additional non-public SQLITE_PREPARE_* flags */ #define SQLITE_PREPARE_SAVESQL 0x80 /* Preserve SQL text */ +#define SQLITE_PREPARE_SAFEOPT 0x40 /* Use only safe optimizations */ #define SQLITE_PREPARE_MASK 0x0f /* Mask of public flags */ /* From 29226fc023767ddbb078ae37b64e1a4d8accf111 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 6 Dec 2022 17:59:05 +0000 Subject: [PATCH 180/282] Clear a few more -Wall warnings and simplify dynaprompt feature keep/omit macros. FossilOrigin-Name: 540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 33 +++++++++++++++++---------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 38783ac47e..e0cc60735d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\sthe\snew\sdynamic\scontinuation\sprompt\slogic\sof\nthe\sCLI. -D 2022-12-06T15:11:13.637 +C Clear\sa\sfew\smore\s-Wall\swarnings\sand\ssimplify\sdynaprompt\sfeature\skeep/omit\smacros. +D 2022-12-06T17:59:05.853 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 -F src/shell.c.in f37273af2d5cd2299091b3c1e260254ebd7cd16d7602426983b36009290e8577 +F src/shell.c.in 991eff7f819f96ae76671e7aa44597ac702274fd441d3b0d9297cbd3b9ce1284 F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c0cfe0582add87981826d124a0763482f51fae4b105b5a970dd56919f1d04d60 -R 19f4ce887fdd7ce982932018e71e1a9d -U drh -Z 087fdbe4ee41c07ef7676876e727f795 +P 0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 +R d2096b6607984433a4c4474baae68bae +U larrybr +Z 397638802057e8cab20b3ed1dff696d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 16c4d729e4..873db8bc54 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 \ No newline at end of file +540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 85e708551d..a5c4c48c15 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -482,24 +482,26 @@ static char continuePrompt[PROMPT_LEN_MAX]; */ #ifdef SQLITE_OMIT_DYNAPROMPT # define CONTINUATION_PROMPT continuePrompt -# define CONTINUE_PROMPT_RESET(p) +# define CONTINUE_PROMPT_RESET # define CONTINUE_PROMPT_AWAITS(p,s) # define CONTINUE_PROMPT_AWAITC(p,c) # define CONTINUE_PAREN_INCR(p,n) -# define CONTINUE_PROMPT_STATE 0 -# define SCAN_TRACKER_REFTYPE void* +# define CONTINUE_PROMPT_PSTATE 0 +typedef void *t_NoDynaPrompt; +# define SCAN_TRACKER_REFTYPE t_NoDynaPrompt #else # define CONTINUATION_PROMPT dynamicContinuePrompt() -# define CONTINUE_PROMPT_RESET(p) \ - if(p) (setLexemeOpen(p,0,0), trackParenLevel(p,0)) +# define CONTINUE_PROMPT_RESET \ + do {setLexemeOpen(&dynPrompt,0,0); trackParenLevel(&dynPrompt,0);} while(0) # define CONTINUE_PROMPT_AWAITS(p,s) \ if(p && stdin_is_interactive) setLexemeOpen(p, s, 0) # define CONTINUE_PROMPT_AWAITC(p,c) \ if(p && stdin_is_interactive) setLexemeOpen(p, 0, c) # define CONTINUE_PAREN_INCR(p,n) \ if(p && stdin_is_interactive) (trackParenLevel(p,n)) -# define SCAN_TRACKER_REFTYPE struct DynaPrompt * -# define CONTINUE_PROMPT_STATE (pDynPrompt) +# define CONTINUE_PROMPT_PSTATE (&dynPrompt) +typedef struct DynaPrompt *t_DynaPromptRef; +# define SCAN_TRACKER_REFTYPE t_DynaPromptRef static struct DynaPrompt { char dynamicPrompt[PROMPT_LEN_MAX]; @@ -507,7 +509,6 @@ static struct DynaPrompt { int inParenLevel; char *zScannerAwaits; } dynPrompt = { {0}, {0}, 0, 0 }; -static struct DynaPrompt *pDynPrompt = &dynPrompt; /* Record parenthesis nesting level change, or force level to 0. */ static void trackParenLevel(struct DynaPrompt *p, int ni){ @@ -542,11 +543,11 @@ static char *dynamicContinuePrompt(void){ PROMPT_LEN_MAX-4); }else{ if( dynPrompt.inParenLevel>9 ){ - strncpy(dynPrompt.dynamicPrompt, "(..", 3); + strncpy(dynPrompt.dynamicPrompt, "(..", 4); }else if( dynPrompt.inParenLevel<0 ){ - strncpy(dynPrompt.dynamicPrompt, ")x!", 3); + strncpy(dynPrompt.dynamicPrompt, ")x!", 4); }else{ - strncpy(dynPrompt.dynamicPrompt, "(x.", 3); + strncpy(dynPrompt.dynamicPrompt, "(x.", 4); dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); } strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); @@ -11288,7 +11289,7 @@ static int process_input(ShellState *p){ } ++p->inputNesting; p->lineno = 0; - CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); + CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); zLine = one_input_line(p->in, zLine, nSql>0); @@ -11307,7 +11308,7 @@ static int process_input(ShellState *p){ && line_is_complete(zSql, nSql) ){ memcpy(zLine,";",2); } - qss = quickscan(zLine, qss, CONTINUE_PROMPT_STATE); + qss = quickscan(zLine, qss, CONTINUE_PROMPT_PSTATE); if( QSS_PLAINWHITE(qss) && nSql==0 ){ /* Just swallow single-line whitespace */ echo_group_input(p, zLine); @@ -11315,7 +11316,7 @@ static int process_input(ShellState *p){ continue; } if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){ - CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); + CONTINUE_PROMPT_RESET; echo_group_input(p, zLine); if( zLine[0]=='.' ){ rc = do_meta_command(zLine, p); @@ -11351,7 +11352,7 @@ static int process_input(ShellState *p){ if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); - CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); + CONTINUE_PROMPT_RESET; nSql = 0; if( p->outCount ){ output_reset(p); @@ -11371,7 +11372,7 @@ static int process_input(ShellState *p){ /* This may be incomplete. Let the SQL parser deal with that. */ echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); - CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); + CONTINUE_PROMPT_RESET; } free(zSql); free(zLine); From 2adb309ead2d8a794a65b374c76d7c7246c95a69 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 Dec 2022 18:48:06 +0000 Subject: [PATCH 181/282] Have sqlite3_stmt_scanstatus_v2() return an NCYCLE value for all loops, not just virtual tables ones. The value returned is the sum of the NCYCLE counts for the various opcodes that move or read data from the table or index cursor associated with the loop. FossilOrigin-Name: 9499b2f51e8174c6b8a67840c92ba23b7dd1dc8dc2b91fca0c5dc07b71662149 --- manifest | 22 ++++++------- manifest.uuid | 2 +- src/vdbe.c | 76 +++++++++++++++++++++---------------------- src/vdbeapi.c | 6 ++-- src/wherecode.c | 16 +++++++-- test/scanstatus2.test | 31 ++++++++++++++---- tool/mkopcodeh.tcl | 5 +++ 7 files changed, 95 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index e0cc60735d..c4286f7f7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clear\sa\sfew\smore\s-Wall\swarnings\sand\ssimplify\sdynaprompt\sfeature\skeep/omit\smacros. -D 2022-12-06T17:59:05.853 +C Have\ssqlite3_stmt_scanstatus_v2()\sreturn\san\sNCYCLE\svalue\sfor\sall\sloops,\snot\sjust\svirtual\stables\sones.\sThe\svalue\sreturned\sis\sthe\ssum\sof\sthe\sNCYCLE\scounts\sfor\sthe\svarious\sopcodes\sthat\smove\sor\sread\sdata\sfrom\sthe\stable\sor\sindex\scursor\sassociated\swith\sthe\sloop. +D 2022-12-06T18:48:06.154 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -717,10 +717,10 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 313f3154e2b85a447326f5dd15de8d31a4df6ab0c3579bd58f426ff634ec9050 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c e744259050ec9bbcd47acf9a03a66fe3305d7429c823995de11ed524ca06d16f +F src/vdbe.c 8fdef91d3c9bfa363e8eb381d362e2f0cadec456975ec5595c0bfdf65da84e53 F src/vdbe.h 6d921884cf8ec6a53efba99f8b68e32e955367631743e29039840e781aaf547c F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 -F src/vdbeapi.c 959cef4c1e5cb6589e67b3295fe9d3e45cbd109ae6618d57b74c900dbd6be0cd +F src/vdbeapi.c 5e265ab5165b0dfeac55e6a2ad3929d64584c4e7ca106b438137d7596da91348 F src/vdbeaux.c 0b81f317c86ed9f4ba822af66a52777fed6e8180edc79d4fc62ffe75c8e52d80 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a @@ -734,7 +734,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 20f4f51d2d5fb19b984e6ea381b26cf627cc93e64dd9b2ce6a94531aec2f5916 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c d3146e215f3a716c0e153ded68bfcc747aad550e114a79729bda887cf4ea6f00 +F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1458,7 +1458,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 7dbcfd6adc6a8df6abc59f83d6da5a27e1bce0b2f6fa55147c8176d7c44e0450 -F test/scanstatus2.test cc0be2f645c32ede50affa7d4ecfaffd2b4572a4fd7f723082397c65a910a2b5 +F test/scanstatus2.test 3f45ec1d8b45dcb0df7f98daf3364fc2e03bf89a8c3dd2428251a5e0a34de7af F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -1996,7 +1996,7 @@ F tool/mkctimec.tcl c185cf1bdcd3d9bd3c06f77a2fd2df8a4a0d07266f992ecda75286965ba3 F tool/mkkeywordhash.c 35bfc41adacc4aa6ef6fca7fd0c63e0ec0534b78daf4d0cfdebe398216bbffc3 F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a895ab33 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef -F tool/mkopcodeh.tcl bcb2bd5affb545fd219ef0304c7978e2a356407ab723f45ec8569235892c1c3f +F tool/mkopcodeh.tcl 769d9e6a8b462323150dc13a8539d6064664b72974f7894befe2491cc73e05cd F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl bd07bd59d45d0f3448e123d6937e9811195f9908a51e09d774609883055bfd3d F tool/mkshellc.tcl 02d0de8349ef830c0fb20d29680320bde2466e2ec422e5bd94c4317a7a7e8cc9 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 -R d2096b6607984433a4c4474baae68bae -U larrybr -Z 397638802057e8cab20b3ed1dff696d1 +P 540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 +R e347a9226ee206cfe1e680e2e5e722d0 +U dan +Z eaf9f5c14b0f034256d24205aa5d85c9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 873db8bc54..79c29e3304 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 \ No newline at end of file +9499b2f51e8174c6b8a67840c92ba23b7dd1dc8dc2b91fca0c5dc07b71662149 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1bf0ceb297..cd27aff44f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2780,7 +2780,7 @@ case OP_Offset: { /* out3 */ ** typeof() function or the IS NULL or IS NOT NULL operators or the ** equivalent. In this case, all content loading can be omitted. */ -case OP_Column: { +case OP_Column: { /* ncycle */ u32 p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The B-Tree cursor corresponding to pC */ @@ -4132,7 +4132,7 @@ case OP_SetCookie: { ** ** See also: OP_OpenRead, OP_ReopenIdx */ -case OP_ReopenIdx: { +case OP_ReopenIdx: { /* ncycle */ int nField; KeyInfo *pKeyInfo; u32 p2; @@ -4153,7 +4153,7 @@ case OP_ReopenIdx: { } /* If the cursor is not currently open or is open on a different ** index, then fall through into OP_OpenRead to force a reopen */ -case OP_OpenRead: +case OP_OpenRead: /* ncycle */ case OP_OpenWrite: assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); @@ -4247,7 +4247,7 @@ open_cursor_set_hints: ** ** Duplicate ephemeral cursors are used for self-joins of materialized views. */ -case OP_OpenDup: { +case OP_OpenDup: { /* ncycle */ VdbeCursor *pOrig; /* The original cursor to be duplicated */ VdbeCursor *pCx; /* The new cursor */ @@ -4309,8 +4309,8 @@ case OP_OpenDup: { ** by this opcode will be used for automatically created transient ** indices in joins. */ -case OP_OpenAutoindex: -case OP_OpenEphemeral: { +case OP_OpenAutoindex: /* ncycle */ +case OP_OpenEphemeral: { /* ncycle */ VdbeCursor *pCx; KeyInfo *pKeyInfo; @@ -4468,7 +4468,7 @@ case OP_OpenPseudo: { ** Close a cursor previously opened as P1. If P1 is not ** currently open, this instruction is a no-op. */ -case OP_Close: { +case OP_Close: { /* ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); p->apCsr[pOp->p1] = 0; @@ -4585,10 +4585,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group */ -case OP_SeekLE: /* jump, in3, group */ -case OP_SeekGE: /* jump, in3, group */ -case OP_SeekGT: { /* jump, in3, group */ +case OP_SeekLT: /* jump, in3, group, ncycle */ +case OP_SeekLE: /* jump, in3, group, ncycle */ +case OP_SeekGE: /* jump, in3, group, ncycle */ +case OP_SeekGT: { /* jump, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -4854,7 +4854,7 @@ seek_not_found: ** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0. ** */ -case OP_SeekScan: { +case OP_SeekScan: { /* ncycle */ VdbeCursor *pC; int res; int nStep; @@ -4976,7 +4976,7 @@ case OP_SeekScan: { ** ** P1 must be a valid b-tree cursor. */ -case OP_SeekHit: { +case OP_SeekHit: { /* ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -5108,7 +5108,7 @@ case OP_IfNotOpen: { /* jump */ ** ** See also: NotFound, Found, NotExists */ -case OP_IfNoHope: { /* jump, in3 */ +case OP_IfNoHope: { /* jump, in3, ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -5122,9 +5122,9 @@ case OP_IfNoHope: { /* jump, in3 */ /* Fall through into OP_NotFound */ /* no break */ deliberate_fall_through } -case OP_NoConflict: /* jump, in3 */ -case OP_NotFound: /* jump, in3 */ -case OP_Found: { /* jump, in3 */ +case OP_NoConflict: /* jump, in3, ncycle */ +case OP_NotFound: /* jump, in3, ncycle */ +case OP_Found: { /* jump, in3, ncycle */ int alreadyExists; int ii; VdbeCursor *pC; @@ -5254,7 +5254,7 @@ case OP_Found: { /* jump, in3 */ ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3 */ +case OP_SeekRowid: { /* jump, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -5279,7 +5279,7 @@ case OP_SeekRowid: { /* jump, in3 */ } /* Fall through into OP_NotExists */ /* no break */ deliberate_fall_through -case OP_NotExists: /* jump, in3 */ +case OP_NotExists: /* jump, in3, ncycle */ pIn3 = &aMem[pOp->p3]; assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -5903,7 +5903,7 @@ case OP_RowData: { ** be a separate OP_VRowid opcode for use with virtual tables, but this ** one opcode now works for both table types. */ -case OP_Rowid: { /* out2 */ +case OP_Rowid: { /* out2, ncycle */ VdbeCursor *pC; i64 v; sqlite3_vtab *pVtab; @@ -6002,8 +6002,8 @@ case OP_NullRow: { ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. */ -case OP_SeekEnd: -case OP_Last: { /* jump */ +case OP_SeekEnd: /* ncycle */ +case OP_Last: { /* jump, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -6108,7 +6108,7 @@ case OP_Sort: { /* jump */ ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump */ +case OP_Rewind: { /* jump, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -6202,7 +6202,7 @@ case OP_SorterNext: { /* jump */ rc = sqlite3VdbeSorterNext(db, pC); goto next_tail; -case OP_Prev: /* jump */ +case OP_Prev: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5==0 || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP @@ -6217,7 +6217,7 @@ case OP_Prev: /* jump */ rc = sqlite3BtreePrevious(pC->uc.pCursor, pOp->p3); goto next_tail; -case OP_Next: /* jump */ +case OP_Next: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5==0 || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP @@ -6409,8 +6409,8 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_DeferredSeek: -case OP_IdxRowid: { /* out2 */ +case OP_DeferredSeek: /* ncycle */ +case OP_IdxRowid: { /* out2, ncycle */ VdbeCursor *pC; /* The P1 index cursor */ VdbeCursor *pTabCur; /* The P2 table cursor (OP_DeferredSeek only) */ i64 rowid; /* Rowid that P1 current points to */ @@ -6472,8 +6472,8 @@ case OP_IdxRowid: { /* out2 */ ** seek operation now, without further delay. If the cursor seek has ** already occurred, this instruction is a no-op. */ -case OP_FinishSeek: { - VdbeCursor *pC; /* The P1 index cursor */ +case OP_FinishSeek: { /* ncycle */ + VdbeCursor *pC; /* The P1 index cursor */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -6528,10 +6528,10 @@ case OP_FinishSeek: { ** If the P1 index entry is less than or equal to the key value then jump ** to P2. Otherwise fall through to the next instruction. */ -case OP_IdxLE: /* jump */ -case OP_IdxGT: /* jump */ -case OP_IdxLT: /* jump */ -case OP_IdxGE: { /* jump */ +case OP_IdxLE: /* jump, ncycle */ +case OP_IdxGT: /* jump, ncycle */ +case OP_IdxLT: /* jump, ncycle */ +case OP_IdxGE: { /* jump, ncycle */ VdbeCursor *pC; int res; UnpackedRecord r; @@ -7952,7 +7952,7 @@ case OP_VDestroy: { ** P1 is a cursor number. This opcode opens a cursor to the virtual ** table and stores that cursor in P1. */ -case OP_VOpen: { +case OP_VOpen: { /* ncycle */ VdbeCursor *pCur; sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; @@ -7999,7 +7999,7 @@ case OP_VOpen: { ** cursor. Register P3 is used to hold the values returned by ** sqlite3_vtab_in_first() and sqlite3_vtab_in_next(). */ -case OP_VInitIn: { /* out2 */ +case OP_VInitIn: { /* out2, ncycle */ VdbeCursor *pC; /* The cursor containing the RHS values */ ValueList *pRhs; /* New ValueList object to put in reg[P2] */ @@ -8036,7 +8036,7 @@ case OP_VInitIn: { /* out2 */ ** ** A jump is made to P2 if the result set after filtering would be empty. */ -case OP_VFilter: { /* jump */ +case OP_VFilter: { /* jump, ncycle */ int nArg; int iQuery; const sqlite3_module *pModule; @@ -8096,7 +8096,7 @@ case OP_VFilter: { /* jump */ ** bits (OPFLAG_LENGTHARG or OPFLAG_TYPEOFARG) but those bits are ** unused by OP_VColumn. */ -case OP_VColumn: { +case OP_VColumn: { /* ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; Mem *pDest; @@ -8148,7 +8148,7 @@ case OP_VColumn: { ** jump to instruction P2. Or, if the virtual table has reached ** the end of its result set, then fall through to the next instruction. */ -case OP_VNext: { /* jump */ +case OP_VNext: { /* jump, ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; int res; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 0644cffa57..028eb7fe64 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -15,6 +15,7 @@ */ #include "sqliteInt.h" #include "vdbeInt.h" +#include "opcodes.h" #ifndef SQLITE_OMIT_DEPRECATED /* @@ -2225,10 +2226,7 @@ int sqlite3_stmt_scanstatus_v2( for(iOp=0; iOpnOp; iOp++){ Op *pOp = &p->aOp[iOp]; if( pOp->p1!=iEnd ) continue; - if( pOp->opcode!=OP_VFilter && pOp->opcode!=OP_VColumn - && pOp->opcode!=OP_Rowid && pOp->opcode!=OP_VOpen - && pOp->opcode!=OP_VNext - ){ + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ continue; } res += p->anCycle[iOp]; diff --git a/src/wherecode.c b/src/wherecode.c index 54c79baf97..4c22e5dd61 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -294,16 +294,26 @@ void sqlite3WhereAddScanStatus( ){ const char *zObj = 0; WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; + + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ zObj = pLoop->u.btree.pIndex->zName; }else{ zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; } sqlite3VdbeScanStatus( v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj ); - if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } } } #endif diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 3c5b4b16a3..9a4758b6d4 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -88,8 +88,8 @@ proc get_graph {stmt} { proc do_graph_test {tn sql res} { db eval $sql set stmt [db version -last-stmt-ptr] - set graph [string trim [get_graph $stmt]] + set graph [regsub -all {nCycle=[0-9]+} $graph nCycle=nnn] uplevel [list do_test $tn [list set {} $graph] [string trim $res]] } @@ -119,9 +119,9 @@ do_graph_test 1.3 { SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 } { QUERY (nCycle=nnn) ---SCAN t2 +--SCAN t2 (nCycle=nnn) --CORRELATED SCALAR SUBQUERY 1 (nCycle=nnn) -----SCAN t1 +----SCAN t1 (nCycle=nnn) } do_graph_test 1.4 { @@ -132,9 +132,9 @@ do_graph_test 1.4 { } { QUERY (nCycle=nnn) --MATERIALIZE v2 (nCycle=nnn) -----SCAN t2 ---SCAN v2 ---SCAN t1 +----SCAN t2 (nCycle=nnn) +--SCAN v2 (nCycle=nnn) +--SCAN t1 (nCycle=nnn) --USE TEMP B-TREE FOR ORDER BY (nCycle=nnn) } @@ -154,6 +154,25 @@ QUERY (nCycle=nnn) --SCAN ft VIRTUAL TABLE INDEX 0:M1 (nCycle=nnn) } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b); + CREATE TABLE x2(c, d); + + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000) + INSERT INTO x1 SELECT i, i FROM s; + INSERT INTO x2 SELECT a, b FROM x1; +} + +do_graph_test 2.1 { + SELECT * FROM x1, x2 WHERE c=+a; +} { +QUERY (nCycle=nnn) +--SCAN x1 (nCycle=nnn) +--SEARCH x2 USING AUTOMATIC COVERING INDEX (c=?) (nCycle=nnn) +} + finish_test diff --git a/tool/mkopcodeh.tcl b/tool/mkopcodeh.tcl index 8b4e345c67..6fb3b75940 100644 --- a/tool/mkopcodeh.tcl +++ b/tool/mkopcodeh.tcl @@ -86,6 +86,7 @@ while {![eof $in]} { set in3($name) 0 set out2($name) 0 set out3($name) 0 + set ncycle($name) 0 for {set i 3} {$i<[llength $line]-1} {incr i} { switch [string trim [lindex $line $i] ,] { same { @@ -107,6 +108,7 @@ while {![eof $in]} { in3 {set in3($name) 1} out2 {set out2($name) 1} out3 {set out3($name) 1} + ncycle {set ncycle($name) 1} } } if {$group($name)} { @@ -140,6 +142,7 @@ foreach name {OP_Noop OP_Explain OP_Abortable} { set in3($name) 0 set out2($name) 0 set out3($name) 0 + set ncycle($name) 0 set op($name) -1 set order($nOp) $name incr nOp @@ -285,6 +288,7 @@ for {set i 0} {$i<=$max} {incr i} { if {$in3($name)} {incr x 8} if {$out2($name)} {incr x 16} if {$out3($name)} {incr x 32} + if {$ncycle($name)} {incr x 64} } set bv($i) $x } @@ -299,6 +303,7 @@ puts "#define OPFLG_IN2 0x04 /* in2: P2 is an input */" puts "#define OPFLG_IN3 0x08 /* in3: P3 is an input */" puts "#define OPFLG_OUT2 0x10 /* out2: P2 is an output */" puts "#define OPFLG_OUT3 0x20 /* out3: P3 is an output */" +puts "#define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */" puts "#define OPFLG_INITIALIZER \173\\" for {set i 0} {$i<=$max} {incr i} { if {$i%8==0} { From 09d953706c29d9189e7c618d6c3fd579d571b335 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 6 Dec 2022 18:48:37 +0000 Subject: [PATCH 182/282] For CLI .sha3sum, emit warning to stderr for any invalidly encoded text fields. FossilOrigin-Name: 8e833ecc81367658e81acd1d3a0242ab954a62d6f719af56f1d185656d0b73e6 --- manifest | 15 ++++------ manifest.uuid | 2 +- src/shell.c.in | 74 ++++++++++++++++++++++++++------------------------ 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index 34ffa10ecc..56f9cf4031 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cause\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sdo\snot\ssurvive\sCAST(CAST(t\sas\sBLOB)\sAS\sTEXT)\sdue\sto\sinvalid\sUTF\sencoding. -D 2022-12-03T16:09:32.554 +C For\sCLI\s.sha3sum,\semit\swarning\sto\sstderr\sfor\sany\sinvalidly\sencoded\stext\sfields. +D 2022-12-06T18:48:37.123 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 -F src/shell.c.in aa3e4c78ae25c24562b0c3497a88875baa61425714437143170b2c462842035d +F src/shell.c.in cf32c16e1b4ff8e434b51fe38753bfddccd75d4a3ecb1fb838a6f96a70e2ff3a F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2065,11 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 -R 9f83c1fb14dddfc272778480afabb0a3 -T *branch * sha3sum_text_validation -T *sym-sha3sum_text_validation * -T -sym-trunk * +P 123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 +R 1a108a993a6e5c4e1463804542f95600 U larrybr -Z 1ba97ab620561a8dccaf913734ec9094 +Z 23502144cb7b8e35b977c4658f2c6efa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b35a727cc3..b0fd6bc454 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 \ No newline at end of file +8e833ecc81367658e81acd1d3a0242ab954a62d6f719af56f1d185656d0b73e6 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1f0ac24c81..a74614eaac 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10077,7 +10077,6 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_stmt *pStmt; /* For querying tables names */ char *zSql; /* SQL to be run */ char *zSep; /* Separator */ - char *zRevText; /* Query for reversible to-blob-to-text check */ ShellText sSql; /* Complete SQL for the query to run the hash */ ShellText sQuery; /* Set of queries used to read all content */ open_db(p, 0); @@ -10126,24 +10125,6 @@ static int do_meta_command(char *zLine, ShellState *p){ " ORDER BY 1 collate nocase"; } sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - zRevText = sqlite3_mprintf( - /* lower-case query is first run, producing upper-case query. */ - "with text_cols as materialized(" - "select tname, cname " - "from (" - " select ss.tname as tname, ti.name as cname, lower(type) as lca" - " from (%s) ss join pragma_table_info(tname) ti" - " where (instr(lca,'int')=0 and " - " instr(lca,'char')+instr(lca,'clob')+instr(lca,'text')>0)" - "))" - "select 'SELECT total(bad_text_count) AS bad_text_count" - " FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query" - " from (select 'SELECT COUNT(*) AS bad_text_count" - " FROM '||tname||' WHERE '" - " ||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname," - " ' OR ') as query, tname from text_cols group by tname)", - zSql); - shell_check_oom(zRevText); initText(&sQuery); initText(&sSql); appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); @@ -10200,34 +10181,56 @@ static int do_meta_command(char *zLine, ShellState *p){ shell_exec(p, zSql, 0); } { - int lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); + int lrc; + char *zRevText = /* Query for reversible to-blob-to-text check */ + "SELECT lower(name) as tname FROM sqlite_schema\n" + "WHERE type='table' AND coalesce(rootpage,0)>1\n" + "AND name NOT LIKE 'sqlite_%%'%s\n" + "ORDER BY 1 collate nocase"; + zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : ""); + zRevText = sqlite3_mprintf( + /* lower-case query is first run, producing upper-case query. */ + "with tabcols as materialized(\n" + "select tname, cname\n" + "from (" + " select ss.tname as tname, ti.name as cname\n" + " from (%z) ss\n inner join pragma_table_info(tname) ti))\n" + "select 'SELECT total(bad_text_count) AS bad_text_count\n" + "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n" + " from (select 'SELECT COUNT(*) AS bad_text_count\n" + "FROM '||tname||' WHERE '\n" + "||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname\n" + "|| ' AND typeof('||cname||')=''text'' ',\n" + "' OR ') as query, tname from tabcols group by tname)" + , zRevText); + shell_check_oom(zRevText); if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); + lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); assert(lrc==SQLITE_OK); + if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); lrc = SQLITE_ROW==sqlite3_step(pStmt); - assert(lrc!=0); if( lrc ){ const char *zGenQuery = sqlite3_column_text(pStmt,0); sqlite3_stmt *pCheckStmt; - double countIrreversible; lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); - assert(lrc==SQLITE_OK); - lrc = SQLITE_ROW==sqlite3_step(pCheckStmt); - assert(lrc!=0); - countIrreversible = sqlite3_column_double(pCheckStmt, 0); - sqlite3_finalize(pStmt); - sqlite3_finalize(pCheckStmt); - if( countIrreversible>0 ){ - int n = (int)(countIrreversible + 0.5); - utf8_printf(stderr, - "Digest includes %d invalidly encoded text field%s.\n", - n, (n>1)? "s": ""); + if( SQLITE_OK==lrc ){ + if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ + double countIrreversible = sqlite3_column_double(pCheckStmt, 0); + if( countIrreversible>0 ){ + int n = (int)(countIrreversible + 0.5); + utf8_printf(stderr, + "Digest includes %d invalidly encoded text field%s.\n", + n, (n>1)? "s": ""); + } + } + sqlite3_finalize(pCheckStmt); } + sqlite3_finalize(pStmt); } - + sqlite3_free(zRevText); } sqlite3_free(zSql); - sqlite3_free(zRevText); }else #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) @@ -11008,6 +11011,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ goto PlainScan; case '`': case '\'': case '"': if(*zLine==cWait){ + /* Swallow doubled end-delimiter.*/ ++zLine; continue; } From 3c01af6ffc4e7614c6b1a271aa48828e5d4b11dc Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 6 Dec 2022 19:20:49 +0000 Subject: [PATCH 183/282] Avoid several -Wall warnings in textfixture build. FossilOrigin-Name: 03ae8680e430c6d2c39ca26fa2cb6eed46e3c3e52b53f7a8bf548560446c6d3d --- ext/misc/base64.c | 3 ++- ext/misc/basexx.c | 1 - manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/test_demovfs.c | 1 - 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/ext/misc/base64.c b/ext/misc/base64.c index d2185b5405..81767379b1 100755 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -91,7 +91,8 @@ static const ubyte b64DigitValues[128] = { static const char b64Numerals[64+1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -#define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80) +#define BX_DV_PROTO(c) \ + ((((ubyte)(c))<0x80)? (ubyte)(b64DigitValues[(ubyte)(c)]) : 0x80) #define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80) #define IS_BX_WS(bdp) ((bdp)==WS) #define IS_BX_PAD(bdp) ((bdp)==PC) diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c index 9cb43efeaf..2ca2906fae 100644 --- a/ext/misc/basexx.c +++ b/ext/misc/basexx.c @@ -69,7 +69,6 @@ int sqlite3_basexx_init(sqlite3 *db, char **pzErr, init_api_ptr(pApi); int rc1 = BASE64_INIT(db); int rc2 = BASE85_INIT(db); - int rc = SQLITE_OK; if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ BASE64_EXPOSE(db, pzErr); diff --git a/manifest b/manifest index c4286f7f7d..8f1f6b3afe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\ssqlite3_stmt_scanstatus_v2()\sreturn\san\sNCYCLE\svalue\sfor\sall\sloops,\snot\sjust\svirtual\stables\sones.\sThe\svalue\sreturned\sis\sthe\ssum\sof\sthe\sNCYCLE\scounts\sfor\sthe\svarious\sopcodes\sthat\smove\sor\sread\sdata\sfrom\sthe\stable\sor\sindex\scursor\sassociated\swith\sthe\sloop. -D 2022-12-06T18:48:06.154 +C Avoid\sseveral\s-Wall\swarnings\sin\stextfixture\sbuild. +D 2022-12-06T19:20:49.029 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,9 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c fb140039d4e15710298e28cafe12d2bdff1b927fc57540cbcd07996f82bf9800 x +F ext/misc/base64.c 1ad313d38dc081abe5a87fa51e44f07ec6cfb618aa9606a77e42bd2fe706a67f x F ext/misc/base85.c ace591246855806a70e63eb6705ce8736f4b782da2137bdbb5d39cc337346e0d -F ext/misc/basexx.c b37ea21f00deea1029d2ddc670f5701d9370ae06f7d173c5bed79d0f26a67ec1 +F ext/misc/basexx.c 58f72b3c159e875bfd2d30a56254e7f2098961c31dc733773a9270de7066aa4c F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -672,7 +672,7 @@ F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 F src/test_config.c 8264637b06a3c1f0727c88d1ea32dcf7986b9e7e358a970cae87cdac8a5b2708 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f -F src/test_demovfs.c 7cc7623d1025d1e92c51da20fd25060759733b7a356a121545a3b7d2faa8a0f1 +F src/test_demovfs.c 38a459d1c78fd9afa770445b224c485e079018d6ac07332ff9bd07b54d2b8ce9 F src/test_devsym.c aff2255ea290d7718da08af30cdf18e470ff7325a5eff63e0057b1496ed66593 F src/test_fs.c ba1e1dc18fd3159fdba0b9c4256f14032159785320dfbd6776eb9973cb75d480 F src/test_func.c 24df3a346c012b1fc9e1001d346db6054deb426db0a7437e92490630e71c9b0a @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 -R e347a9226ee206cfe1e680e2e5e722d0 -U dan -Z eaf9f5c14b0f034256d24205aa5d85c9 +P 9499b2f51e8174c6b8a67840c92ba23b7dd1dc8dc2b91fca0c5dc07b71662149 +R 6d3961aec4c1529b866a56388835711c +U larrybr +Z 815c066cfe76a0b0ad34b27b7be96db7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 79c29e3304..914d5983d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9499b2f51e8174c6b8a67840c92ba23b7dd1dc8dc2b91fca0c5dc07b71662149 \ No newline at end of file +03ae8680e430c6d2c39ca26fa2cb6eed46e3c3e52b53f7a8bf548560446c6d3d \ No newline at end of file diff --git a/src/test_demovfs.c b/src/test_demovfs.c index 29307675bd..e990e98f29 100644 --- a/src/test_demovfs.c +++ b/src/test_demovfs.c @@ -461,7 +461,6 @@ static int demoDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ if( rc==0 && dirSync ){ int dfd; /* File descriptor open on directory */ - int i; /* Iterator variable */ char *zSlash; char zDir[MAXPATHNAME+1]; /* Name of directory containing file zPath */ From b54f71e205c578303fa50cfb11a985e9b73ca3b0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 7 Dec 2022 00:14:25 +0000 Subject: [PATCH 184/282] Fix a (harmless) off-by-one error in code generation that comes up when doing a DISTINCT query against a virtual table with an OR term in the WHERE clause and where the ORDER BY clause has 64 or more references to the result set. [forum:/forumpost/dfe8084751|Forum post dfe8084751]. FossilOrigin-Name: 04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/where.c | 2 +- test/where.test | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index fc2a006d5f..8e9257d9fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sare\snot\sto-blob-to-text\sreversible. -D 2022-12-06T19:32:07.195 +C Fix\sa\s(harmless)\soff-by-one\serror\sin\scode\sgeneration\sthat\scomes\sup\swhen\ndoing\sa\sDISTINCT\squery\sagainst\sa\svirtual\stable\swith\san\sOR\sterm\sin\sthe\nWHERE\sclause\sand\swhere\sthe\sORDER\sBY\sclause\shas\s64\sor\smore\sreferences\sto\nthe\sresult\sset.\s\s[forum:/forumpost/dfe8084751|Forum\spost\sdfe8084751]. +D 2022-12-07T00:14:25.828 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 20f4f51d2d5fb19b984e6ea381b26cf627cc93e64dd9b2ce6a94531aec2f5916 +F src/where.c adc6783cd3b136271a70001182e899d3523e199bda41ead5c3e2ab4a84456af1 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -1886,7 +1886,7 @@ F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2a F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec F test/wapptest.tcl 1bea58a6a8e68a73f542ee4fca28b771b84ed803bd0c9e385087070b3d747b3c x -F test/where.test 98208c95b574269980132c347b4bdb8992c6d5fc30c1954938593336d12e7447 +F test/where.test 3954cf22ba7b17f9606e177001d2963bcd1ecfbc6e1e7caadb14462f7eecd099 F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 @@ -2067,9 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 03ae8680e430c6d2c39ca26fa2cb6eed46e3c3e52b53f7a8bf548560446c6d3d 1d01f8483af7b6a4e93a49fd8bbb6a2b9300e969b574efeb96d56081a33055c4 -R 2aee8123cf630f6dc747669c6dfb6cfe -T +closed 1d01f8483af7b6a4e93a49fd8bbb6a2b9300e969b574efeb96d56081a33055c4 -U larrybr -Z 5b22d70d9e69e4deb5a1464c5dd02f65 +P bbde0f36d03cdbbc749427fe7d2dafd5c5031c9e655ebd772857b521f53eb18f +R 250ed9b5c78a34895ae8f1ae12a9ecc7 +U drh +Z d0ffb89df284ba3dde1a0bb1536bd8ea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0a576fcc73..d731886fa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbde0f36d03cdbbc749427fe7d2dafd5c5031c9e655ebd772857b521f53eb18f \ No newline at end of file +04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a133799472..1cf9592070 100644 --- a/src/where.c +++ b/src/where.c @@ -67,7 +67,7 @@ int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ ** block sorting is required. */ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; + return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat; } /* diff --git a/test/where.test b/test/where.test index 3f4aa8e1be..4f7c2f84b2 100644 --- a/test/where.test +++ b/test/where.test @@ -1618,4 +1618,19 @@ do_execsql_test where-28.1 { 19 5 } +# 2022-12-07 Yong Heng [https://sqlite.org/forum/forumpost/dfe8084751] +# +do_execsql_test where-29.1 { + SELECT DISTINCT 'xyz' FROM pragma_cache_size + WHERE rowid OR abs(0) + ORDER BY + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1; +} {xyz} + finish_test From 30da58c5d66c386129a872ccd3a13b452c67717a Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 7 Dec 2022 03:42:39 +0000 Subject: [PATCH 185/282] Add addOnDispose() method to Jaccwabyt and code-adjacent minor internal cleanups. FossilOrigin-Name: 6a2723fe3f28dd94328d901e64e1e9ee9a1b2e9eeaed6c54038a5b83c914db78 --- ext/wasm/api/sqlite3-v-helper.js | 4 +-- ext/wasm/jaccwabyt/jaccwabyt.js | 47 ++++++++++++++++++++------------ ext/wasm/jaccwabyt/jaccwabyt.md | 33 ++++++++++++++++++++-- ext/wasm/tester1.c-pp.js | 19 +++++++++++-- manifest | 18 ++++++------ manifest.uuid | 2 +- 6 files changed, 88 insertions(+), 35 deletions(-) diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index a23070b24c..1f2950f749 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -350,10 +350,10 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ methods which take a (sqlite3_vtab_cursor*) _except_ for xClose(), in which case use... - - wrapCursor(pCursor, true) will remove the m apping of pCursor to a + - wrapCursor(pCursor, true) will remove the mapping of pCursor to a capi.sqlite3_vtab_cursor object and return that object. The caller must call dispose() on the returned object. This is - intended to be called form xClose() or in error handling of a + intended to be called from xClose() or in error handling of a failed xOpen(). */ vt.xWrapCursor = __xWrapFactory(capi.sqlite3_vtab_cursor); diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index d5225f6e4d..f02c8bbdef 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -10,9 +10,12 @@ *********************************************************************** - The Jaccwabyt API is documented in detail in an external file. + The Jaccwabyt API is documented in detail in an external file, + _possibly_ called jaccwabyt.md in the same directory as this file. - Project home: https://fossil.wanderinghorse.net/r/jaccwabyt + Project homes: + - https://fossil.wanderinghorse.net/r/jaccwabyt + - https://sqlite.org/src/dir/ext/wasm/jaccwabyt */ 'use strict'; @@ -219,15 +222,9 @@ self.Jaccwabyt = function StructBinderFactory(config){ const __freeStruct = function(ctor, obj, m){ if(!m) m = __instancePointerMap.get(obj); if(m) { - if(obj.ondispose instanceof Function){ - try{obj.ondispose()} - catch(e){ - /*do not rethrow: destructors must not throw*/ - console.warn("ondispose() for",ctor.structName,'@', - m,'threw. NOT propagating it.',e); - } - }else if(Array.isArray(obj.ondispose)){ - obj.ondispose.forEach(function(x){ + if(Array.isArray(obj.ondispose)){ + let x; + while((x = obj.ondispose.shift())){ try{ if(x instanceof Function) x.call(obj); else if(x instanceof StructType) x.dispose(); @@ -238,7 +235,14 @@ self.Jaccwabyt = function StructBinderFactory(config){ console.warn("ondispose() for",ctor.structName,'@', m,'threw. NOT propagating it.',e); } - }); + } + }else if(obj.ondispose instanceof Function){ + try{obj.ondispose()} + catch(e){ + /*do not rethrow: destructors must not throw*/ + console.warn("ondispose() for",ctor.structName,'@', + m,'threw. NOT propagating it.',e); + } } delete obj.ondispose; __instancePointerMap.delete(obj); @@ -333,7 +337,9 @@ self.Jaccwabyt = function StructBinderFactory(config){ /** Impl of X.memberKeys() for StructType and struct ctors. */ const __structMemberKeys = rop(function(){ const a = []; - Object.keys(this.structInfo.members).forEach((k)=>a.push(this.memberKey(k))); + for(const k of Object.keys(this.structInfo.members)){ + a.push(this.memberKey(k)); + } return a; }); @@ -399,15 +405,15 @@ self.Jaccwabyt = function StructBinderFactory(config){ Adds value v to obj.ondispose, creating ondispose, or converting it to an array, if needed. */ - const __addOnDispose = function(obj, v){ + const __addOnDispose = function(obj, ...v){ if(obj.ondispose){ - if(obj.ondispose instanceof Function){ + if(!Array.isArray(obj.ondispose)){ obj.ondispose = [obj.ondispose]; - }/*else assume it's an array*/ + } }else{ obj.ondispose = []; } - obj.ondispose.push(v); + obj.ondispose.push(...v); }; /** @@ -494,6 +500,13 @@ self.Jaccwabyt = function StructBinderFactory(config){ return __setMemberCString(this, memberName, str); }) }); + // Function-type non-Property inherited members + Object.assign(StructType.prototype,{ + addOnDispose: function(...v){ + __addOnDispose(this,...v); + return this; + } + }); /** "Static" properties for StructType. diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index 49e2d3b8ff..dd80ed1c68 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -4,7 +4,6 @@ Jaccwabyt 🐇 **Jaccwabyt**: _JavaScript ⇄ C Struct Communication via WASM Byte Arrays_ - Welcome to Jaccwabyt, a JavaScript API which creates bindings for WASM-compiled C structs, defining them in such a way that changes to their state in JS are visible in C/WASM, and vice versa, permitting @@ -27,8 +26,28 @@ are based solely on feature compatibility tables provided at **Formalities:** - Author: [Stephan Beal][sgb] -- License: Public Domain -- Project Home: +- Project Homes: + - \ + Is the primary home but... + - \ + ... most development happens here. + +The license for both this documentation and the software it documents +is the same as [sqlite3][], the project from which this spinoff +project was spawned: + +----- + +> 2022-06-30: +> +> The author disclaims copyright to this source code. In place of a +> legal notice, here is a blessing: +> +> May you do good and not evil. +> May you find forgiveness for yourself and forgive others. +> May you share freely, never taking more than you give. + +----- Table of Contents @@ -571,6 +590,14 @@ only called by the [StructBinder][]-generated has the following "static" properties (^Which are accessible from individual instances via `theInstance.constructor`.): +- `addOnDispose(...value)`\ + If this object has no `ondispose` property, this function creates it + as an array and pushes the given value(s) onto it. If the object has + a function-typed `ondispose` property, this call replaces it with an + array and moves that function into the array. In all other cases, + `ondispose` is assumed to be an array and the argument(s) is/are + appended to it. Returns `this`. + - `allocCString(str)` Identical to the [StructBinder][] method of the same name. diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 7013f3cea8..910bbc94e6 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -710,7 +710,7 @@ self.sqlite3InitModule = sqlite3InitModule; }/*WhWasmUtil*/) //////////////////////////////////////////////////////////////////// - .t('sqlite3.StructBinder (jaccwabyt)', function(sqlite3){ + .t('sqlite3.StructBinder (jaccwabyt🐇)', function(sqlite3){ const S = sqlite3, W = S.wasm; const MyStructDef = { sizeof: 16, @@ -864,6 +864,18 @@ self.sqlite3InitModule = sqlite3InitModule; }finally{ wts.dispose(); } + + if(1){ // ondispose of other struct instances + const s1 = new WTStruct, s2 = new WTStruct, s3 = new WTStruct; + T.assert(s1.lookupMember instanceof Function) + .assert(s1.addOnDispose instanceof Function); + s1.addOnDispose(s2,"testing variadic args"); + T.assert(2===s1.ondispose.length); + s2.addOnDispose(s3); + s1.dispose(); + T.assert(!s2.pointer,"Expecting s2 to be ondispose'd by s1."); + T.assert(!s3.pointer,"Expecting s3 to be ondispose'd by s2."); + } }/*StructBinder*/) //////////////////////////////////////////////////////////////////// @@ -1549,6 +1561,7 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(args[0] === 'testvtab') .assert(args[1] === 'main') .assert(args[2] === 'testvtab'); + console.debug("xConnect() args =",args); const rc = capi.sqlite3_declare_vtab( pDb, "CREATE TABLE ignored(a,b)" ); @@ -1577,8 +1590,8 @@ self.sqlite3InitModule = sqlite3InitModule; xOpen: function(pVtab, ppCursor){ try{ const t = vth.xWrapVtab(pVtab), c = vth.xWrapCursor(); - T.assert(t instanceof capi.sqlite3_vtab); - T.assert(c instanceof capi.sqlite3_vtab_cursor); + T.assert(t instanceof capi.sqlite3_vtab) + .assert(c instanceof capi.sqlite3_vtab_cursor); wasm.setPtrValue(ppCursor, c.pointer); c._rowId = 0; return 0; diff --git a/manifest b/manifest index 3bfa27aa70..e691430191 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C JS\svtables:\sadd\sinfrastructure\srelated\sto\saccessing\sand\smodifying\ssqlite3_index_info. -D 2022-12-06T11:21:46.303 +C Add\saddOnDispose()\smethod\sto\sJaccwabyt\sand\scode-adjacent\sminor\sinternal\scleanups. +D 2022-12-07T03:42:39.134 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -509,7 +509,7 @@ F ext/wasm/api/sqlite3-api-prologue.js a7596c30392d9ca8f5b7c14feb4e6788107d1159f F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 -F ext/wasm/api/sqlite3-v-helper.js 6b408ee4e926cb0f7fe41a63a1205049283af301fe3f5de3c038845ccf9106df +F ext/wasm/api/sqlite3-v-helper.js 2125255f102ab07a2653d74d4931d716b0d27c5a89bebc188ad828de0dd1dcef F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 8ec510fee735c646fb18a3b99f0ca5ca461f9e066c43cdc404d7144f12ae6ed6 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 723522a6c2a2463884a83fa1cc7ae5770deaaf0856a1058cc1023b2bfa1c898b @@ -539,8 +539,8 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff -F ext/wasm/jaccwabyt/jaccwabyt.js 35c7eaa61ba4b875cd49da5a06a35d9935fd19121de65dd5f467cbe376359782 -F ext/wasm/jaccwabyt/jaccwabyt.md 888aff20e486abb6c955d432f9a3af271e2cdd2cd99a92c6f87077116ca57091 +F ext/wasm/jaccwabyt/jaccwabyt.js b7efd07ea3927e9d0bc75b3819c6d40bdfb0a03cbc8d93331be16799f35261c3 +F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 -F ext/wasm/tester1.c-pp.js c5679da7895377df03e6075fc0e9dff8b5f570bd4edb63f714154d3030279fce +F ext/wasm/tester1.c-pp.js 8144e0e0f76b14fcd5223fad190bd94a50caf19b2419848b782448cda3ccbc78 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0ce51bed65d5e430364f74bf959fb76c42ac5eec0769490231d8c8110a1f388c -R a1b37a98bcb406acaf386e5a65bbf6e3 +P 0d77c348039926c24e0fb50a7dc7e4b62895cd201c021f8e29832917e4b8b09f +R c60d510be7a6225cada64eb18dd4b778 U stephan -Z 3e0a3245949f854c4eed760afd40e3ce +Z 9a43ff51fca7fb341935fe3d300c9a6e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a5ae05d6f7..ae8e40499b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d77c348039926c24e0fb50a7dc7e4b62895cd201c021f8e29832917e4b8b09f \ No newline at end of file +6a2723fe3f28dd94328d901e64e1e9ee9a1b2e9eeaed6c54038a5b83c914db78 \ No newline at end of file From 1eb1b59b893efb2fd4244d0116beb3c7db250d48 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 7 Dec 2022 07:22:34 +0000 Subject: [PATCH 186/282] Work on an alternate (slightly simpler) approach to binding JS vtabs. Non-eponymous vtabs are not working, for reasons as yet unknown. FossilOrigin-Name: 6a0fefb93bcccd950df211cf5c2f49660c7b92115dd01b2b508a4ab9e3ab3d23 --- ext/wasm/api/sqlite3-api-prologue.js | 55 ++- ext/wasm/api/sqlite3-v-helper.js | 258 ++++++++--- ext/wasm/jaccwabyt/jaccwabyt.js | 2 +- ext/wasm/tester1.c-pp.js | 625 +++++++++++++++++---------- manifest | 18 +- manifest.uuid | 2 +- 6 files changed, 659 insertions(+), 301 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index bccae8b199..e6a8c0fc3c 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -185,28 +185,49 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** Constructs this object with a message depending on its arguments: - - If it's passed only a single integer argument, it is assumed - to be an sqlite3 C API result code. The message becomes the - result of sqlite3.capi.sqlite3_js_rc_str() or (if that returns - falsy) a synthesized string which contains that integer. + If its first argument is an integer, it is assumed to be + an SQLITE_... result code and it is passed to + sqlite3.capi.sqlite3_js_rc_str() to stringify it. - - If passed 2 arguments and the 2nd is a object, it behaves - like the Error(string,object) constructor except that the first - argument is subject to the is-integer semantics from the - previous point. + If called with exactly 2 arguments and the 2nd is an object, + that object is treated as the 2nd argument to the parent + constructor. - - Else all arguments are concatenated with a space between each - one, using args.join(' '), to create the error message. + The exception's message is created by concatenating its + arguments with a space between each, except for the + two-args-with-an-objec form and that the first argument will + get coerced to a string, as described above, if it's an + integer. + + If passed an integer first argument, the error object's + `resultCode` member will be set to the given integer value, + else it will be set to capi.SQLITE_ERROR. */ constructor(...args){ - if(1===args.length && __isInt(args[0])){ - super(__rcStr(args[0])); - }else if(2===args.length && 'object'===typeof args[1]){ - if(__isInt(args[0])) super(__rcStr(args[0]), args[1]); - else super(...args); - }else{ - super(args.join(' ')); + let rc; + if(args.length){ + if(__isInt(args[0])){ + rc = args[0]; + if(1===args.length){ + super(__rcStr(args[0])); + }else{ + const rcStr = __rcStr(rc); + if('object'===typeof args[1]){ + super(rcStr,args[1]); + }else{ + args[0] = rcStr+':'; + super(args.join(' ')); + } + } + }else{ + if(2===args.length && 'object'===typeof args[1]){ + super(...args); + }else{ + super(args.join(' ')); + } + } } + this.resultCode = rc || capi.SQLITE_ERROR; this.name = 'SQLite3Error'; } }; diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index 1f2950f749..73ba8cc49e 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -16,7 +16,7 @@ */ 'use strict'; self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss; + const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss3; const vh = Object.create(null), vt = Object.create(null); sqlite3.VfsHelper = vh; @@ -72,21 +72,29 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Installs a StructBinder-bound function pointer member of the given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws + that proxy to be cleaned up when tgt.dispose() is called. Throws on the slightest hint of error, e.g. tgt is-not-a StructType, name does not map to a struct-bound member, etc. - Returns a proxy for this function which is bound to tgt and takes - 2 args (name,func). That function returns the same thing, - permitting calls to be chained. + As a special case, if the given function is a pointer, it is + assumed to be an existing WASM-bound function pointer and is used + as-is with no extra level of proxying or cleanup. Results are + undefined if it's a pointer and it's _not_ a function pointer. + It is legal to pass a value of 0, indicating a NULL pointer, with + the caveat that 0 _is_ a legal function pointer in WASM but it + will not be accepted as such _here_. (Justification: the function + at address zero must be one which initially came from the WASM + module, not a method we want to bind to a virtual table or VFS.) + + This function returns a proxy for itself which is bound to tgt + and takes 2 args (name,func). That function returns the same + thing as this one, permitting calls to be chained. If called with only 1 arg, it has no side effects but returns a func with the same signature as described above. - If tgt.ondispose is set before this is called then it _must_ - be an array, to which this function will append entries. - ACHTUNG: because we cannot generically know how to transform JS exceptions into result codes, the installed functions do no automatic catching of exceptions. It is critical, to avoid @@ -94,20 +102,20 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this function do not throw. The exception, as it were, to that rule is... - If applyArgcCheck is true then each method gets wrapped in a - proxy which asserts that it is passed the expected number of - arguments, throwing if the argument count does not match - expectations. That is only intended for dev-time usage for sanity - checking, and will leave the C environment in an undefined - state. For non-dev-time use, it is a given that the C API will - never call one of the generated function wrappers with the wrong - argument count. + If applyArgcCheck is true then each JS function (as opposed to + function pointers) gets wrapped in a proxy which asserts that it + is passed the expected number of arguments, throwing if the + argument count does not match expectations. That is only intended + for dev-time usage for sanity checking, and will leave the C + environment in an undefined state. */ vh.installMethod = vt.installMethod = function callee( tgt, name, func, applyArgcCheck = callee.installMethodArgcCheck ){ if(!(tgt instanceof sqlite3.StructBinder.StructType)){ toss("Usage error: target object is-not-a StructType."); + }else if(!(func instanceof Function) && !wasm.isPtr(func)){ + toss("Usage errror: expecting a Function or WASM pointer to one."); } if(1===arguments.length){ return (n,f)=>callee(tgt, n, f, applyArgcCheck); @@ -143,26 +151,33 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*static init*/ const sigN = tgt.memberSignature(name); if(sigN.length<2){ - toss("Member",name," is not a function pointer. Signature =",sigN); + toss("Member",name,"does not have a function pointer signature:",sigN); } const memKey = tgt.memberKey(name); - const fProxy = applyArgcCheck + const fProxy = (applyArgcCheck && !wasm.isPtr(func)) /** This middle-man proxy is only for use during development, to confirm that we always pass the proper number of arguments. We know that the C-level code will always use the correct argument count. */ ? callee.argcProxy(tgt, memKey, func, sigN) : func; - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose) tgt.ondispose = []; - else if(!Array.isArray(tgt.ondispose)) tgt.ondispose = [tgt.ondispose]; - if(!tgt.ondispose.__removeFuncList){ - tgt.ondispose.push('ondispose.__removeFuncList handler', + if(wasm.isPtr(fProxy)){ + if(fProxy && !wasm.functionEntry(fProxy)){ + toss("Pointer",fProxy,"is not a WASM function table entry."); + } + tgt[memKey] = fProxy; + }else{ + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + else if(!Array.isArray(tgt.ondispose)) tgt.ondispose = [tgt.ondispose]; + if(!tgt.ondispose || !tgt.ondispose.__removeFuncList){ + tgt.addOnDispose('ondispose.__removeFuncList handler', callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); return (n,f)=>callee(tgt, n, f, applyArgcCheck); }/*installMethod*/; vh.installMethod.installMethodArgcCheck = false; @@ -269,8 +284,9 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this.installMethods(o.struct, o.methods, !!o.applyArgcCheck); if('vfs'===key){ if(!o.struct.$zName && 'string'===typeof o.name){ - o.struct.$zName = wasm.allocCString(o.name); - /* Note that we leak that C-string. */ + o.struct.addOnDispose( + o.struct.$zName = wasm.allocCString(o.name) + ); } this.registerVfs(o.struct, !!o.asDefault); } @@ -292,24 +308,20 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ vt.sqlite3ValuesToJs = capi.sqlite3_create_function_v2.udfConvertArgs; /** - Factory function for wrapXyz() impls. + Factory function for xAbc() impls. */ - const __xWrapFactory = function(structType){ - return function(ptr,remove=false){ + const __xWrapFactory = function(methodName,structType){ + return function(ptr,removeMapping=false){ if(0===arguments.length) ptr = new structType; if(ptr instanceof structType){ //T.assert(!this.has(ptr.pointer)); this.set(ptr.pointer, ptr); return ptr; }else if(!wasm.isPtr(ptr)){ - sqlite3.SQLite3Error.toss("Invalid argument to xWrapFactory"); + sqlite3.SQLite3Error.toss("Invalid argument to",methodName+"()"); } let rc = this.get(ptr); - if(remove) this.delete(ptr); - /*arguable else if(!rc){ - rc = new structType(ptr); - this.set(ptr, rc); - }*/ + if(removeMapping) this.delete(ptr); return rc; }.bind(new Map); }; @@ -336,7 +348,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ called from sqlite3_module::xDisconnect() implementations or in error handling of a failed xCreate() or xConnect(). */ - vt.xWrapVtab = __xWrapFactory(capi.sqlite3_vtab); + vt.xVtab = __xWrapFactory('xVtab',capi.sqlite3_vtab); /** EXPERIMENTAL. DO NOT USE IN CLIENT CODE. @@ -356,7 +368,16 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ intended to be called from xClose() or in error handling of a failed xOpen(). */ - vt.xWrapCursor = __xWrapFactory(capi.sqlite3_vtab_cursor); + vt.xCursor = __xWrapFactory('xCursor',capi.sqlite3_vtab_cursor); + + /** + Convenience form of creating an sqlite3_index_info wrapper, + intended for use in xBestIndex implementations. Note that the + caller is expected to call dispose() on the returned object + before returning. Though not _strictly_ required, as that object + does not own the pIdxInfo memory, it is nonetheless good form. + */ + vt.xIndexInfo = (pIdxInfo)=>new capi.sqlite3_index_info(pIdxInfo); /** Given an error object, this function returns @@ -394,27 +415,38 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ let rc = ... return rc; }catch(e){ - return sqlite3.VtabHelper.xMethodError( + return sqlite3.VtabHelper.xError( 'xColumn', e, sqlite3.capi.SQLITE_XYZ); - // where SQLITE_XYZ is some call-appropriate result code - // defaulting to SQLITE_ERROR. + // where SQLITE_XYZ is some call-appropriate result code. } ``` - If xMethodError.errorReporter is a function, it is called in + If no 3rd argument is provided, its default depends on + the error type: + + - An sqlite3.WasmAllocError always resolves to capi.SQLITE_NOMEM. + + - If err is an SQLite3Error then its `resultCode` property + is used. + + - If all else fails, capi.SQLITE_ERROR is used. + + If xError.errorReporter is a function, it is called in order to report the error, else the error is not reported. If that function throws, that exception is ignored. */ - vt.xMethodError = function f(methodName, err, defaultRc=capi.SQLITE_ERROR){ + vt.xError = function f(methodName, err, defaultRc){ if(f.errorReporter instanceof Function){ try{f.errorReporter("sqlite3_module::"+methodName+"(): "+err.message);} catch(e){/*ignored*/} } - return (err instanceof sqlite3.WasmAllocError) - ? capi.SQLITE_NOMEM - : defaultRc; + let rc; + if(err instanceof sqlite3.WasmAllocError) rc = capi.SQLITE_NOMEM; + else if(arguments.length>2) rc = defaultRc; + else if(err instanceof sqlite3.SQLite3Error) rc = err.resultCode; + return rc || capi.SQLITE_ERROR; }; - vt.xMethodError.errorReporter = 1 ? console.error.bind(console) : false; + vt.xError.errorReporter = 1 ? console.error.bind(console) : false; /** "The problem" with this is that it introduces an outer function with @@ -426,15 +458,139 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** vt.methodCatcher = function(methodName, method, defaultErrRc=capi.SQLITE_ERROR){ return function(...args){ try { method(...args); } - }catch(e){ return vt.xMethodError(methodName, e, defaultRc) } + }catch(e){ return vt.xError(methodName, e, defaultRc) } }; */ /** - A helper for sqlite3_vtab::xRow() implementations. It must be + A helper for sqlite3_vtab::xRowid() implementations. It must be passed that function's 2nd argument and the value for that pointer. Returns the same as wasm.setMemValue() and will throw if the 1st or 2nd arguments are invalid for that function. */ - vt.setRowId = (ppRowid64, value)=>wasm.setMemValue(ppRowid64, value, 'i64'); + vt.xRowid = (ppRowid64, value)=>wasm.setMemValue(ppRowid64, value, 'i64'); + + /** + Sets up an sqlite3_module() object for later installation into + individual databases using sqlite3_create_module(). Requires an + object with the following properties: + + - `methods`: an object containing a mapping of properties with + the C-side names of the sqlite3_module methods, e.g. xCreate, + xBestIndex, etc., to JS implementations for those functions. + Certain special-case handling is performed, as described below. + + - `catchExceptions` (default=false): if truthy, the given methods + are not mapped as-is, but are instead wrapped inside wrappers + which translate exceptions into result codes of SQLITE_ERROR or + SQLITE_NOMEM, depending on whether the exception is an + sqlite3.WasmAllocError. In the case of the xConnect and xCreate + methods, the exception handler also sets the output error + string to the exception's error string. + + - OPTIONAL `struct`: a sqlite3.capi.sqlite3_module() instance. If + not set, one will be created automatically and (on success) + added to the object. + + - OPTIONAL `iVersion`: if set, it must be an integer value and it + gets assigned to the `$iVersion` member of the struct object. + If it's _not_ set, and the passed-in `struct` object's `$iVersion` + is 0 (the default) then this function attempts to define a value + for that property based on the list of methods it has. + + If `catchExceptions` is false, it is up to the client to ensure + that no exceptions escape the methods, as doing so would move + them through the C API, leading to undefined + behavior. (VtabHelper.xError() is intended to assist in reporting + such exceptions.) + + If `methods.xConnect` is `true` then the value of + `methods.xCreate` is used in its place, and vice versa. This is + to facilitate creation of those methods inline in the passed-in + object without requiring the client to explicitly get a reference + to one of them in order to assign it to the other one. Note that + sqlite treats those two functions specially if they are exactly + the same function (same pointer value). The + `catchExceptions`-installed handlers will account for identical + references to those two functions and will install the same + wrapper function for both. + + The given methods are expected to return integer values, as + expected by the C API. If `catchExceptions` is truthy, the return + value of the wrapped function will be used as-is and will be + translated to 0 if the function returns a falsy value (e.g. if it + does not have an explicit return). If `catchExceptions` is _not_ + active, the method implementations must explicitly return integer + values. + + Throws on error. Returns the sqlite3_module object on success. + */ + vt.setupModule = function(opt){ + const mod = opt.struct || new capi.sqlite3_module(); + try{ + const methods = opt.methods || toss("Missing 'methods' object."); + if(true===methods.xConnect) methods.xConnect = methods.xCreate; + else if(true===methods.xCreate) methods.xCreate = methods.xConnect; + if(opt.catchExceptions){ + const fwrap = function(methodName, func){ + if(['xConnect','xCreate'].indexOf(methodName) >= 0){ + return function(pDb, pAux, argc, argv, ppVtab, pzErr){ + try{return func(...arguments) || 0;} + catch(e){ + if(!(e instanceof sqlite3.WasmAllocError)){ + wasm.setPtrValue(pzErr, wasm.allocCString(e.message)); + } + return vt.xError(methodName, e); + } + }; + }else{ + return function(...args){ + try{return func(...args) || 0;} + catch(e){ + return vt.xError(methodName, e); + } + }; + } + }; + const mnames = [ + 'xCreate', 'xConnect', 'xBestIndex', 'xDisconnect', + 'xDestroy', 'xOpen', 'xClose', 'xFilter', 'xNext', + 'xEof', 'xColumn', 'xRowid', 'xUpdate', + 'xBegin', 'xSync', 'xCommit', 'xRollback', + 'xFindFunction', 'xRename', 'xSavepoint', 'xRelease', + 'xRollbackTo', 'xShadowName' + ]; + const remethods = Object.create(null); + for(const k of mnames){ + const m = methods[k]; + if(!(m instanceof Function)) continue; + else if('xConnect'===k && methods.xCreate===m){ + remethods[k] = methods.xCreate; + }else if('xCreate'===k && methods.xConnect===m){ + remethods[k] = methods.xConnect; + }else{ + remethods[k] = fwrap(k, m); + } + } + this.installMethods(mod, remethods, false); + }else{ + this.installMethods( + mod, methods, !!opt.applyArgcCheck/*undocumented option*/ + ); + } + if(0===mod.$iVersion){ + let v; + if('number'===typeof opt.iVersion) v = opt.iVersion; + else if(mod.$xShadowName) v = 3; + else if(mod.$xSavePoint || mod.$xRelease || mod.$xRollbackTo) v = 2; + else v = 1; + mod.$iVersion = v; + } + }catch(e){ + if(!opt.struct) mod.dispose(); + throw e; + } + if(!opt.struct) opt.struct = mod; + return mod; + }/*setupModule()*/; }/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index f02c8bbdef..d4ec719fb5 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -222,6 +222,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ const __freeStruct = function(ctor, obj, m){ if(!m) m = __instancePointerMap.get(obj); if(m) { + __instancePointerMap.delete(obj); if(Array.isArray(obj.ondispose)){ let x; while((x = obj.ondispose.shift())){ @@ -245,7 +246,6 @@ self.Jaccwabyt = function StructBinderFactory(config){ } } delete obj.ondispose; - __instancePointerMap.delete(obj); if(ctor.debugFlags.__flags.dealloc){ log("debug.dealloc:",(obj[xPtrPropName]?"EXTERNAL":""), ctor.structName,"instance:", diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 910bbc94e6..83b02522e4 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -301,14 +301,12 @@ self.sqlite3InitModule = sqlite3InitModule; addTest: function(name, callback){ let predicate; if(1===arguments.length){ - const opt = arguments[0]; - predicate = opt.predicate; - name = opt.name; - callback = opt.test; + this.currentTestGroup.addTest(arguments[0]); + }else{ + this.currentTestGroup.addTest({ + name, predicate, test: callback + }); } - this.currentTestGroup.addTest({ - name, predicate, test: callback - }); return this; }, runTests: async function(sqlite3){ @@ -399,12 +397,21 @@ self.sqlite3InitModule = sqlite3InitModule; catch(e){T.assert("test ing ." === e.message)} try{ throw new sqlite3.SQLite3Error(capi.SQLITE_SCHEMA) } - catch(e){ T.assert('SQLITE_SCHEMA' === e.message) } + catch(e){ + T.assert('SQLITE_SCHEMA' === e.message) + .assert(capi.SQLITE_SCHEMA === e.resultCode); + } try{ sqlite3.SQLite3Error.toss(capi.SQLITE_CORRUPT,{cause: true}) } catch(e){ - T.assert('SQLITE_CORRUPT'===e.message) + T.assert('SQLITE_CORRUPT' === e.message) + .assert(capi.SQLITE_CORRUPT === e.resultCode) .assert(true===e.cause); } + try{ sqlite3.SQLite3Error.toss("resultCode check") } + catch(e){ + T.assert(capi.SQLITE_ERROR === e.resultCode) + .assert('resultCode check' === e.message); + } }) //////////////////////////////////////////////////////////////////// .t('strglob/strlike', function(sqlite3){ @@ -988,6 +995,20 @@ self.sqlite3InitModule = sqlite3InitModule; const dbFile = '/tester1.db'; wasm.sqlite3_wasm_vfs_unlink(0, dbFile); const db = this.db = new sqlite3.oo1.DB(dbFile, 0 ? 'ct' : 'c'); + db.onclose = { + disposeThese: [], + after: function(){ + while(this.disposeThese.length){ + const v = this.disposeThese.shift(); + console.debug("db.onclose cleaning up:",v); + if(wasm.isPtr(v)) wasm.dealloc(v); + else if(v instanceof sqlite3.StructBinder.StructType){ + v.dispose(); + } + } + } + }; + T.assert(Number.isInteger(db.pointer)) .mustThrowMatching(()=>db.pointer=1, /read-only/) .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) @@ -1539,219 +1560,6 @@ self.sqlite3InitModule = sqlite3InitModule; T.mustThrow(()=>db.exec("select * from foo.bar")); }) - //////////////////////////////////////////////////////////////////////// - .t({ - name: 'Custom virtual tables', - predicate: ()=>wasm.bigIntEnabled, - test: function(sqlite3){ - warn("The vtab/module JS bindings are experimental and subject to change."); - const vth = sqlite3.VtabHelper; - const tmplCols = Object.assign(Object.create(null),{ - A: 0, B: 1 - }); - /** - The vtab demonstrated here is a JS-ification of - ext/misc/templatevtab.c. - */ - const tmplMethods = { - xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ - try{ - const args = wasm.cArgvToJs(argc, argv); - T.assert(args.length>=3) - .assert(args[0] === 'testvtab') - .assert(args[1] === 'main') - .assert(args[2] === 'testvtab'); - console.debug("xConnect() args =",args); - const rc = capi.sqlite3_declare_vtab( - pDb, "CREATE TABLE ignored(a,b)" - ); - if(0===rc){ - const t = vth.xWrapVtab(); - wasm.setPtrValue(ppVtab, t.pointer); - T.assert(t === vth.xWrapVtab(wasm.getPtrValue(ppVtab))); - } - return rc; - }catch(e){ - if(!(e instanceof sqlite3.WasmAllocError)){ - wasm.setPtrValue(pzErr, wasm.allocCString(e.message)); - } - return vth.xMethodError('xConnect',e); - } - }, - xDisconnect: function(pVtab){ - try { - const t = vth.xWrapVtab(pVtab, true); - t.dispose(); - return 0; - }catch(e){ - return vth.xMethodError('xDisconnect',e); - } - }, - xOpen: function(pVtab, ppCursor){ - try{ - const t = vth.xWrapVtab(pVtab), c = vth.xWrapCursor(); - T.assert(t instanceof capi.sqlite3_vtab) - .assert(c instanceof capi.sqlite3_vtab_cursor); - wasm.setPtrValue(ppCursor, c.pointer); - c._rowId = 0; - return 0; - }catch(e){ - return vth.xMethodError('xOpen',e); - } - }, - xClose: function(pCursor){ - try{ - const c = vth.xWrapCursor(pCursor,true); - T.assert(c instanceof capi.sqlite3_vtab_cursor) - .assert(!vth.xWrapCursor(pCursor)); - c.dispose(); - return 0; - }catch(e){ - return vth.xMethodError('xClose',e); - } - }, - xNext: function(pCursor){ - try{ - const c = vth.xWrapCursor(pCursor); - ++c._rowId; - return 0; - }catch(e){ - return vth.xMethodError('xNext',e); - } - - }, - xColumn: function(pCursor, pCtx, iCol){ - try{ - const c = vth.xWrapCursor(pCursor); - switch(iCol){ - case tmplCols.A: - capi.sqlite3_result_int(pCtx, 1000 + c._rowId); - break; - case tmplCols.B: - capi.sqlite3_result_int(pCtx, 2000 + c._rowId); - break; - default: sqlite3.SQLite3Error.toss("Invalid column id",iCol); - } - return 0; - }catch(e){ - return vth.xMethodError('xColumn',e); - } - }, - xRowid: function(pCursor, ppRowid64){ - try{ - const c = vth.xWrapCursor(pCursor); - vth.setRowId(ppRowid64, c._rowId); - return 0; - }catch(e){ - return vth.xMethodError('xRowid',e); - } - }, - xEof: function(pCursor){ - const c = vth.xWrapCursor(pCursor); - return c._rowId>=10; - }, - xFilter: function(pCursor, idxNum, idxCStr, - argc, argv/* [sqlite3_value* ...] */){ - try{ - const c = vth.xWrapCursor(pCursor); - c._rowId = 0; - const list = vth.sqlite3ValuesToJs(argc, argv); - T.assert(argc === list.length); - //log(argc,"xFilter value(s):",list); - return 0; - }catch(e){ - return vth.xMethodError('xFilter',e); - } - }, - xBestIndex: function(pVtab, pIdxInfo){ - try{ - //const t = vth.xWrapVtab(pVtab); - const sii = capi.sqlite3_index_info; - const pii = new sii(pIdxInfo); - pii.$estimatedRows = 10; - pii.$estimatedCost = 10.0; - //log("xBestIndex $nConstraint =",pii.$nConstraint); - if(pii.$nConstraint>0){ - // Validate nthConstraint() and nthConstraintUsage() - const max = pii.$nConstraint; - for(let i=0; i < max; ++i ){ - let v = pii.nthConstraint(i,true); - T.assert(wasm.isPtr(v)); - v = pii.nthConstraint(i); - T.assert(v instanceof sii.sqlite3_index_constraint) - .assert(v.pointer >= pii.$aConstraint); - v.dispose(); - v = pii.nthConstraintUsage(i,true); - T.assert(wasm.isPtr(v)); - v = pii.nthConstraintUsage(i); - T.assert(v instanceof sii.sqlite3_index_constraint_usage) - .assert(v.pointer >= pii.$aConstraintUsage); - v.$argvIndex = i;//just to get some values into xFilter - v.dispose(); - } - } - //log("xBestIndex $nOrderBy =",pii.$nOrderBy); - if(pii.$nOrderBy>0){ - // Validate nthOrderBy() - const max = pii.$nOrderBy; - for(let i=0; i < max; ++i ){ - let v = pii.nthOrderBy(i,true); - T.assert(wasm.isPtr(v)); - v = pii.nthOrderBy(i); - T.assert(v instanceof sii.sqlite3_index_orderby) - .assert(v.pointer >= pii.$aOrderBy); - v.dispose(); - } - } - pii.dispose(); - return 0; - }catch(e){ - return vth.xMethodError('xBestIndex',e); - } - } - }; - /** - The vtab API places relevance on whether xCreate and - xConnect are exactly the same function (same pointer - address). Two JS-side references to the same method will - end up, without acrobatics to counter it, being compiled as - two different WASM-side bindings, i.e. two different - pointers. - - In order to account for this, VtabHelper.installMethods() - checks for duplicate function entries and maps them to the - same WASM-compiled instance. - */ - if(1){ - tmplMethods.xCreate = tmplMethods.xConnect; - } - - const tmplMod = new sqlite3.capi.sqlite3_module(); - tmplMod.ondispose = []; - tmplMod.$iVersion = 0; - vth.installMethods(tmplMod, tmplMethods, true); - if(tmplMethods.xCreate){ - T.assert(tmplMod.$xCreate) - .assert(tmplMod.$xCreate === tmplMod.$xConnect, - "installMethods() must avoid re-compiling identical functions"); - tmplMod.$xCreate = 0; - } - let rc = capi.sqlite3_create_module( - this.db, "testvtab", tmplMod, 0 - ); - this.db.checkRc(rc); - - const list = this.db.selectArrays( - "SELECT a,b FROM testvtab where a<9999 and b>1 order by a, b" - /* Query is shaped so that it will ensure that some constraints - end up in xBestIndex(). */ - ); - T.assert(10===list.length) - .assert(1000===list[0][0]) - .assert(2009===list[list.length-1][1]) - } - })/*vtab sanity checks*/ - //////////////////////////////////////////////////////////////////// .t({ name: 'C-side WASM tests', @@ -1838,6 +1646,379 @@ self.sqlite3InitModule = sqlite3InitModule; } }/* jaccwabyt-specific tests */) + //////////////////////////////////////////////////////////////////////// + .t({ + name: 'virtual table #1', + predicate: ()=>!!capi.sqlite3_index_info, + test: function(sqlite3){ + warn("The vtab/module JS bindings are experimental and subject to change."); + const vth = sqlite3.VtabHelper; + const tmplCols = Object.assign(Object.create(null),{ + A: 0, B: 1 + }); + /** + The vtab demonstrated here is a JS-ification of + ext/misc/templatevtab.c. + */ + const tmplMethods = { + xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ + try{ + const args = wasm.cArgvToJs(argc, argv); + T.assert(args.length>=3) + .assert(args[0] === 'testvtab') + .assert(args[1] === 'main') + .assert(args[2] === 'testvtab'); + console.debug("xConnect() args =",args); + const rc = capi.sqlite3_declare_vtab( + pDb, "CREATE TABLE ignored(a,b)" + ); + if(0===rc){ + const t = vth.xVtab(); + wasm.setPtrValue(ppVtab, t.pointer); + T.assert(t === vth.xVtab(wasm.getPtrValue(ppVtab))); + } + return rc; + }catch(e){ + if(!(e instanceof sqlite3.WasmAllocError)){ + wasm.setPtrValue(pzErr, wasm.allocCString(e.message)); + } + return vth.xError('xConnect',e); + } + }, + xDisconnect: function(pVtab){ + try { + const t = vth.xVtab(pVtab, true); + t.dispose(); + return 0; + }catch(e){ + return vth.xError('xDisconnect',e); + } + }, + xOpen: function(pVtab, ppCursor){ + try{ + const t = vth.xVtab(pVtab), c = vth.xCursor(); + T.assert(t instanceof capi.sqlite3_vtab) + .assert(c instanceof capi.sqlite3_vtab_cursor); + wasm.setPtrValue(ppCursor, c.pointer); + c._rowId = 0; + return 0; + }catch(e){ + return vth.xError('xOpen',e); + } + }, + xClose: function(pCursor){ + try{ + const c = vth.xCursor(pCursor,true); + T.assert(c instanceof capi.sqlite3_vtab_cursor) + .assert(!vth.xCursor(pCursor)); + c.dispose(); + return 0; + }catch(e){ + return vth.xError('xClose',e); + } + }, + xNext: function(pCursor){ + try{ + const c = vth.xCursor(pCursor); + ++c._rowId; + return 0; + }catch(e){ + return vth.xError('xNext',e); + } + + }, + xColumn: function(pCursor, pCtx, iCol){ + try{ + const c = vth.xCursor(pCursor); + switch(iCol){ + case tmplCols.A: + capi.sqlite3_result_int(pCtx, 1000 + c._rowId); + break; + case tmplCols.B: + capi.sqlite3_result_int(pCtx, 2000 + c._rowId); + break; + default: sqlite3.SQLite3Error.toss("Invalid column id",iCol); + } + return 0; + }catch(e){ + return vth.xError('xColumn',e); + } + }, + xRowid: function(pCursor, ppRowid64){ + try{ + const c = vth.xCursor(pCursor); + vth.xRowid(ppRowid64, c._rowId); + return 0; + }catch(e){ + return vth.xError('xRowid',e); + } + }, + xEof: function(pCursor){ + const c = vth.xCursor(pCursor), + rc = c._rowId>=10; + c.dispose(); + return rc; + }, + xFilter: function(pCursor, idxNum, idxCStr, + argc, argv/* [sqlite3_value* ...] */){ + try{ + const c = vth.xCursor(pCursor); + c._rowId = 0; + const list = vth.sqlite3ValuesToJs(argc, argv); + T.assert(argc === list.length); + //log(argc,"xFilter value(s):",list); + c.dispose(); + return 0; + }catch(e){ + return vth.xError('xFilter',e); + } + }, + xBestIndex: function(pVtab, pIdxInfo){ + try{ + //const t = vth.xVtab(pVtab); + const sii = capi.sqlite3_index_info; + const pii = new sii(pIdxInfo); + pii.$estimatedRows = 10; + pii.$estimatedCost = 10.0; + //log("xBestIndex $nConstraint =",pii.$nConstraint); + if(pii.$nConstraint>0){ + // Validate nthConstraint() and nthConstraintUsage() + const max = pii.$nConstraint; + for(let i=0; i < max; ++i ){ + let v = pii.nthConstraint(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthConstraint(i); + T.assert(v instanceof sii.sqlite3_index_constraint) + .assert(v.pointer >= pii.$aConstraint); + v.dispose(); + v = pii.nthConstraintUsage(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthConstraintUsage(i); + T.assert(v instanceof sii.sqlite3_index_constraint_usage) + .assert(v.pointer >= pii.$aConstraintUsage); + v.$argvIndex = i;//just to get some values into xFilter + v.dispose(); + } + } + //log("xBestIndex $nOrderBy =",pii.$nOrderBy); + if(pii.$nOrderBy>0){ + // Validate nthOrderBy() + const max = pii.$nOrderBy; + for(let i=0; i < max; ++i ){ + let v = pii.nthOrderBy(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthOrderBy(i); + T.assert(v instanceof sii.sqlite3_index_orderby) + .assert(v.pointer >= pii.$aOrderBy); + v.dispose(); + } + } + pii.dispose(); + return 0; + }catch(e){ + return vth.xError('xBestIndex',e); + } + } + }; + /** + The vtab API places relevance on whether xCreate and + xConnect are exactly the same function (same pointer + address). Two JS-side references to the same method will + end up, without acrobatics to counter it, being compiled as + two different WASM-side bindings, i.e. two different + pointers. + + In order to account for this, VtabHelper.installMethods() + checks for duplicate function entries and maps them to the + same WASM-compiled instance. + */ + if(1){ + tmplMethods.xCreate = tmplMethods.xConnect; + } + + const tmplMod = new sqlite3.capi.sqlite3_module(); + tmplMod.$iVersion = 0; + this.db.onclose.disposeThese.push(tmplMod); + vth.installMethods(tmplMod, tmplMethods, true); + if(tmplMethods.xCreate){ + T.assert(tmplMod.$xCreate) + .assert(tmplMod.$xCreate === tmplMod.$xConnect, + "installMethods() must avoid re-compiling identical functions"); + tmplMod.$xCreate = 0; + } + let rc = capi.sqlite3_create_module( + this.db, "testvtab", tmplMod, 0 + ); + this.db.checkRc(rc); + + const list = this.db.selectArrays( + "SELECT a,b FROM testvtab where a<9999 and b>1 order by a, b" + /* Query is shaped so that it will ensure that some constraints + end up in xBestIndex(). */ + ); + T.assert(10===list.length) + .assert(1000===list[0][0]) + .assert(2009===list[list.length-1][1]) + } + })/*custom vtab #1*/ + + //////////////////////////////////////////////////////////////////////// + .t({ + name: 'virtual table #2 (w/ automated exception wrapping)', + predicate: ()=>!!capi.sqlite3_index_info, + test: function(sqlite3){ + warn("The vtab/module JS bindings are experimental and subject to change."); + const vth = sqlite3.VtabHelper; + const tmplCols = Object.assign(Object.create(null),{ + A: 0, B: 1 + }); + /** + The vtab demonstrated here is a JS-ification of + ext/misc/templatevtab.c. + */ + let throwOnConnect = 1 ? 0 : capi.SQLITE_CANTOPEN + /* ^^^ just for testing exception wrapping. Note that sqlite + always translates errors from a vtable to a generic + SQLITE_ERROR unless it's from xConnect()/xCreate() and that + callback sets an error string. */; + const modConfig = { + /* catchExceptions changes how the methods are wrapped */ + catchExceptions: false, + name: "vtab2test", + methods:{ + xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ + if(throwOnConnect){ + sqlite3.SQLite3Error.toss( + throwOnConnect, + "Throwing a test exception." + ); + } + const args = wasm.cArgvToJs(argc, argv); + console.debug("xCreate/xConnect args:",args); + T.assert(args.length>=3); + const rc = capi.sqlite3_declare_vtab( + pDb, "CREATE TABLE ignored(a,b)" + ); + if(0===rc){ + const t = vth.xVtab(); + wasm.setPtrValue(ppVtab, t.pointer); + T.assert(t === vth.xVtab(wasm.getPtrValue(ppVtab))); + } + return rc; + }, + xDisconnect: function(pVtab){ + const t = vth.xVtab(pVtab, true); + t.dispose(); + }, + xOpen: function(pVtab, ppCursor){ + const t = vth.xVtab(pVtab), c = vth.xCursor(); + T.assert(t instanceof capi.sqlite3_vtab) + .assert(c instanceof capi.sqlite3_vtab_cursor); + wasm.setPtrValue(ppCursor, c.pointer); + c._rowId = 0; + }, + xClose: function(pCursor){ + const c = vth.xCursor(pCursor,true); + T.assert(c instanceof capi.sqlite3_vtab_cursor) + .assert(!vth.xCursor(pCursor)); + c.dispose(); + }, + xNext: function(pCursor){ + const c = vth.xCursor(pCursor); + ++c._rowId; + }, + xColumn: function(pCursor, pCtx, iCol){ + const c = vth.xCursor(pCursor); + switch(iCol){ + case tmplCols.A: + capi.sqlite3_result_int(pCtx, 1000 + c._rowId); + break; + case tmplCols.B: + capi.sqlite3_result_int(pCtx, 2000 + c._rowId); + break; + default: sqlite3.SQLite3Error.toss("Invalid column id",iCol); + } + }, + xRowid: function(pCursor, ppRowid64){ + const c = vth.xCursor(pCursor); + vth.xRowid(ppRowid64, c._rowId); + c.dispose(); + }, + xEof: function(pCursor){ + const c = vth.xCursor(pCursor), + rc = c._rowId>=10; + c.dispose(); + return rc; + }, + xFilter: function(pCursor, idxNum, idxCStr, + argc, argv/* [sqlite3_value* ...] */){ + const c = vth.xCursor(pCursor); + c._rowId = 0; + const list = vth.sqlite3ValuesToJs(argc, argv); + T.assert(argc === list.length); + c.dispose(); + }, + xBestIndex: function(pVtab, pIdxInfo){ + //const t = vth.xVtab(pVtab); + const pii = vth.xIndexInfo(pIdxInfo); + pii.$estimatedRows = 10; + pii.$estimatedCost = 10.0; + pii.dispose(); + } + }/*methods*/ + }; + const doEponymous = + /* Bug (somewhere): non-eponymous is behaving as is + the call to sqlite3_create_module() is missing + or failed: + + SQL TRACE #63 create virtual table testvtab2 using vtab2test(arg1, arg2) + + => sqlite3 result code 1: no such module: vtab2test + */ true; + if(doEponymous){ + warn("Reminder: non-eponymous mode is still not working here.", + "Details are in the code comments."); + modConfig.methods.xCreate = 0; + }else{ + modConfig.methods.xCreate = (...args)=>0; + } + const tmplMod = vth.setupModule(modConfig); + T.assert(tmplMod instanceof capi.sqlite3_module) + .assert(1===tmplMod.$iVersion); + if(doEponymous){ + if(modConfig.methods.xCreate !== 0){ + T.assert(modConfig.methods.xCreate === modConfig.methods.xConnect) + .assert(tmplMod.$xCreate === tmplMod.$xConnect); + }else{ + T.assert(0 === tmplMod.$xCreate); + } + } + this.db.onclose.disposeThese.push(tmplMod); + this.db.checkRc(capi.sqlite3_create_module( + this.db, modConfig.name, tmplMod, 0 + )); + if(!doEponymous){ + this.db.exec([ + "create virtual table testvtab2 using ", + modConfig.name, + "(arg1, arg2)" + ]); + } + const list = this.db.selectArrays( + ["SELECT a,b FROM ", + (doEponymous ? modConfig.name : "testvtab2"), + " where a<9999 and b>1 order by a, b" + ]/* Query is shaped so that it will ensure that some + constraints end up in xBestIndex(). */ + ); + T.assert(10===list.length) + .assert(1000===list[0][0]) + .assert(2009===list[list.length-1][1]) + } + })/*custom vtab #2*/ + + //////////////////////////////////////////////////////////////////////// .t('Close db', function(){ T.assert(this.db).assert(wasm.isPtr(this.db.pointer)); wasm.sqlite3_wasm_db_reset(this.db); diff --git a/manifest b/manifest index e691430191..8786729db6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\saddOnDispose()\smethod\sto\sJaccwabyt\sand\scode-adjacent\sminor\sinternal\scleanups. -D 2022-12-07T03:42:39.134 +C Work\son\san\salternate\s(slightly\ssimpler)\sapproach\sto\sbinding\sJS\svtabs.\sNon-eponymous\svtabs\sare\snot\sworking,\sfor\sreasons\sas\syet\sunknown. +D 2022-12-07T07:22:34.835 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,11 +505,11 @@ F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a9578430388 F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 F ext/wasm/api/sqlite3-api-glue.js 8fa55af37c9880f94a803f32591dc0304750cc2f750048daf41fe942757bee64 F ext/wasm/api/sqlite3-api-oo1.js 416e6398721a4cbb80ddfa3d7b303216790f1d344efdbbc36239d39abc66aa27 -F ext/wasm/api/sqlite3-api-prologue.js a7596c30392d9ca8f5b7c14feb4e6788107d1159fd5f90eb26708d653d36c9bc +F ext/wasm/api/sqlite3-api-prologue.js 1380e933325c11786b2afc93fc8ff88c2fd1ffeac3e0081da35e5a7317f20e09 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 -F ext/wasm/api/sqlite3-v-helper.js 2125255f102ab07a2653d74d4931d716b0d27c5a89bebc188ad828de0dd1dcef +F ext/wasm/api/sqlite3-v-helper.js c0c56d4fb1272140629ac858297a825e0a8e04005df92c70ef4a4aa75d4d4645 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 8ec510fee735c646fb18a3b99f0ca5ca461f9e066c43cdc404d7144f12ae6ed6 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 723522a6c2a2463884a83fa1cc7ae5770deaaf0856a1058cc1023b2bfa1c898b @@ -539,7 +539,7 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html c806b6005145b71d64240606e9c6e0bf56878ee8829c66fe7486cebf34b0e6b1 F ext/wasm/index.html f151b7c7b5cfdc066567d556acd168e769efd4e982286dc5f849a5ee69ecd0ff -F ext/wasm/jaccwabyt/jaccwabyt.js b7efd07ea3927e9d0bc75b3819c6d40bdfb0a03cbc8d93331be16799f35261c3 +F ext/wasm/jaccwabyt/jaccwabyt.js 06f2ef1ad640c26c593def3d960336e9bb789819b920516480895c38ed5f58fa F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 980680c8acfa3c8ae6a5aa223512d1b8e78040ced20f8ba2c382129bc73ec028 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 @@ -555,7 +555,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac F ext/wasm/tester1-worker.html d43f3c131d88f10d00aff3e328fed13c858d674ea2ff1ff90225506137f85aa9 F ext/wasm/tester1.c-pp.html d34bef3d48e5cbc1c7c06882ad240fec49bf88f5f65696cc2c72c416933aa406 -F ext/wasm/tester1.c-pp.js 8144e0e0f76b14fcd5223fad190bd94a50caf19b2419848b782448cda3ccbc78 +F ext/wasm/tester1.c-pp.js 661c9461fa104f231ff9e767c89ba1fc4a4af6a61db076772ca634d562afd35d F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d77c348039926c24e0fb50a7dc7e4b62895cd201c021f8e29832917e4b8b09f -R c60d510be7a6225cada64eb18dd4b778 +P 6a2723fe3f28dd94328d901e64e1e9ee9a1b2e9eeaed6c54038a5b83c914db78 +R 8623f61e63ac3c807b096f8d7028ccc0 U stephan -Z 9a43ff51fca7fb341935fe3d300c9a6e +Z 80cda1da5b829b2851b20f834a26edf8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ae8e40499b..c832b9405d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a2723fe3f28dd94328d901e64e1e9ee9a1b2e9eeaed6c54038a5b83c914db78 \ No newline at end of file +6a0fefb93bcccd950df211cf5c2f49660c7b92115dd01b2b508a4ab9e3ab3d23 \ No newline at end of file From 7c66faa634be0e89553e0a985270788751f82238 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 7 Dec 2022 16:58:04 +0000 Subject: [PATCH 187/282] Streamline and improve testing of the locking in the memdb VFS. Follow-on to [15f0be8a640e7bfa]. FossilOrigin-Name: d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/memdb.c | 28 ++++++++++++++++++++-------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 8e9257d9fa..d4b5f8cb25 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s(harmless)\soff-by-one\serror\sin\scode\sgeneration\sthat\scomes\sup\swhen\ndoing\sa\sDISTINCT\squery\sagainst\sa\svirtual\stable\swith\san\sOR\sterm\sin\sthe\nWHERE\sclause\sand\swhere\sthe\sORDER\sBY\sclause\shas\s64\sor\smore\sreferences\sto\nthe\sresult\sset.\s\s[forum:/forumpost/dfe8084751|Forum\spost\sdfe8084751]. -D 2022-12-07T00:14:25.828 +C Streamline\sand\simprove\stesting\sof\sthe\slocking\sin\sthe\smemdb\sVFS.\nFollow-on\sto\s[15f0be8a640e7bfa]. +D 2022-12-07T16:58:04.002 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -615,7 +615,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c 5a3dbd8ac8a6501152a4fc1fcae9b0900c2d7eb0589c4ec7456fdde15725a26c -F src/memdb.c fa280078fb48c4bb7ef47e361cd958938a1a3c46a9a45f6622da6efcb57cc055 +F src/memdb.c 3c1f3a3daa670294bad0056e66c17f5fe75cfa1da1850056a5a94347ec32e6bd F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bbde0f36d03cdbbc749427fe7d2dafd5c5031c9e655ebd772857b521f53eb18f -R 250ed9b5c78a34895ae8f1ae12a9ecc7 +P 04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 +R 12ec76fa12aeff3e27ce79fb7684d7d6 U drh -Z d0ffb89df284ba3dde1a0bb1536bd8ea +Z af8ae60fb39ff0cbbaef800ceb1294d9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d731886fa9..96dc67715e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 \ No newline at end of file +d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 \ No newline at end of file diff --git a/src/memdb.c b/src/memdb.c index 7485f51ffb..4ea3cffde8 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -370,21 +370,33 @@ static int memdbLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; if( eLock==pThis->eLock ) return SQLITE_OK; memdbEnter(p); + assert( p->nWrLock==0 || p->nWrLock==1 ); /* No more than 1 write lock */ if( eLock>SQLITE_LOCK_SHARED ){ assert( pThis->eLock>=SQLITE_LOCK_SHARED ); if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){ rc = SQLITE_READONLY; }else if( eLock==SQLITE_LOCK_EXCLUSIVE ){ - /* Taking an EXCLUSIVE lock. Fail if we only have SHARED and any - ** other client has any kind of write-lock. Also fail if any other - ** client is holding read-lock. */ - if( pThis->eLock<=SQLITE_LOCK_SHARED && p->nWrLock ){ - rc = SQLITE_BUSY; - }else if( p->nRdLock>1 ){ + /* We never go for an EXCLUSIVE lock unless we already hold SHARED or + ** higher */ + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); + testcase( pThis->eLock==SQLITE_LOCK_SHARED ); + + /* Because we are holding SHARED or more, there must be at least + ** one read lock */ + assert( p->nRdLock>0 ); + + /* The only way that there can be an existing write lock is if we + ** currently hold it. Otherwise, we would have never been able to + ** promote from NONE to SHARED. */ + assert( p->nWrLock==0 || pThis->eLock>SQLITE_LOCK_SHARED ); + + if( p->nRdLock>1 ){ + /* Cannot take EXCLUSIVE if somebody else is holding SHARED */ rc = SQLITE_BUSY; + }else{ + p->nWrLock = 1; } - p->nWrLock = 1; - }else if( pThis->eLock<=SQLITE_LOCK_SHARED ){ + }else if( ALWAYS(pThis->eLock<=SQLITE_LOCK_SHARED) ){ /* Upgrading to RESERVED or PENDING from SHARED. Fail if any other ** client has a write-lock of any kind. */ if( p->nWrLock ){ From f2cc3387f78aac0a0cd2c0bca251fa581a9fd7f7 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Dec 2022 17:29:17 +0000 Subject: [PATCH 188/282] Have sqlite3_stmt_scanstatus() report cycle, loop and row counts separately for creating an automatic index and using that automatic index. FossilOrigin-Name: 3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/where.c | 51 ++++++++++++++++++++++++++++++++++++++ test/scanstatus2.test | 57 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d4b5f8cb25..2d0b3556cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Streamline\sand\simprove\stesting\sof\sthe\slocking\sin\sthe\smemdb\sVFS.\nFollow-on\sto\s[15f0be8a640e7bfa]. -D 2022-12-07T16:58:04.002 +C Have\ssqlite3_stmt_scanstatus()\sreport\scycle,\sloop\sand\srow\scounts\sseparately\sfor\screating\san\sautomatic\sindex\sand\susing\sthat\sautomatic\sindex. +D 2022-12-07T17:29:17.346 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c adc6783cd3b136271a70001182e899d3523e199bda41ead5c3e2ab4a84456af1 +F src/where.c 3ee4050e453dfd03da4a8e4b5561d20b20b2302ec9d02437fb62c6f29bb78401 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -1458,7 +1458,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 7dbcfd6adc6a8df6abc59f83d6da5a27e1bce0b2f6fa55147c8176d7c44e0450 -F test/scanstatus2.test 3f45ec1d8b45dcb0df7f98daf3364fc2e03bf89a8c3dd2428251a5e0a34de7af +F test/scanstatus2.test 07e2d1b86a07cd8f6cbef9546c9d6b27af0de5e613b6f38f98666f9c729bd4c7 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 -R 12ec76fa12aeff3e27ce79fb7684d7d6 -U drh -Z af8ae60fb39ff0cbbaef800ceb1294d9 +P d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 +R aa03a3260eaa51e2eda4cccd90080651 +U dan +Z 28dcdc9b3132851feb893087f0e4ae04 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 96dc67715e..a5f6f520a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 \ No newline at end of file +3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1cf9592070..25ad18b282 100644 --- a/src/where.c +++ b/src/where.c @@ -812,6 +812,51 @@ static int termCanDriveIndex( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Argument pIdx represents an automatic index that the current statement +** will create and populate. Add an OP_Explain with text of the form: +** +** CREATE AUTOMATIC INDEX ON () [WHERE ] +** +** This is only required if sqlite3_stmt_scanstatus() is enabled, to +** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP +** values with. In order to avoid breaking legacy code and test cases, +** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command. +*/ +static void explainAutomaticIndex( + Parse *pParse, + Index *pIdx, /* Automatic index to explain */ + int bPartial, /* True if pIdx is a partial index */ + int *pAddrExplain /* OUT: Address of OP_Explain */ +){ + if( pParse->explain!=2 ){ + Table *pTab = pIdx->pTable; + const char *zSep = ""; + char *zText = 0; + int ii = 0; + zText = sqlite3_mprintf("CREATE AUTOMATIC INDEX ON %s(", pTab->zName); + assert( pIdx->nColumn>1 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID ); + for(ii=0; ii<(pIdx->nColumn-1); ii++){ + const char *zName = 0; + int iCol = pIdx->aiColumn[ii]; + + zName = pTab->aCol[iCol].zCnName; + zText = sqlite3_mprintf("%z%s%s", zText, zSep, zName); + zSep = ", "; + } + *pAddrExplain = sqlite3VdbeExplain( + pParse, 0, "%s)%s", zText, (bPartial ? " WHERE " : "") + ); + sqlite3_free(zText); + } +} +#else +# define explainAutomaticIndex(a,b,c,d) +#endif + /* ** Generate code to construct the Index object for an automatic index ** and to set up the WhereLevel object pLevel so that the code generator @@ -847,6 +892,9 @@ static SQLITE_NOINLINE void constructAutomaticIndex( SrcItem *pTabItem; /* FROM clause term being indexed */ int addrCounter = 0; /* Address where integer counter is initialized */ int regBase; /* Array of registers where record is assembled */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp = 0; /* Address of OP_Explain */ +#endif /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -970,6 +1018,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( pIdx->azColl[n] = sqlite3StrBINARY; /* Create the automatic index */ + explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp); assert( pLevel->iIdxCur>=0 ); pLevel->iIdxCur = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); @@ -1005,6 +1054,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, regBase, pLoop->u.btree.nEq); } + sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); @@ -1025,6 +1075,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1); end_auto_index_create: sqlite3ExprDelete(pParse->db, pPartial); diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 9a4758b6d4..1b5c18cb8b 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -170,9 +170,66 @@ do_graph_test 2.1 { } { QUERY (nCycle=nnn) --SCAN x1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON x2(c, d) (nCycle=nnn) --SEARCH x2 USING AUTOMATIC COVERING INDEX (c=?) (nCycle=nnn) } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE rt1 (id INTEGER PRIMARY KEY, x1, x2); + CREATE TABLE rt2 (id, x1, x2); +} + +do_graph_test 4.1 { + SELECT * FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=rt1.x1; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id, x2) (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.2 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=rt1.x1; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id) (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.3 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND (rt2.x1+1)=(rt1.x1+1); +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--SCAN rt2 (nCycle=nnn) +} + +do_graph_test 4.4 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=(rt1.x1+1) AND rt2.id>5; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id) WHERE (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC PARTIAL COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.5 { + SELECT v1.cnt FROM rt1, ( + SELECT count(*) AS cnt, rt2.x1 AS x1 FROM rt2 GROUP BY x1 + ) AS v1 WHERE rt1.x1=v1.x1 +} { +QUERY (nCycle=nnn) +--MATERIALIZE v1 (nCycle=nnn) +----SCAN rt2 (nCycle=nnn) +----USE TEMP B-TREE FOR GROUP BY +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON v1(x1, cnt) (nCycle=nnn) +--SEARCH v1 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + finish_test From 7474de702d76cfa49e8dca5660806068fb73437d Mon Sep 17 00:00:00 2001 From: larrybr Date: Wed, 7 Dec 2022 19:29:13 +0000 Subject: [PATCH 189/282] Omit CLI use of pragma_table_xinfo when it is not defined in the build. FossilOrigin-Name: 976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2d0b3556cf..601f881b94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\ssqlite3_stmt_scanstatus()\sreport\scycle,\sloop\sand\srow\scounts\sseparately\sfor\screating\san\sautomatic\sindex\sand\susing\sthat\sautomatic\sindex. -D 2022-12-07T17:29:17.346 +C Omit\sCLI\suse\sof\spragma_table_xinfo\swhen\sit\sis\snot\sdefined\sin\sthe\sbuild. +D 2022-12-07T19:29:13.407 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 -F src/shell.c.in b5bc6e14605b8d1e8de729454d98d3353ad7e529a8d2b6e34080d068e99bff99 +F src/shell.c.in fbd48504e6d8953950deddb3ef83de58758a84e58e7994b77792b4568981ea98 F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 -R aa03a3260eaa51e2eda4cccd90080651 -U dan -Z 28dcdc9b3132851feb893087f0e4ae04 +P 3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba +R eb3572b995ebd9a522f40116cadefed1 +U larrybr +Z dfd7917e394baecbbad398b94b2b49b6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a5f6f520a5..fffb99e602 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba \ No newline at end of file +976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 9c1fe111f5..44105bf1b3 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10328,6 +10328,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ shell_exec(p, zSql, 0); } +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && !defined(SQLITE_OMIT_VIRTUALTABLE) { int lrc; char *zRevText = /* Query for reversible to-blob-to-text check */ @@ -10378,6 +10379,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zRevText); } +#endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */ sqlite3_free(zSql); }else From 209dbab9979628ea1587a07e11c0bcfe78971dd8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 7 Dec 2022 19:51:48 +0000 Subject: [PATCH 190/282] Fix harmless compiler warning in the dynamic continuation prompt of the CLI. FossilOrigin-Name: 68947b0a1147365a29e335d5e4dc55fe5d9afe3562a0709b111067d6017c5a42 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 601f881b94..e1ead94721 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sCLI\suse\sof\spragma_table_xinfo\swhen\sit\sis\snot\sdefined\sin\sthe\sbuild. -D 2022-12-07T19:29:13.407 +C Fix\sharmless\scompiler\swarning\sin\sthe\sdynamic\scontinuation\sprompt\sof\sthe\sCLI. +D 2022-12-07T19:51:48.626 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 -F src/shell.c.in fbd48504e6d8953950deddb3ef83de58758a84e58e7994b77792b4568981ea98 +F src/shell.c.in bcf8552c82f2c84650e39a6d638373569c2035942c0497b83eef197169e0305a F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba -R eb3572b995ebd9a522f40116cadefed1 -U larrybr -Z dfd7917e394baecbbad398b94b2b49b6 +P 976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab +R b08d7359f3a0adfc8758cdabe6250a50 +U drh +Z 38d73aa067c1a85ec056e9ed86a84d61 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fffb99e602..070399b39f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab \ No newline at end of file +68947b0a1147365a29e335d5e4dc55fe5d9afe3562a0709b111067d6017c5a42 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 44105bf1b3..e4b054f450 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -537,7 +537,7 @@ static char *dynamicContinuePrompt(void){ if( dynPrompt.zScannerAwaits ){ int ncp = strlen(continuePrompt), ndp = strlen(dynPrompt.zScannerAwaits); if( ndp > ncp-3 ) return continuePrompt; - strncpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits, ndp); + strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); From 7f4b066eb2b5ab99ed126809e56e8a9bd41a91e3 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Dec 2022 20:09:54 +0000 Subject: [PATCH 191/282] Reduce the overhead of SQLITE_ENABLE_STMT_SCANSTATUS some. FossilOrigin-Name: 212927e97e7be7d237de08359dce0dfb9211ac406b32009a6e15afd79c006475 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/vdbe.c | 42 ++++++++++++------------------------------ src/vdbe.h | 4 ++++ src/vdbeInt.h | 6 ------ src/vdbeapi.c | 22 +++++++++++++--------- src/vdbeaux.c | 30 ++++++++++-------------------- 7 files changed, 51 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index e1ead94721..bf71b14d97 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\sthe\sdynamic\scontinuation\sprompt\sof\sthe\sCLI. -D 2022-12-07T19:51:48.626 +C Reduce\sthe\soverhead\sof\sSQLITE_ENABLE_STMT_SCANSTATUS\ssome. +D 2022-12-07T20:09:54.641 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -717,11 +717,11 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 313f3154e2b85a447326f5dd15de8d31a4df6ab0c3579bd58f426ff634ec9050 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 8fdef91d3c9bfa363e8eb381d362e2f0cadec456975ec5595c0bfdf65da84e53 -F src/vdbe.h 6d921884cf8ec6a53efba99f8b68e32e955367631743e29039840e781aaf547c -F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 -F src/vdbeapi.c 5e265ab5165b0dfeac55e6a2ad3929d64584c4e7ca106b438137d7596da91348 -F src/vdbeaux.c 0b81f317c86ed9f4ba822af66a52777fed6e8180edc79d4fc62ffe75c8e52d80 +F src/vdbe.c df9ff3fcaa3c60fa9b0bca340c3f3cfc63a51361901ca06fe18bdeadeb2a1a24 +F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c +F src/vdbeInt.h 8651e4c4e04d1860d0bdcf330cb8294e3778a9d4222be30ce4c490d9220af783 +F src/vdbeapi.c df3f73a4d0a487f2068e3c84776cd6e3fba5ae80ff612659dcfda4307686420b +F src/vdbeaux.c 25691b395bf57ae0a754570fc3ba2d6f8bd2f0a8944f1d2589b9e68114e01c2b F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab -R b08d7359f3a0adfc8758cdabe6250a50 -U drh -Z 38d73aa067c1a85ec056e9ed86a84d61 +P 68947b0a1147365a29e335d5e4dc55fe5d9afe3562a0709b111067d6017c5a42 +R 4a013de93da29e8129599899cb5c1b54 +U dan +Z 4aecefd86ed6f42fbe2469d178e09d33 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 070399b39f..01cc2da2af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68947b0a1147365a29e335d5e4dc55fe5d9afe3562a0709b111067d6017c5a42 \ No newline at end of file +212927e97e7be7d237de08359dce0dfb9211ac406b32009a6e15afd79c006475 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index cd27aff44f..12b22992b9 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -790,15 +790,12 @@ int sqlite3VdbeExec( assert( pOp>=aOp && pOp<&aOp[p->nOp]); nVmStep++; #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - if( p->anExec ){ - assert( p->anExec && p->anCycle ); - p->anExec[(int)(pOp-aOp)]++; - pnCycle = &p->anCycle[pOp-aOp]; + pOp->nExec++; + pnCycle = &pOp->nCycle; # ifdef VDBE_PROFILE - if( sqlite3NProfileCnt==0 ) + if( sqlite3NProfileCnt==0 ) # endif - *pnCycle -= sqlite3Hwtime(); - } + *pnCycle -= sqlite3Hwtime(); #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -7152,10 +7149,6 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - pFrame->anExec = p->anExec; - pFrame->anCycle = p->anCycle; -#endif #ifdef SQLITE_DEBUG pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; #endif @@ -7192,10 +7185,6 @@ case OP_Program: { /* jump */ memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8); p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - p->anExec = 0; - p->anCycle = 0; -#endif #ifdef SQLITE_DEBUG /* Verify that second and subsequent executions of the same trigger do not ** try to reuse register values from the first use. */ @@ -8732,17 +8721,11 @@ default: { /* This is really OP_Noop, OP_Explain */ } #if defined(VDBE_PROFILE) - assert( pnCycle ); - if( pnCycle ){ - *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); - assert( *pnCycle < (((u64)1) << 48) ); - pnCycle = 0; - } + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( pnCycle ){ - *pnCycle += sqlite3Hwtime(); - pnCycle = 0; - } + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; #endif /* The following code adds nothing to the actual functionality @@ -8819,11 +8802,10 @@ abort_due_to_error: ** top. */ vdbe_return: #if defined(VDBE_PROFILE) - if( pnCycle ){ - *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); - assert( *pnCycle < (((u64)1) << 48) ); - pnCycle = 0; - } + if( pnCycle ){ + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; + } #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pnCycle ){ *pnCycle += sqlite3Hwtime(); diff --git a/src/vdbe.h b/src/vdbe.h index a199247434..3caf1f7872 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -71,6 +71,10 @@ struct VdbeOp { u32 iSrcLine; /* Source-code line that generated this opcode ** with flags in the upper 8 bits */ #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 nExec; + u64 nCycle; +#endif }; typedef struct VdbeOp VdbeOp; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index df47e05448..cd112e9c48 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -171,8 +171,6 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ - i64 *anExec; /* Event counters from parent frame */ - u64 *anCycle; /* Cycle counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ u8 *aOnce; /* Bitmask used by OP_Once */ @@ -492,10 +490,6 @@ struct Vdbe { u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ AuxData *pAuxData; /* Linked list of auxdata allocations */ -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - i64 *anExec; /* Number of times each op has been executed */ - u64 *anCycle; /* Sum of sqlite3HwTime() spent in each opcode */ -#endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int nScan; /* Entries in aScan[] */ ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 028eb7fe64..414745e8ba 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2123,14 +2123,12 @@ int sqlite3_stmt_scanstatus_v2( ScanStatus *pScan; int idx; - /* If the v2 flag is clear, then this function must ignore any ScanStatus - ** structures with ScanStatus.addrLoop set to 0. */ if( iScan<0 ){ int ii; if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ i64 res = 0; for(ii=0; iinOp; ii++){ - res += p->anCycle[ii]; + res += p->aOp[ii].nCycle; } *(i64*)pOut = res; return 0; @@ -2141,6 +2139,8 @@ int sqlite3_stmt_scanstatus_v2( idx = iScan; pScan = &p->aScan[idx]; }else{ + /* If the COMPLEX flag is clear, then this function must ignore any + ** ScanStatus structures with ScanStatus.addrLoop set to 0. */ for(idx=0; idxnScan; idx++){ pScan = &p->aScan[idx]; if( pScan->zName ){ @@ -2154,7 +2154,7 @@ int sqlite3_stmt_scanstatus_v2( switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { if( pScan->addrLoop>0 ){ - *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + *(sqlite3_int64*)pOut = p->aOp[pScan->addrLoop].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -2162,7 +2162,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_NVISIT: { if( pScan->addrVisit>0 ){ - *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + *(sqlite3_int64*)pOut = p->aOp[pScan->addrVisit].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -2218,7 +2218,7 @@ int sqlite3_stmt_scanstatus_v2( if( iIns==0 ) break; if( iIns>0 ){ while( iIns<=iEnd ){ - res += p->anCycle[iIns]; + res += p->aOp[iIns].nCycle; iIns++; } }else{ @@ -2229,7 +2229,7 @@ int sqlite3_stmt_scanstatus_v2( if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ continue; } - res += p->anCycle[iOp]; + res += p->aOp[iOp].nCycle; } } } @@ -2261,7 +2261,11 @@ int sqlite3_stmt_scanstatus( */ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; - memset(p->anExec, 0, p->nOp * sizeof(i64)); - memset(p->anCycle, 0, p->nOp * sizeof(u64)); + int ii; + for(ii=0; iinOp; ii++){ + Op *pOp = &p->aOp[ii]; + pOp->nExec = 0; + pOp->nCycle = 0; + } } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e22b741196..8d284746f1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -262,6 +262,10 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS pOp->zComment = 0; #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; +#endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ sqlite3VdbePrintOp(0, i, &p->aOp[i]); @@ -2508,8 +2512,10 @@ void sqlite3VdbeRewind(Vdbe *p){ p->iStatement = 0; p->nFkConstraint = 0; #ifdef VDBE_PROFILE - memset(p->anExec, 0, sizeof(i64)*p->nOp); - memset(p->anCycle, 0, sizeof(u64)*p->nOp); + for(i=0; inOp; i++){ + p->aOp[i].nExec = 0; + p->aOp[i].nCycle = 0; + } #endif } @@ -2617,10 +2623,6 @@ void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); - p->anCycle = allocSpace(&x, 0, p->nOp*sizeof(u64)); -#endif if( x.nNeeded ){ x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); x.nFree = x.nNeeded; @@ -2629,10 +2631,6 @@ void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); - p->anCycle = allocSpace(&x, p->anCycle, p->nOp*sizeof(u64)); -#endif } } @@ -2647,10 +2645,6 @@ void sqlite3VdbeMakeReady( p->nMem = nMem; initMemArray(p->aMem, nMem, db, MEM_Undefined); memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - memset(p->anExec, 0, p->nOp*sizeof(i64)); - memset(p->anCycle, 0, p->nOp*sizeof(u64)); -#endif } sqlite3VdbeRewind(p); } @@ -2708,10 +2702,6 @@ static void closeCursorsInFrame(Vdbe *p){ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; closeCursorsInFrame(v); -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - v->anExec = pFrame->anExec; - v->anCycle = pFrame->anCycle; -#endif v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; @@ -3543,8 +3533,8 @@ int sqlite3VdbeReset(Vdbe *p){ } for(i=0; inOp; i++){ char zHdr[100]; - i64 cnt = p->anExec[i]; - i64 cycles = p->anCycle[i]; + i64 cnt = p->aOp[i].nExec; + i64 cycles = p->aOp[i].nCycle; sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", cnt, cycles, From ab8b22a03d033adaa61f82e5c080de64c8200549 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 8 Dec 2022 04:19:38 +0000 Subject: [PATCH 192/282] Remove some dead JS code and tweak some docs. FossilOrigin-Name: 0ee495452c014680697aa9035c245024df127a52d1820ab0e02580a015d96ecb --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 27 ++++---------- ext/wasm/api/sqlite3-v-helper.js | 1 + ext/wasm/speedtest1-worker.html | 2 +- ext/wasm/speedtest1.html | 6 ++-- ext/wasm/tester1.c-pp.js | 45 +++++++++++++++--------- manifest | 20 +++++------ manifest.uuid | 2 +- 7 files changed, 50 insertions(+), 53 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 339ec126d0..1456ae08d2 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -13,7 +13,7 @@ A Worker which manages asynchronous OPFS handles on behalf of a synchronous API which controls it via a combination of Worker messages, SharedArrayBuffer, and Atomics. It is the asynchronous - counterpart of the API defined in sqlite3-api-opfs.js. + counterpart of the API defined in sqlite3-vfs-opfs.js. Highly indebted to: @@ -343,16 +343,6 @@ const installAsyncProxy = function(self){ const affirmNotRO = function(opName,fh){ if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); }; - const affirmLocked = function(opName,fh){ - //if(!fh.syncHandle) toss(opName+"(): File does not have a lock: "+fh.filenameAbs); - /** - Currently a no-op, as speedtest1 triggers xRead() without a - lock (that seems like a bug but it's currently uninvestigated). - This means, however, that some OPFS VFS routines may trigger - acquisition of a lock but never let it go until xUnlock() is - called (which it likely won't be if xLock() was not called). - */ - }; /** We track 2 different timers: the "metrics" timer records how much @@ -393,7 +383,6 @@ const installAsyncProxy = function(self){ */ let flagAsyncShutdown = false; - /** Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods methods, as well as helpers like mkdir(). Maintenance reminder: @@ -427,11 +416,11 @@ const installAsyncProxy = function(self){ }, xAccess: async (filename)=>{ mTimeStart('xAccess'); - /* OPFS cannot support the full range of xAccess() queries sqlite3 - calls for. We can essentially just tell if the file is - accessible, but if it is it's automatically writable (unless - it's locked, which we cannot(?) know without trying to open - it). OPFS does not have the notion of read-only. + /* OPFS cannot support the full range of xAccess() queries + sqlite3 calls for. We can essentially just tell if the file + is accessible, but if it is then it's automatically writable + (unless it's locked, which we cannot(?) know without trying + to open it). OPFS does not have the notion of read-only. The return semantics of this function differ from sqlite3's xAccess semantics because we are limited in what we can @@ -519,7 +508,6 @@ const installAsyncProxy = function(self){ let rc = 0; wTimeStart('xFileSize'); try{ - affirmLocked('xFileSize',fh); const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); state.s11n.serialize(Number(sz)); }catch(e){ @@ -615,7 +603,6 @@ const installAsyncProxy = function(self){ let rc = 0, nRead; const fh = __openFiles[fid]; try{ - affirmLocked('xRead',fh); wTimeStart('xRead'); nRead = (await getSyncHandle(fh,'xRead')).read( fh.sabView.subarray(0, n), @@ -659,7 +646,6 @@ const installAsyncProxy = function(self){ const fh = __openFiles[fid]; wTimeStart('xTruncate'); try{ - affirmLocked('xTruncate',fh); affirmNotRO('xTruncate', fh); await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ @@ -696,7 +682,6 @@ const installAsyncProxy = function(self){ const fh = __openFiles[fid]; wTimeStart('xWrite'); try{ - affirmLocked('xWrite',fh); affirmNotRO('xWrite', fh); rc = ( n === (await getSyncHandle(fh,'xWrite')) diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index 73ba8cc49e..88c86a6a0a 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -538,6 +538,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try{return func(...arguments) || 0;} catch(e){ if(!(e instanceof sqlite3.WasmAllocError)){ + wasm.dealloc(wasm.getPtrValue(pzErr)); wasm.setPtrValue(pzErr, wasm.allocCString(e.message)); } return vt.xError(methodName, e); diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index 9fcd29ea72..cd0fdb027c 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -75,7 +75,7 @@
  • The easiest way to try different optimization levels is, from this directory: - 
    $ rm -f speedtest1.js; make -e emcc_opt='-O2' speedtest1.js
    + 
    $ rm -f jswasm/speedtest1.js; make -e emcc_opt='-O2' speedtest1
    Then reload this page. -O2 seems to consistently produce the fastest results.
    • diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html index 5286b9e482..9cc20924e9 100644 --- a/ext/wasm/speedtest1.html +++ b/ext/wasm/speedtest1.html @@ -40,9 +40,9 @@