diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index dc3c2d2555..1b7c5269e8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -911,8 +911,10 @@ ifeq (0,$(wasm-bare-bones)) sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js endif sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js +sqlite3-api.jses += $(dir.api)/opfs-common-shared.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js # Parallel builds can fail if $(sqlite3-license-version.js) is not # created early enough, so make all files in $(sqlite-api.jses) except @@ -1182,13 +1184,13 @@ all: demos ####################################################################### # -# "SOAP" is a static file which is not part of the amalgamation but -# gets copied into the build output folder and into each of the fiddle -# builds. +# "SOAP" is not part of the amalgamation but gets copied into the +# build output folder and into each of the fiddle builds. # sqlite3.ext.js += $(dir.dout)/sqlite3-opfs-async-proxy.js -$(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js - @$(call b.cp,@,$<,$@) +$(eval $(call b.c-pp.target,soap,\ + $(dir.api)/sqlite3-opfs-async-proxy.c-pp.js,\ + $(dir.dout)/sqlite3-opfs-async-proxy.js)) # # Add a dep of $(sqlite3.ext.js) on every individual build's JS file. diff --git a/ext/wasm/api/opfs-common-inline.c-pp.js b/ext/wasm/api/opfs-common-inline.c-pp.js new file mode 100644 index 0000000000..04ae17d555 --- /dev/null +++ b/ext/wasm/api/opfs-common-inline.c-pp.js @@ -0,0 +1,191 @@ +//#if nope +/** + This file is for preprocessor #include into the "opfs" and + "opfs-wl" impls, as well as their async-proxy part. It must be + inlined in those files, as opposed to being a shared copy in the + library, because (A) the async proxy does not load the library and + (B) it references an object which is local to each of those files + but which has a 99% identical structure for each. +*/ +//#endif +//#// vfs.metrics.enable is a refactoring crutch. +//#define vfs.metrics.enable=0 +const initS11n = function(){ + /** + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset + state.sabS11nSize]. Only one dataset is + recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from this thread or the + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. + */ + state.s11n.deserialize = function(clear=false){ +//#if vfs.metrics.enable + ++metrics.s11n.deserialize.count; +//#endif + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); +//#if vfs.metrics.enable + metrics.s11n.deserialize.time += performance.now() - t; +//#endif + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); +//#if vfs.metrics.enable + ++metrics.s11n.serialize.count; +//#endif + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } +//#if vfs.metrics.enable + metrics.s11n.serialize.time += performance.now() - t; +//#endif + }; + +//#if defined opfs-async-proxy + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; +//#endif + + return state.s11n; +//#undef vfs.metrics.enable +}/*initS11n()*/; diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js new file mode 100644 index 0000000000..11acf40bdc --- /dev/null +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -0,0 +1,1301 @@ +//#if not target:node +/* + 2026-03-04 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js. It + creates a private/internal sqlite3.opfs namespace common to the two + and used (only) by them and the test framework. It is not part of + the public API. The library deletes sqlite3.opfs in its final + bootstrapping steps unless it's specifically told to keep them (for + testing purposes only) using an undocumented and unsupported + mechanism. +*/ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + 'use strict'; + if( sqlite3.config.disable?.vfs?.opfs && + sqlite3.config.disable.vfs['opfs-vfs'] ){ + return; + } + const toss = sqlite3.util.toss, + capi = sqlite3.capi, + util = sqlite3.util, + wasm = sqlite3.wasm; + + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + + This is an internal/private namespace intended for use solely by + the OPFS VFSes and test code for them. The library bootstrapping + process removes this object in non-testing contexts. + */ + const opfsUtil = sqlite3.opfs = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + opfsUtil.thisThreadHasOPFS = ()=>{ + return globalThis.FileSystemHandle && + globalThis.FileSystemDirectoryHandle && + globalThis.FileSystemFileHandle && + globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator?.storage?.getDirectory; + }; + + /** + Must be called by the OPFS VFSes immediately after they determine + whether OPFS is available by calling + thisThreadHasOPFS(). Resolves to the OPFS storage root directory + and sets opfsUtil.rootDirectory to that value. + */ + opfsUtil.getRootDir = async function f(){ + return f.promise ??= navigator.storage.getDirectory().then(d=>{ + opfsUtil.rootDirectory = d; + return d; + }).catch(e=>{ + delete f.promise; + throw e; + }); + }; + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg + is true, the result is returned as an array of path elements, + else an absolute path string is returned. + */ + opfsUtil.getResolvedPath = function(filename,splitIt){ + const p = new URL(filename, "file://irrelevant").pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = await opfsUtil.getRootDir(); + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't or if an + exception is intercepted while trying to make the + determination. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + opfsUtil.randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(""); + /* + An alternative impl. with an unpredictable length + but much simpler: + + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) + */ + }; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + const dir = await opfsUtil.getRootDir(); + await doDir(dir, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). + */ + opfsUtil.rmfr = async function(){ + const rd = await opfsUtil.getRootDir(); + const dir = rd, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each + entry. The argument may be either a callback function or an + options object with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: await opfsUtil.getRootDir() + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + /** + Impl of opfsUtil.importDb() when it's given a function as its + second argument. + */ + const importDbChunked = async function(filename, callback){ + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + let sah = await hFile.createSyncAccessHandle(); + let nWrote = 0, chunk, checkedHeader = false, err = false; + try{ + sah.truncate(0); + while( undefined !== (chunk = await callback()) ){ + if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); + if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ + util.affirmDbHeader(chunk); + checkedHeader = true; + } + sah.write(chunk, {at: nWrote}); + nWrote += chunk.byteLength; + } + if( nWrote < 512 || 0!==nWrote % 512 ){ + toss("Input size",nWrote,"is not correct for an SQLite database."); + } + if( !checkedHeader ){ + const header = new Uint8Array(20); + sah.read( header, {at: 0} ); + util.affirmDbHeader( header ); + } + sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); + return nWrote; + }catch(e){ + await sah.close(); + sah = undefined; + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally { + if( sah ) await sah.close(); + } + }; + + /** + Asynchronously imports the given bytes (a byte array or + ArrayBuffer) into the given database file. + + Results are undefined if the given db name refers to an opened + db. + + If passed a function for its second argument, its behaviour + changes: imports its data in chunks fed to it by the given + callback function. It calls the callback (which may be async) + repeatedly, expecting either a Uint8Array or ArrayBuffer (to + denote new input) or undefined (to denote EOF). For so long as + the callback continues to return non-undefined, it will append + incoming data to the given VFS-hosted database file. When + called this way, the resolved value of the returned Promise is + the number of bytes written to the target file. + + It very specifically requires the input to be an SQLite3 + database and throws if that's not the case. It does so in + order to prevent this function from taking on a larger scope + than it is specifically intended to. i.e. we do not want it to + become a convenience for importing arbitrary files into OPFS. + + This routine rewrites the database header bytes in the output + file (not the input array) to force disabling of WAL mode. + + On error this throws and the state of the input file is + undefined (it depends on where the exception was triggered). + + On success, resolves to the number of bytes written. + */ + opfsUtil.importDb = async function(filename, bytes){ + if( bytes instanceof Function ){ + return importDbChunked(filename, bytes); + } + if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); + util.affirmIsDb(bytes); + const n = bytes.byteLength; + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + let sah, err, nWrote = 0; + try { + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + sah = await hFile.createSyncAccessHandle(); + sah.truncate(0); + nWrote = sah.write(bytes, {at: 0}); + if(nWrote != n){ + toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); + } + sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; + return nWrote; + }catch(e){ + if( sah ){ await sah.close(); sah = undefined; } + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally{ + if( sah ) await sah.close(); + } + }; + + /** + Checks for features required for OPFS VFSes and throws with a + descriptive error message if they're not found. This is intended + to be run as part of async VFS installation steps. + */ + opfsUtil.vfsInstallationFeatureCheck = function(vfsName){ + if( !globalThis.SharedArrayBuffer || !globalThis.Atomics ){ + toss("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics.", + "The server must emit the COOP/COEP response headers to enable those.", + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep"); + }else if( 'undefined'===typeof WorkerGlobalScope ){ + toss("The OPFS sqlite3_vfs cannot run in the main thread", + "because it requires Atomics.wait()."); + }else if( !globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle || + !navigator?.storage?.getDirectory ){ + toss("Missing required OPFS APIs."); + }else if( 'opfs-wl'===vfsName && !globalThis.Atomics.waitAsync ){ + toss('The',vfsName,'VFS requires Atomics.waitAsync(), which is not available.'); + } + }; + + /** + Must be called by the VFS's main installation routine and passed + the options object that function receives and a reference to that + function itself (we don't need this anymore). + + It throws if OPFS is not available. + + If it returns falsy, it detected that OPFS should be disabled, in + which case the callee should immediately return/resolve to the + sqlite3 object. + + Else it returns a new copy of the options object, fleshed out + with any missing defaults. The caller must: + + - Set up any local state they need. + + - Call opfsUtil.createVfsState(vfsName,opt), where opt is the + object returned by this function. + + - Set up any references they may need to state returned + by the previous step. + + - Call opfvs.bindVfs() + */ + opfsUtil.initOptions = function callee(vfsName, options){ + const urlParams = new URL(globalThis.location.href).searchParams; + if( urlParams.has(vfsName+'-disable') ){ + //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); + return; + } + try{ + opfsUtil.vfsInstallationFeatureCheck(vfsName); + }catch(e){ + return; + } + options = util.nu(options); + options.vfsName = vfsName; + options.verbose ??= urlParams.has('opfs-verbose') + ? +urlParams.get('opfs-verbose') : 1; + options.sanityChecks ??= urlParams.has('opfs-sanity-check'); + + if( !opfsUtil.proxyUri ){ + opfsUtil.proxyUri = "sqlite3-opfs-async-proxy.js"; + if( sqlite3.scriptInfo?.sqlite3Dir ){ + /* Doing this from one scope up, outside of this function, does + not work. */ + opfsUtil.proxyUri = ( + sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri + ); + } + } + options.proxyUri ??= opfsUtil.proxyUri; + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } + //sqlite3.config.warn("opfsUtil options =",JSON.stringify(options), 'urlParams =', urlParams); + return opfsUtil.options = options; + }; + + /** + Creates, populates, and returns the main state object used by the + "opfs" and "opfs-wl" VFSes, and transfered from those to their + async counterparts. + + The returned object's vfs property holds the fully-populated + capi.sqlite3_vfs instance, tagged with lots of extra state which + the current VFSes need to have exposed to them. + + After setting up any local state needed, the caller must call + theVfs.bindVfs(X,Y), where X is an object containing the + sqlite3_io_methods to override and Y is a callback which gets + triggered if init succeeds, before the final Promise decides + whether or not to reject. + + This object must, when it's passed to the async part, contain + only cloneable or sharable objects. After the worker's "inited" + message arrives, other types of data may be added to it. + */ + opfsUtil.createVfsState = function(){ + const state = util.nu(); + const options = opfsUtil.options; + state.verbose = options.verbose; + + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const vfsName = options.vfsName + || toss("Maintenance required: missing VFS name"); + const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level](vfsName+":",...args); + }; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; + + const opfsVfs = state.vfs = new capi.sqlite3_vfs(); + const opfsIoMethods = opfsVfs.ioMethods = new capi.sqlite3_io_methods(); + + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit + is undocumented/unspecified. */; + opfsVfs.$zName = wasm.allocCString(vfsName); + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, opfsIoMethods + /** + Pedantic sidebar: the entries in this array are items to + clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown + order of the wasm engine and the JS one are undefined so + there is no guaranty that the opfsVfs instance would be + available in one environment or the other when + sqlite3_os_end() is called (_if_ it gets called at all in a + wasm build, which is undefined). i.e. addOnDispose() here is + a matter of "correctness", not necessity. It just wouldn't do + to leave the impression that we're blindly leaking memory. + */ + ); + + opfsVfs.metrics = util.nu({ + counters: util.nu(), + dump: function(){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; + } + sqlite3.config.log(globalThis.location.href, + "metrics for",globalThis.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); + sqlite3.config.log("Serialization metrics:",opfsVfs.metrics.counters.s11n); + opfsVfs.worker?.postMessage?.({type:'opfs-async-metrics'}); + }, + reset: function(){ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + const m = opfsVfs.metrics.counters; + for(k in state.opIds){ + r(m[k] = Object.create(null)); + } + let s = m.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = m.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + } + })/*opfsVfs.metrics*/; + + /** + asyncIdleWaitTime is how long (ms) to wait, in the async proxy, + for each Atomics.wait() when waiting on inbound VFS API calls. + We need to wake up periodically to give the thread a chance to + do other things. If this is too high (e.g. 500ms) then even two + workers/tabs can easily run into locking errors. Some multiple + of this value is also used for determining how long to wait on + lock contention to free up. + */ + state.asyncIdleWaitTime = 150; + + /** + Whether the async counterpart should log exceptions to + the serialization channel. That produces a great deal of + noise for seemingly innocuous things like xAccess() checks + for missing files, so this option may have one of 3 values: + + 0 = no exception logging. + + 1 = only log exceptions for "significant" ops like xOpen(), + xRead(), and xWrite(). Exceptions related to, e.g., wait/retry + loops in acquiring SyncAccessHandles are not logged. + + 2 = log all exceptions. + */ + state.asyncS11nExceptions = 1; + /* Size of file I/O buffer block. 64k = max sqlite3 page size, and + xRead/xWrite() will never deal in blocks larger than that. */ + state.fileBufferSize = 1024 * 64; + state.sabS11nOffset = state.fileBufferSize; + /** + The size of the block in our SAB for serializing arguments and + result values. Needs to be large enough to hold serialized + values of any of the proxied APIs. Filenames are the largest + part but are limited to opfsVfs.$mxPathname bytes. We also + store exceptions there, so it needs to be long enough to hold + a reasonably long exception string. + */ + state.sabS11nSize = opfsVfs.$mxPathname * 2; + /** + The SAB used for all data I/O between the synchronous and + async halves (file i/o and arg/result s11n). + */ + state.sabIO = new SharedArrayBuffer( + state.fileBufferSize/* file i/o block */ + + state.sabS11nSize/* argument/result serialization block */ + ); + + /** + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. state.opIds holds the SAB slot + IDs of each of those. + */ + state.opIds = Object.create(null); + { + /* Indexes for use in our SharedArrayBuffer... */ + let i = 0; + /* SAB slot used to communicate which operation is desired + between both workers. This worker writes to it and the other + listens for changes and clears it. The values written to it + are state.opIds.x[A-Z][a-z]+, defined below.*/ + state.opIds.whichOp = i++; + /* Slot for storing return values. This side listens to that + slot and the async proxy writes to it. */ + state.opIds.rc = i++; + /* Each function gets an ID which this worker writes to the + state.opIds.whichOp slot. The async-api worker uses + Atomic.wait() on the whichOp slot to figure out which + operation to run next. */ + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xDeleteNoWait = i++; + state.opIds.xFileSize = i++; + state.opIds.xLock = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xUnlock = i++; + state.opIds.xWrite = i++; + state.opIds.mkdir = i++ /*currently unused*/; + /** Internal signals which are used only during development and + testing via the dev console. */ + state.opIds['opfs-async-metrics'] = i++; + state.opIds['opfs-async-shutdown'] = i++; + /* The retry slot is used by the async part for wait-and-retry + semantics. It is never written to, only used as a convenient + place to wait-with-timeout for a value which will never be + written, i.e. sleep()ing, before retrying a failed attempt to + acquire a SharedAccessHandle. */ + state.opIds.retry = i++; + state.sabOP = new SharedArrayBuffer( + i * 4/* 4==sizeof int32, noting that Atomics.wait() and + friends can only function on Int32Array views of an + SAB. */); + } + /** + SQLITE_xxx constants to export to the async worker + counterpart... + */ + state.sq3Codes = Object.create(null); + for(const k of [ + 'SQLITE_ACCESS_EXISTS', + 'SQLITE_ACCESS_READWRITE', + 'SQLITE_BUSY', + 'SQLITE_CANTOPEN', + 'SQLITE_ERROR', + 'SQLITE_IOERR', + 'SQLITE_IOERR_ACCESS', + 'SQLITE_IOERR_CLOSE', + 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_LOCK', + 'SQLITE_IOERR_READ', + 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_TRUNCATE', + 'SQLITE_IOERR_UNLOCK', + 'SQLITE_IOERR_WRITE', + 'SQLITE_LOCK_EXCLUSIVE', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCKED', + 'SQLITE_MISUSE', + 'SQLITE_NOTFOUND', + 'SQLITE_OPEN_CREATE', + 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', + 'SQLITE_OPEN_READONLY', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_EXCLUSIVE' + ]){ + state.sq3Codes[k] = + capi[k] ?? toss("Maintenance required: not found:",k); + } + + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" + enables this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + Flag for use with xOpen(). URI flag "delete-before-open=1" + tells the VFS to delete the db file before attempting to open + it. This can be used, e.g., to replace a db which has been + corrupted (without forcing us to expose a delete/unlink() + function in the public API). + + Failure to unlink the file is ignored but may lead to + downstream errors. An unlink can fail if, e.g., another tab + has the handle open. + + It goes without saying that deleting a file out from under + another instance results in Undefined Behavior. + */ + OPFS_UNLINK_BEFORE_OPEN: 0x02, + /** + If true, any async routine which must implicitly acquire a + sync access handle (i.e. an OPFS lock), without an active + xLock(), will release that lock at the end of the call which + acquires it. If false, such implicit locks are not released + until the VFS is idle for some brief amount of time, as + defined by state.asyncIdleWaitTime. + + The benefit of enabling this is higher concurrency. The + down-side is much-reduced performance (as much as a 4x + decrease in speedtest1). + */ + defaultUnlockAsap: false + }); + + opfsVfs.metrics.reset()/*must not be called until state.opIds is set up*/; + const metrics = opfsVfs.metrics.counters; + + /** + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The 2nd + and subsequent arguments must be the arguments for the async op + (see sqlite3-opfs-async-proxy.c-pp.js). + */ + const opRun = opfsVfs.opRun = (op,...args)=>{ + const opNdx = state.opIds[op] || toss(opfsVfs.vfsName+": Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, state.opIds.rc, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + /* async thread will take over here */; + const t = performance.now(); + while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ + /* + The reason for this loop is buried in the details of a long + discussion at: + + https://github.com/sqlite/sqlite-wasm/issues/12 + + Summary: in at least one browser flavor, under high loads, + the wait()/notify() pairings can get out of sync and/or + spuriously wake up. Calling wait() here until it returns + 'not-equal' gets them back in sync. + */ + } + /* When the above wait() call returns 'not-equal', the async + half will have completed the operation and reported its + results in the state.opIds.rc slot of the SAB. It may have + also serialized an exception for us. */ + const rc = Atomics.load(state.sabOPView, state.opIds.rc); + metrics[op].wait += performance.now() - t; + if(rc && state.asyncS11nExceptions){ + const err = state.s11n.deserialize(); + if(err) error(op+"() async error:",...err); + } + return rc; + }; + + const opTimer = Object.create(null); + opTimer.op = undefined; + opTimer.start = undefined; + const mTimeStart = opfsVfs.mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + ++metrics[op].count; + }; + const mTimeEnd = opfsVfs.mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start + ); + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = opfsVfs.__openFiles = Object.create(null); + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = opfsVfs.ioSyncWrappers = util.nu({ + xCheckReservedLock: function(pFile,pOut){ + /** + After consultation with a topic expert: "opfs-wl" will + continue to use the same no-op impl which "opfs" does + because: + + - xCheckReservedLock() is just a hint. If SQLite needs to + lock, it's still going to try to lock. + + - We cannot do this check synchronously in "opfs-wl", + so would need to pass it to the async proxy. That would + make it inordinately expensive considering that it's + just a hint. + */ + wasm.poke(pOut, 0, 'i32'); + return 0; + }, + xClose: function(pFile){ + mTimeStart('xClose'); + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + mTimeEnd(); + return rc; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + /*mTimeStart('xFileControl'); + mTimeEnd();*/ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + mTimeStart('xFileSize'); + let rc = opRun('xFileSize', pFile); + if(0==rc){ + try { + const sz = state.s11n.deserialize()[0]; + wasm.poke(pSz64, sz, 'i64'); + }catch(e){ + error("Unexpected error reading xFileSize() result:",e); + rc = state.sq3Codes.SQLITE_IOERR; + } + } + mTimeEnd(); + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + mTimeStart('xRead'); + const f = __openFiles[pFile]; + let rc; + try { + rc = opRun('xRead',pFile, n, Number(offset64)); + if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ + /** + Results get written to the SharedArrayBuffer f.sabView. + Because the heap is _not_ a SharedArrayBuffer, we have + to copy the results. TypedArray.set() seems to be the + fastest way to copy this. */ + wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); + } + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + mTimeEnd(); + return rc; + }, + xSync: function(pFile,flags){ + mTimeStart('xSync'); + const rc = opRun('xSync', pFile, flags); + mTimeEnd(); + return rc; + }, + xTruncate: function(pFile,sz64){ + mTimeStart('xTruncate'); + const rc = opRun('xTruncate', pFile, Number(sz64)); + mTimeEnd(); + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + mTimeStart('xWrite'); + const f = __openFiles[pFile]; + let rc; + try { + f.sabView.set(wasm.heap8u().subarray( + Number(pSrc), Number(pSrc) + n + )); + rc = opRun('xWrite', pFile, n, Number(offset64)); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_WRITE; + } + mTimeEnd(); + return rc; + } + })/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = opfsVfs.vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + mTimeStart('xAccess'); + const rc = opRun('xAccess', wasm.cstrToJs(zName)); + wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); + mTimeEnd(); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + mTimeStart('xDelete'); + const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); + mTimeEnd(); + return rc; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return ipMethods is NULL. */ + if(fh.readOnly){ + wasm.poke(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); + } + __openFiles[pFile] = fh; + fh.sabView = state.sabFileBufView; + fh.sq3File = new capi.sqlite3_file(pFile); + if( zToFree ) fh.sq3File.addOnDispose(zToFree); + fh.sq3File.$pMethods = opfsIoMethods.pointer; + fh.lockType = capi.SQLITE_LOCK_NONE; + } + mTimeEnd(); + return rc; + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + if(pDVfs){ + const dVfs = new capi.sqlite3_vfs(pDVfs); + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + dVfs.dispose(); + } + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + opfsVfs.vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + const npOut = Number(pOut); + for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + /* If we can inherit an xSleep() impl from the default VFS then + assume it's sane and use it, otherwise install a JS-based + one. */ + opfsVfs.vfsSyncWrappers.xSleep = function(pVfs,ms){ + mTimeStart('xSleep'); + Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); + mTimeEnd(); + return 0; + }; + } + +//#define vfs.metrics.enable +//#// import initS11n() +//#include api/opfs-common-inline.c-pp.js +//#undef vfs.metrics.enable + opfsVfs.initS11n = initS11n; + + /** + To be called by the VFS's main installation routine after it has + wired up enough state to provide its overridden io-method impls + (which must be properties of the ioMethods argument). Returns a + Promise which the installation routine must return. callback must + be a function which performs any post-bootstrap touchups, namely + plugging in a sqlite3.oo1 wrapper. It is passed (sqlite3, opfsVfs), + where opfsVfs is the sqlite3_vfs object which was set up by + opfsUtil.createVfsState(). + */ + opfsVfs.bindVfs = function(ioMethods, callback){ + Object.assign(opfsVfs.ioSyncWrappers, ioMethods); + const thePromise = new Promise(function(promiseResolve_, promiseReject_){ + let promiseWasRejected = undefined; + const promiseReject = (err)=>{ + promiseWasRejected = true; + opfsVfs.dispose(); + return promiseReject_(err); + }; + const promiseResolve = ()=>{ + try{ + callback(sqlite3, opfsVfs); + }catch(e){ + return promiseReject(e); + } + promiseWasRejected = false; + return promiseResolve_(sqlite3); + }; + const options = opfsUtil.options; + let proxyUri = options.proxyUri +( + (options.proxyUri.indexOf('?')<0) ? '?' : '&' + )+'vfs='+vfsName; + //sqlite3.config.error("proxyUri",options.proxyUri, (new Error())); + const W = opfsVfs.worker = +//#if target:es6-bundler-friendly + (()=>{ + /* _Sigh_... */ + switch(vfsName){ + case 'opfs': + return new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); + case 'opfs-wl': + return new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); + } + })(); +//#elif target:es6-module + new Worker(new URL(proxyUri, import.meta.url)); +//#else + new Worker(proxyUri); +//#endif + let zombieTimer = setTimeout(()=>{ + /* At attempt to work around a browser-specific quirk in which + the Worker load is failing in such a way that we neither + resolve nor reject it. This workaround gives that resolve/reject + a time limit and rejects if that timer expires. Discussion: + https://sqlite.org/forum/forumpost/a708c98dcb3ef */ + if(undefined===promiseWasRejected){ + promiseReject( + new Error("Timeout while waiting for OPFS async proxy worker.") + ); + } + }, 4000); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + + const opRun = opfsVfs.opRun; +//#if nope + /** + Not part of the public API. Only for test/development use. + */ + opfsVfs.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; +//#endif + + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new capi.sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.poke(pOut,0,'i64'); + rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.peek(pOut, 'i64')); + rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.poke(readBuf+6,0); + let jRead = wasm.cstrToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(opfsVfs.vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = opfsVfs.ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //sqlite3.config.warn(vfsName,"Worker.onmessage:",data); + switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; + case 'opfs-async-loaded': + /* Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async + worker. */ + delete state.vfs; + W.postMessage({type: 'opfs-async-init', args: util.nu(state)}); + break; + case 'opfs-async-inited': { + /* Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin... */ + if(true===promiseWasRejected){ + break /* promise was already rejected via timer */; + } + clearTimeout(zombieTimer); + zombieTimer = null; + try { + sqlite3.vfs.installVfs({ + io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} + }); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + opfsVfs.initS11n(); + delete opfsVfs.initS11n; + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(); + }).catch(promiseReject); + }else{ + promiseResolve(); + } + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + case 'debug': + warn("debug message from worker:",data); + break; + default: { + const errMsg = ( + "Unexpected message from the OPFS async worker: " + + JSON.stringify(data) + ); + error(errMsg); + promiseReject(new Error(errMsg)); + break; + } + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; + }/*bindVfs()*/; + + return state; + }/*createVfsState()*/; + +}/*sqlite3ApiBootstrap.initializers*/); +//#endif target:node diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index 670051bd86..348f80ea0b 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -34,6 +34,7 @@ Module.runSQLite3PostLoadInit = async function( - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls - sqlite3-vfs-opfs.c-pp.js => OPFS VFS - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS + - sqlite3-vfs-opfs-wl.c-pp.js => WebLock-using OPFS VFS - post-js-footer.js => this file's epilogue And all of that gets sandwiched between extern-pre-js.js and diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 5e6514b35d..5e1c6ba54d 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -99,6 +99,13 @@ used in WASMFS-capable builds of the library (which the canonical builds do not include). + - `disable` (as of 3.53.0) may be an object with the following + properties: + - `vfs`, an object, may contain a map of VFS names to booleans. + Any mapping to falsy are disabled. The supported names + are: "kvvfs", "opfs", "opfs-sahpool", "opfs-wl". + - Other disabling options may be added in the future. + [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, enabling delayed evaluation. @@ -145,7 +152,8 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( ); return sqlite3ApiBootstrap.sqlite3; } - const config = Object.assign(Object.create(null),{ + const nu = (...obj)=>Object.assign(Object.create(null),...obj); + const config = nu({ exports: undefined, memory: undefined, bigIntEnabled: !!globalThis.BigInt64Array, @@ -162,7 +170,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( certain wasm.xWrap.resultAdapter()s. */ useStdAlloc: false - }, apiConfig || {}); + }, apiConfig); Object.assign(config, { allocExportName: config.useStdAlloc ? 'malloc' : 'sqlite3_malloc', @@ -195,7 +203,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( not documented are installed as 1-to-1 proxies for their C-side counterparts. */ - const capi = Object.create(null); + const capi = nu(); /** Holds state which are specific to the WASM-related infrastructure and glue code. @@ -204,7 +212,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( dynamically after the api object is fully constructed, so not all are documented in this file. */ - const wasm = Object.create(null); + const wasm = nu(); /** Internal helper for SQLite3Error ctor. */ const __rcStr = (rc)=>{ @@ -752,6 +760,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( toss: function(...args){throw new Error(args.join(' '))}, toss3, typedArrayPart: wasm.typedArrayPart, + nu, assert: function(arg,msg){ if( !arg ){ util.toss("Assertion failed:",msg); @@ -1008,7 +1017,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; }; } - const rc = Object.create(null), ov = [0,0]; + const rc = nu(), ov = [0,0]; let i = 0, k; while((k = capi.sqlite3_compileoption_get(i++))){ f._opt(k,ov); @@ -1016,7 +1025,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } return f._result = rc; }else if(Array.isArray(optName)){ - const rc = Object.create(null); + const rc = nu(); optName.forEach((v)=>{ rc[v] = capi.sqlite3_compileoption_used(v); }); @@ -1067,7 +1076,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( The memory lives in the WASM heap and can be used with routines such as wasm.poke() and wasm.heap8u().slice(). */ - wasm.pstack = Object.assign(Object.create(null),{ + wasm.pstack = nu({ /** Sets the current pstack position to the given pointer. Results are undefined if the passed-in value did not come from @@ -1289,7 +1298,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( // sqlite3__wasm_init_wasmfs() is not available return this.dir = ""; } - }.bind(Object.create(null)); + }.bind(nu()); /** Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a @@ -1665,7 +1674,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( default: return capi.SQLITE_MISUSE; } - }.bind(Object.create(null)); + }.bind(nu()); /** Given a (sqlite3_value*), this function attempts to convert it @@ -1899,7 +1908,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); const pv = wasm.peekPtr(this.ptr); return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; - }.bind(Object.create(null)); + }.bind(nu()); /** A wrapper around sqlite3_preupdate_new() which fetches the @@ -1939,6 +1948,62 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( 'sqlite3changeset_old'); }/*changeset/preupdate additions*/ + /** + EXPERIMENTAL. For tentative addition in 3.53.0. + + sqlite3_js_retry_busy(maxTimes,callback[,beforeRetry]) + + Calls the given _synchronous_ callback function. If that function + returns sqlite3.capi.SQLITE_BUSY _or_ throws an SQLite3Error + which a resultCode property of that value then it will suppress + that error and try again, up to the given maximum number of + times. If the callback returns any other value than that, + it is returned. If the maximum number of retries has been + reached, an SQLite3Error with a resultCode value of + sqlite3.capi.SQLITE_BUSY is thrown. If the callback throws any + exception other than the aforementioned BUSY exception, it is + propagated. If it throws a BUSY exception on its final attempt, + that is propagated as well. + + If the beforeRetry argument is given, it must be a _synchronous_ + function. It is called immediately before each retry of the + callback (not for the initial call), passed the attempt number + (so it starts with 2, not 1). If it throws, the exception is + handled as described above. Its result value is ignored. + + To effectively retry "forever", pass a negative maxTimes value, + with the caveat that there is no recovery from that if the code + gets stuck in a deadlock situation. + + TODO: an async variant of this. + */ + capi.sqlite3_js_retry_busy = function(maxTimes, callback, beforeRetry){ + for(let n = 1; n <= maxTimes; ++n){ + try{ + if( beforeRetry && n>1 ) beforeRetry(n); + const rc = callback(); + if( capi.SQLITE_BUSY===rc ){ + if( n===maxTimes ){ + throw new SQLite3Error(rc, [ + "sqlite3_js_retry_busy() max retry attempts (", + maxTimes, + ") reached." + ].join('')); + } + continue; + } + return rc; + }catch(e){ + if( npostMessage({type, payload:args}); const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; @@ -66,6 +78,13 @@ const installAsyncProxy = function(){ */ const state = Object.create(null); + /* initS11n() is preprocessor-injected so that we have identical + copies in the synchronous and async halves. This side does not + load the SQLite library, so does not have access to that copy. */ +//#define opfs-async-proxy +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-async-proxy + /** verbose: @@ -82,7 +101,7 @@ const installAsyncProxy = function(){ 2:console.log.bind(console) }; const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); + if(state.verbose>level) loggers[level](vfsName+' async-proxy',workerId+":",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); @@ -97,12 +116,13 @@ const installAsyncProxy = function(){ */ const __openFiles = Object.create(null); /** - __implicitLocks is a Set of sqlite3_file pointers (integers) which were - "auto-locked". i.e. those for which we obtained a sync access - handle without an explicit xLock() call. Such locks will be - released during db connection idle time, whereas a sync access - handle obtained via xLock(), or subsequently xLock()'d after - auto-acquisition, will not be released until xUnlock() is called. + __implicitLocks is a Set of sqlite3_file pointers (integers) + which were "auto-locked". i.e. those for which we necessarily + obtain a sync access handle without an explicit xLock() call + guarding access. Such locks will be released during + `waitLoop()`'s idle time, whereas a sync access handle obtained + via xLock(), or subsequently xLock()'d after auto-acquisition, + will not be released until xUnlock() is called. Maintenance reminder: if we relinquish auto-locks at the end of the operation which acquires them, we pay a massive performance @@ -271,10 +291,11 @@ const installAsyncProxy = function(){ In order to help alleviate cross-tab contention for a dabase, if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to some fixed number of - times. If acquisition still fails at that point it will give up - and propagate the exception. Client-level code will see that as - an I/O error. + will wait briefly and try again, up to `maxTries` of times. If + acquisition still fails at that point it will give up and + propagate the exception. Client-level code will see that either + as an I/O error or SQLITE_BUSY, depending on the exception and + the context. 2024-06-12: there is a rare race condition here which has been reported a single time: @@ -289,13 +310,31 @@ const installAsyncProxy = function(){ there's another race condition there). That's easy to say but creating a viable test for that condition has proven challenging so far. + + Interface quirk: if fh.xLock is falsy and the handle is acquired + then fh.fid is added to __implicitLocks(). If fh.xLock is truthy, + it is not added as an implicit lock. i.e. xLock() impls must set + fh.xLock immediately _before_ calling this and must arrange to + restore it to its previous value if this function throws. + + 2026-03-06: + + - baseWaitTime is the number of milliseconds to wait for the + first retry, increasing by one factor for each retry. It defaults + to (state.asyncIdleWaitTime*2). + + - maxTries is the number of attempt to make, each one spaced out + by one additional factor of the baseWaitTime (e.g. 300, then 600, + then 900, the 1200...). This MUST be an integer >0. + + Only the Web Locks impl should use the 3rd and 4th parameters. */ - const getSyncHandle = async (fh,opName)=>{ + const getSyncHandle = async (fh, opName, baseWaitTime, maxTries = 6)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, - msBase = state.asyncIdleWaitTime * 2; + const msBase = baseWaitTime ?? (state.asyncIdleWaitTime * 2); + maxTries ??= 6; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -329,6 +368,9 @@ const installAsyncProxy = function(){ /** Stores the given value at state.sabOPView[state.opIds.rc] and then Atomics.notify()'s it. + + The opName is only used for logging and debugging - all result + codes are expected on the same state.sabOPView slot. */ const storeAndNotify = (opName, value)=>{ log(opName+"() => notify(",value,")"); @@ -458,24 +500,12 @@ const installAsyncProxy = function(){ await releaseImplicitLock(fh); storeAndNotify('xFileSize', rc); }, - xLock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - const fh = __openFiles[fid]; - let rc = 0; - const oldLockType = fh.xLock; - fh.xLock = lockType; - if( !fh.syncHandle ){ - try { - await getSyncHandle(fh,'xLock'); - __implicitLocks.delete(fid); - }catch(e){ - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); - fh.xLock = oldLockType; - } - } - storeAndNotify('xLock',rc); - }, + /** + The first argument is semantically invalid here - it's an + address in the synchronous side's heap. We can do nothing with + it here except use it as a unique-per-file identifier. + i.e. a lookup key. + */ xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags/*SQLITE_OPEN_...*/, opfsFlags/*OPFS_...*/){ @@ -533,7 +563,7 @@ const installAsyncProxy = function(){ rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ - error("xRead() failed",e,fh); + //error("xRead() failed",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } @@ -560,15 +590,192 @@ const installAsyncProxy = function(){ affirmNotRO('xTruncate', fh); await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ - error("xTruncate():",e,fh); + //error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } await releaseImplicitLock(fh); storeAndNotify('xTruncate',rc); }, - xUnlock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ + xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ + let rc; + const fh = __openFiles[fid]; + try{ + affirmNotRO('xWrite', fh); + rc = ( + n === (await getSyncHandle(fh,'xWrite')) + .write(fh.sabView.subarray(0, n), + {at: Number(offset64)}) + ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + }catch(e){ + //error("xWrite():",e,fh); + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); + } + await releaseImplicitLock(fh); + storeAndNotify('xWrite',rc); + } + }/*vfsAsyncImpls*/; + + if( isWebLocker ){ + /* We require separate xLock() and xUnlock() implementations for the + original and Web Lock implementations. The ones in this block + are for the WebLock impl. + + The Golden Rule for this impl is: if we have a web lock, we + must also hold the SAH. When "upgrading" an implicit lock to a + requested (explicit) lock, we must remove the SAH from the + __implicitLocks set. When we unlock, we release both the web + lock and the SAH. That invariant must be kept intact or race + conditions on SAHs will ensue. + */ + /** Registry of active Web Locks: fid -> { mode, resolveRelease } */ + const __activeWebLocks = Object.create(null); + + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/, + isFromUnlock/*only if called from this.xUnlock()*/){ + const whichOp = isFromUnlock ? 'xUnlock' : 'xLock'; + const fh = __openFiles[fid]; + //error("xLock()",fid, lockType, isFromUnlock, fh); + const requestedMode = (lockType >= state.sq3Codes.SQLITE_LOCK_RESERVED) + ? 'exclusive' : 'shared'; + const existing = __activeWebLocks[fid]; + if( existing ){ + if( existing.mode === requestedMode + || (existing.mode === 'exclusive' + && requestedMode === 'shared') ) { + fh.xLock = lockType; + storeAndNotify(whichOp, 0); + /* Don't do this: existing.mode = requestedMode; + + Paraphrased from advice given by a consulting developer: + + If you hold an exclusive lock and SQLite requests shared, + you should keep exiting.mode as exclusive in because the + underlying Web Lock is still exclusive. Changing it to + shared would trick xLock into thinking it needs to + perform a release/re-acquire dance if an exclusive is + later requested. + */ + return 0 /* Already held at required or higher level */; + } + /* + Upgrade path: we must release shared and acquire exclusive. + This transition is NOT atomic in Web Locks API. + + It _effectively_ is atomic if we don't call + closeSyncHandle(fh), as no other worker can lock that until + we let it go. But we can't do that without eventually + leading to deadly embrace situations, so we don't do that. + (That's not a hypothetical, it has happened.) + */ + await closeSyncHandle(fh); + existing.resolveRelease(); + delete __activeWebLocks[fid]; + } + + const lockName = "sqlite3-vfs-opfs:" + fh.filenameAbs; + const oldLockType = fh.xLock; + return new Promise((resolveWaitLoop) => { + //error("xLock() initial promise entered..."); + navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => { + //error("xLock() Web Lock entered.", fh); + __implicitLocks.delete(fid); + let rc = 0; + try{ + fh.xLock = lockType/*must be set before getSyncHandle() is called!*/; + await getSyncHandle(fh, 'xLock', state.asyncIdleWaitTime, 5); + }catch(e){ + fh.xLock = oldLockType; + state.s11n.storeException(1, e); + rc = GetSyncHandleError.convertRc(e, state.sq3Codes.SQLITE_BUSY); + } + const releasePromise = rc + ? undefined + : new Promise((resolveRelease) => { + __activeWebLocks[fid] = { mode: requestedMode, resolveRelease }; + }); + storeAndNotify(whichOp, rc) /* unblock the C side */; + resolveWaitLoop(0) /* unblock waitLoop() */; + await releasePromise /* hold the lock until xUnlock */; + }); + }); + }; + + /** Internal helper for the opfs-wl xUnlock() */ + const wlCloseHandle = async(fh)=>{ + let rc = 0; + try{ + /* For the record, we've never once seen closeSyncHandle() + throw, nor should it because destructors do not throw. */ + await closeSyncHandle(fh); + }catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + } + return rc; + }; + + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + const fh = __openFiles[fid]; + const existing = __activeWebLocks[fid]; + if( !existing ){ + const rc = await wlCloseHandle(fh); + storeAndNotify('xUnlock', rc); + return rc; + } + //error("xUnlock()",fid, lockType, fh); + let rc = 0; + if( lockType === state.sq3Codes.SQLITE_LOCK_NONE ){ + /* SQLite usually unlocks all the way to NONE */ + rc = await wlCloseHandle(fh); + existing.resolveRelease(); + delete __activeWebLocks[fid]; + fh.xLock = lockType; + }else if( lockType === state.sq3Codes.SQLITE_LOCK_SHARED + && existing.mode === 'exclusive' ){ + /* downgrade Exclusive -> Shared */ + rc = await wlCloseHandle(fh); + if( 0===rc ){ + fh.xLock = lockType; + existing.resolveRelease(); + delete __activeWebLocks[fid]; + return vfsAsyncImpls.xLock(fid, lockType, true); + } + }else{ + /* ??? */ + error("xUnlock() unhandled condition", fh); + } + storeAndNotify('xUnlock', rc); + return 0; + } + + }else{ + /* Original/"legacy" xLock() and xUnlock() */ + + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + const fh = __openFiles[fid]; + let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; + if( !fh.syncHandle ){ + try { + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); + }catch(e){ + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); + fh.xLock = oldLockType; + } + } + storeAndNotify('xLock',rc); + }; + + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ let rc = 0; const fh = __openFiles[fid]; if( fh.syncHandle @@ -582,173 +789,112 @@ const installAsyncProxy = function(){ } } storeAndNotify('xUnlock',rc); - }, - xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ - let rc; - const fh = __openFiles[fid]; - try{ - affirmNotRO('xWrite', fh); - rc = ( - n === (await getSyncHandle(fh,'xWrite')) - .write(fh.sabView.subarray(0, n), - {at: Number(offset64)}) - ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; - }catch(e){ - error("xWrite():",e,fh); - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); - } - await releaseImplicitLock(fh); - storeAndNotify('xWrite',rc); } - }/*vfsAsyncImpls*/; - const initS11n = ()=>{ - /** - ACHTUNG: this code is 100% duplicated in the other half of this - proxy! The documentation is maintained in the "synchronous half". - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - state.s11n.deserialize = function(clear=false){ - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - return rc; - }; - state.s11n.serialize = function(...args){ - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - }; - - state.s11n.storeException = state.asyncS11nExceptions - ? ((priority,e)=>{ - if(priority<=state.asyncS11nExceptions){ - state.s11n.serialize([e.name,': ',e.message].join("")); - } - }) - : ()=>{}; - - return state.s11n; - }/*initS11n()*/; + }/*xLock() and xUnlock() impls*/ const waitLoop = async function f(){ - const opHandlers = Object.create(null); - for(let k of Object.keys(state.opIds)){ - const vi = vfsAsyncImpls[k]; - if(!vi) continue; - const o = Object.create(null); - opHandlers[state.opIds[k]] = o; - o.key = k; - o.f = vi; + if( !f.inited ){ + f.inited = true; + f.opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + f.opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } } + const opIds = state.opIds; + const opView = state.sabOPView; + const slotWhichOp = opIds.whichOp; + const idleWaitTime = state.asyncIdleWaitTime; + const hasWaitAsync = !!Atomics.waitAsync; +//#if nope + error("waitLoop init: isWebLocker",isWebLocker, + "idleWaitTime",idleWaitTime, + "hasWaitAsync",hasWaitAsync); +//#endif while(!flagAsyncShutdown){ try { - if('not-equal'!==Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, state.asyncIdleWaitTime - )){ - /* Maintenance note: we compare against 'not-equal' because + let opId; + if( hasWaitAsync ){ + opId = Atomics.load(opView, slotWhichOp); + if( 0===opId ){ + const rv = Atomics.waitAsync(opView, slotWhichOp, 0, + idleWaitTime); + if( rv.async ) await rv.value; + await releaseImplicitLocks(); + continue; + } + }else{ + /** + For browsers without Atomics.waitAsync(), we require + the legacy implementation. Browser versions where + waitAsync() arrived: - https://github.com/tomayac/sqlite-wasm/issues/12 + Chrome: 90 (2021-04-13) + Firefox: 145 (2025-11-11) + Safari: 16.4 (2023-03-27) - is reporting that this occasionally, under high loads, - returns 'ok', which leads to the whichOp being 0 (which - isn't a valid operation ID and leads to an exception, - along with a corresponding ugly console log - message). Unfortunately, the conditions for that cannot - be reliably reproduced. The only place in our code which - writes a 0 to the state.opIds.whichOp SharedArrayBuffer - index is a few lines down from here, and that instance - is required in order for clear communication between - the sync half of this proxy and this half. + The "opfs" VFS was not born until Chrome was somewhere in + the v104-108 range (Summer/Autumn 2022) and did not work + with Safari < v17 (2023-09-18) due to a WebKit bug which + restricted OPFS access from sub-Workers. + + The waitAsync() counterpart of this block can be used by + both "opfs" and "opfs-wl", whereas this block can only be + used by "opfs". Performance comparisons between the two + in high-contention tests have been indecisive. */ - await releaseImplicitLocks(); - continue; + if('not-equal'!==Atomics.wait( + state.sabOPView, slotWhichOp, 0, state.asyncIdleWaitTime + )){ + /* Maintenance note: we compare against 'not-equal' because + + https://github.com/tomayac/sqlite-wasm/issues/12 + + is reporting that this occasionally, under high loads, + returns 'ok', which leads to the whichOp being 0 (which + isn't a valid operation ID and leads to an exception, + along with a corresponding ugly console log + message). Unfortunately, the conditions for that cannot + be reliably reproduced. The only place in our code which + writes a 0 to the state.opIds.whichOp SharedArrayBuffer + index is a few lines down from here, and that instance + is required in order for clear communication between + the sync half of this proxy and this half. + + Much later (2026-03-07): that phenomenon is apparently + called a spurious wakeup. + */ + await releaseImplicitLocks(); + continue; + } + opId = Atomics.load(state.sabOPView, slotWhichOp); } - const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); - Atomics.store(state.sabOPView, state.opIds.whichOp, 0); - const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + Atomics.store(opView, slotWhichOp, 0); + const hnd = f.opHandlers[opId]?.f ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with an exception string written by the upcoming operation */ ) || []; - //warn("waitLoop() whichOp =",opId, hnd, args); - if(hnd.f) await hnd.f(...args); - else error("Missing callback for opId",opId); + //error("waitLoop() whichOp =",opId, f.opHandlers[opId].key, args); +//#if nope + if( isWebLocker && (opId==opIds.xLock || opIds==opIds.xUnlock) ){ + /* An expert suggests that this introduces a race condition, + but my eyes aren't seeing it. The hope was that this + would improve the lock speed a tick, but it does not + appear to. */ + hnd(...args); + continue; + } +//#endif + await hnd(...args); }catch(e){ - error('in waitLoop():',e); + error('in waitLoop():', e); } } }; @@ -756,6 +902,7 @@ const installAsyncProxy = function(){ navigator.storage.getDirectory().then(function(d){ state.rootDir = d; globalThis.onmessage = function({data}){ + //log(globalThis.location.href,"onmessage()",data); switch(data.type){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ @@ -771,6 +918,7 @@ const installAsyncProxy = function(){ } }); initS11n(); + //warn("verbosity =",opt.verbose, state.verbose); log("init state",state); wPost('opfs-async-inited'); waitLoop(); @@ -782,22 +930,27 @@ const installAsyncProxy = function(){ flagAsyncShutdown = false; waitLoop(); } - break; + break; } }; wPost('opfs-async-loaded'); }).catch((e)=>error("error initializing OPFS asyncer:",e)); }/*installAsyncProxy()*/; -if(!globalThis.SharedArrayBuffer){ +if(globalThis.window === globalThis){ + wPost('opfs-unavailable', + "This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); +}else if(!globalThis.SharedArrayBuffer){ wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", "The server must emit the COOP/COEP response headers to enable that."); }else if(!globalThis.Atomics){ wPost('opfs-unavailable', "Missing Atomics API.", "The server must emit the COOP/COEP response headers to enable that."); +}else if(isWebLocker && !globalThis.Atomics.waitAsync){ + wPost('opfs-unavailable',"Missing required Atomics.waitSync() for "+vfsName); }else if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle || !navigator?.storage?.getDirectory){ wPost('opfs-unavailable',"Missing required OPFS APIs."); }else{ diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index e3fb722877..88f81226a6 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -28,6 +28,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //#@policy error //#savepoint begin //#define kvvfs-v2-added-in=3.52.0 + /** kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates storage of its pages and metadata to a key-value store. @@ -85,6 +86,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ backups. */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( sqlite3.config.disable?.vfs?.kvvfs ){ + return; + } 'use strict'; const capi = sqlite3.capi, sqlite3_kvvfs_methods = capi.sqlite3_kvvfs_methods, @@ -1137,9 +1141,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*.ioJrnl*/ }/*methodOverrides*/; +//#if nope debug("pVfs and friends", pVfs, pIoDb, pIoJrnl, kvvfsMethods, capi.sqlite3_file.structInfo, KVVfsFile.structInfo); +//#endif + try { util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough" /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index c1fee5e1ab..689f53d4a7 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -55,6 +55,10 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; + if( sqlite3.config.disable?.vfs?.['opfs-sahpool'] ){ + return; + } + const toss = sqlite3.util.toss; const toss3 = sqlite3.util.toss3; const initPromises = Object.create(null) /* cache of (name:result) of VFS init results */; diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js new file mode 100644 index 0000000000..dbd786410a --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -0,0 +1,133 @@ +//#if not target:node +/* + 2026-02-20 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is a reimplementation of the "opfs" VFS (as distinct from + "opfs-sahpool") which uses WebLocks for locking instead of a bespoke + custom Atomics.wait()/notify() protocol. This file holds the + "synchronous half" of the VFS, whereas it shares the "asynchronous + half" of the "opfs" VFS. + + This file is intended to be appended to the main sqlite3 JS + deliverable somewhere after sqlite3-api-oo1.js. + + TODOs (2026-03-03): + + - For purposes of tester1.js we need to figure out which of these + VFSes will install the (internal-use-only) sqlite3.opfs utility code + namespace. We need that in order to clean up OPFS files during test + runs. Alternately, move those into their own + sqlite3ApiBootstrap.initializers entry which precedes both of the + VFSes, so they'll have access to it during bootstrapping. The + sqlite3.opfs namespace is removed at the end of bootstrapping unless + the library is told to run in testing mode (which is not a + documented feature). +*/ +'use strict'; +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.['opfs-wl'] ){ + return; + } + const util = sqlite3.util, + toss = sqlite3.util.toss; + const opfsUtil = sqlite3.opfs; + const vfsName = 'opfs-wl'; +/** + installOpfsWlVfs() returns a Promise which, on success, installs an + sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs + which accept a VFS. It is intended to be called via + sqlite3ApiBootstrap.initializers or an equivalent mechanism. + + This VFS is essentially identical to the "opfs" VFS but uses + WebLocks for its xLock() and xUnlock() implementations. + + Quirks specific to this VFS: + + - The (officially undocumented) 'opfs-wl-disable' URL + argument will disable OPFS, making this function a no-op. + + Aside from locking differences in the VFSes, this function + otherwise behaves the same as + sqlite3-vfs-opfs.c-pp.js:installOpfsVfs(). +*/ +const installOpfsWlVfs = async function(options){ + options = opfsUtil.initOptions(vfsName,options); + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState(), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + debug = (...args)=>sqlite3.config.debug(vfsName+":",...args), + warn = (...args)=>sqlite3.config.warn(vfsName+":",...args), + __openFiles = opfsVfs.__openFiles; + + //debug("state",JSON.stringify(options)); + /* + At this point, createVfsState() has populated: + + - state: the configuration object we share with the async proxy. + + - opfsVfs: an sqlite3_vfs instance with lots of JS state attached + to it. + + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... + */ + return opfsVfs.bindVfs(util.nu({ + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + //debug("xLock()..."); + const f = __openFiles[pFile]; + const rc = opRun('xLock', pFile, lockType); + if( !rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + const f = __openFiles[pFile]; + const rc = opRun('xUnlock', pFile, lockType); + if( !rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + } + }), function(sqlite3, vfs){ + /* Post-VFS-registration initialization... */ + if(sqlite3.oo1){ + const OpfsWlDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = vfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsWlDb = OpfsWlDb; + OpfsWlDb.importDb = opfsUtil.importDb; + /* The "opfs" VFS variant adds a + oo1.DB.dbCtorHelper.setVfsPostOpenCallback() callback to set + a high busy_timeout. That was a design mis-decision and is + inconsistent with sqlite3_open() and friends, but is retained + against the risk of introducing regressions if it's removed. + This variant does not repeat that mistake. + */ + } + })/*bindVfs()*/; +}/*installOpfsWlVfs()*/; +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + return installOpfsWlVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install the",vfsName,"sqlite3_vfs:",e); + }); +}); +}/*sqlite3ApiBootstrap.initializers.push()*/); +//#endif target:node diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index ffa90ed068..53619bfc92 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -20,7 +20,12 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ -/** + if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.opfs ){ + return; + } + const util = sqlite3.util, + opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs"); + /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via @@ -58,7 +63,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The argument may optionally be a plain object with the following configuration options: - - proxyUri: name of the async proxy JS file. + - proxyUri: name of the async proxy JS file or a synchronous function + which, when called, returns such a name. - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables logging of errors. 2 enables logging of warnings and errors. 3 @@ -70,1391 +76,105 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Promise resolves. This is only intended for testing and development of the VFS, not client-side use. + Additionaly, the (officially undocumented) 'opfs-disable' URL + argument will disable OPFS, making this function a no-op. + On success, the Promise resolves to the top-most sqlite3 namespace - object and that object gets a new object installed in its - `opfs` property, containing several OPFS-specific utilities. + object. Success does not necessarily mean that it installs the VFS, + as there are legitimate non-error reasons for OPFS not to be + available. */ -const installOpfsVfs = function callee(options){ - if(!globalThis.SharedArrayBuffer - || !globalThis.Atomics){ - return Promise.reject( - new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ - "The server must emit the COOP/COEP response headers to enable those. "+ - "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") - ); - }else if('undefined'===typeof WorkerGlobalScope){ - return Promise.reject( - new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ - "because it requires Atomics.wait().") - ); - }else if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - return Promise.reject( - new Error("Missing required OPFS APIs.") - ); - } - if(!options || 'object'!==typeof options){ - options = Object.create(null); - } - const urlParams = new URL(globalThis.location.href).searchParams; - if(urlParams.has('opfs-disable')){ - //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); - return Promise.resolve(sqlite3); - } - if(undefined===options.verbose){ - options.verbose = urlParams.has('opfs-verbose') - ? (+urlParams.get('opfs-verbose') || 2) : 1; - } - if(undefined===options.sanityChecks){ - options.sanityChecks = urlParams.has('opfs-sanity-check'); - } - if(undefined===options.proxyUri){ - options.proxyUri = callee.defaultProxyUri; - } +const installOpfsVfs = async function(options){ + options = opfsUtil.initOptions('opfs',options); + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState(), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + debug = (...args)=>sqlite3.config.debug("opfs:",...args), + warn = (...args)=>sqlite3.config.warn("opfs:",...args), + __openFiles = opfsVfs.__openFiles; - //sqlite3.config.warn("OPFS options =",options,globalThis.location); + //debug("options:",JSON.stringify(options)); + /* + At this point, createVfsState() has populated: - if('function' === typeof options.proxyUri){ - options.proxyUri = options.proxyUri(); - } - const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); - const toss = sqlite3.util.toss; - const capi = sqlite3.capi; - const util = sqlite3.util; - const wasm = sqlite3.wasm; - const sqlite3_vfs = capi.sqlite3_vfs; - const sqlite3_file = capi.sqlite3_file; - const sqlite3_io_methods = capi.sqlite3_io_methods; - /** - Generic utilities for working with OPFS. This will get filled out - by the Promise setup and, on success, installed as sqlite3.opfs. + - state: the configuration object we share with the async proxy. - ACHTUNG: do not rely on these APIs in client code. They are - experimental and subject to change or removal as the - OPFS-specific sqlite3_vfs evolves. - */ - const opfsUtil = Object.create(null); + - opfsVfs: an sqlite3_vfs instance with lots of JS state attached + to it. - /** - Returns true if _this_ thread has access to the OPFS APIs. - */ - const thisThreadHasOPFS = ()=>{ - return globalThis.FileSystemHandle && - globalThis.FileSystemDirectoryHandle && - globalThis.FileSystemFileHandle && - globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator?.storage?.getDirectory; - }; - - /** - Not part of the public API. Solely for internal/development - use. - */ - opfsUtil.metrics = { - dump: function(){ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; - } - sqlite3.config.log(globalThis.location.href, - "metrics for",globalThis.location.href,":",metrics, - "\nTotal of",n,"op(s) for",t, - "ms (incl. "+w+" ms of waiting on the async side)"); - sqlite3.config.log("Serialization metrics:",metrics.s11n); - W.postMessage({type:'opfs-async-metrics'}); - }, - reset: function(){ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; - } - }/*metrics*/; - const opfsIoMethods = new sqlite3_io_methods(); - const opfsVfs = new sqlite3_vfs() - .addOnDispose( ()=>opfsIoMethods.dispose()); - let promiseWasRejected = undefined; - const promiseReject = (err)=>{ - promiseWasRejected = true; - opfsVfs.dispose(); - return promiseReject_(err); - }; - const promiseResolve = ()=>{ - promiseWasRejected = false; - return promiseResolve_(sqlite3); - }; - const W = -//#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url)); -//#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); -//#else - new Worker(options.proxyUri); -//#endif - setTimeout(()=>{ - /* At attempt to work around a browser-specific quirk in which - the Worker load is failing in such a way that we neither - resolve nor reject it. This workaround gives that resolve/reject - a time limit and rejects if that timer expires. Discussion: - https://sqlite.org/forum/forumpost/a708c98dcb3ef */ - if(undefined===promiseWasRejected){ - promiseReject( - new Error("Timeout while waiting for OPFS async proxy worker.") - ); - } - }, 4000); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - error("Error initializing OPFS asyncer:",err); - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit - is undocumented/unspecified. */; - opfsVfs.$zName = wasm.allocCString("opfs"); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.addOnDispose( - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) - ); - /** - Pedantic sidebar about opfsVfs.ondispose: the entries in that array - are items to clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the opfsVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); - state.verbose = options.verbose; - state.littleEndian = (()=>{ - const buffer = new ArrayBuffer(2); - new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); - // Int16Array uses the platform's endianness. - return new Int16Array(buffer)[0] === 256; - })(); - /** - asyncIdleWaitTime is how long (ms) to wait, in the async proxy, - for each Atomics.wait() when waiting on inbound VFS API calls. - We need to wake up periodically to give the thread a chance to - do other things. If this is too high (e.g. 500ms) then even two - workers/tabs can easily run into locking errors. Some multiple - of this value is also used for determining how long to wait on - lock contention to free up. - */ - state.asyncIdleWaitTime = 150; - - /** - Whether the async counterpart should log exceptions to - the serialization channel. That produces a great deal of - noise for seemingly innocuous things like xAccess() checks - for missing files, so this option may have one of 3 values: - - 0 = no exception logging. - - 1 = only log exceptions for "significant" ops like xOpen(), - xRead(), and xWrite(). - - 2 = log all exceptions. - */ - state.asyncS11nExceptions = 1; - /* Size of file I/O buffer block. 64k = max sqlite3 page size, and - xRead/xWrite() will never deal in blocks larger than that. */ - state.fileBufferSize = 1024 * 64; - state.sabS11nOffset = state.fileBufferSize; - /** - The size of the block in our SAB for serializing arguments and - result values. Needs to be large enough to hold serialized - values of any of the proxied APIs. Filenames are the largest - part but are limited to opfsVfs.$mxPathname bytes. We also - store exceptions there, so it needs to be long enough to hold - a reasonably long exception string. - */ - state.sabS11nSize = opfsVfs.$mxPathname * 2; - /** - The SAB used for all data I/O between the synchronous and - async halves (file i/o and arg/result s11n). - */ - state.sabIO = new SharedArrayBuffer( - state.fileBufferSize/* file i/o block */ - + state.sabS11nSize/* argument/result serialization block */ - ); - state.opIds = Object.create(null); - const metrics = Object.create(null); - { - /* Indexes for use in our SharedArrayBuffer... */ - let i = 0; - /* SAB slot used to communicate which operation is desired - between both workers. This worker writes to it and the other - listens for changes. */ - state.opIds.whichOp = i++; - /* Slot for storing return values. This worker listens to that - slot and the other worker writes to it. */ - state.opIds.rc = i++; - /* Each function gets an ID which this worker writes to - the whichOp slot. The async-api worker uses Atomic.wait() - on the whichOp slot to figure out which operation to run - next. */ - state.opIds.xAccess = i++; - state.opIds.xClose = i++; - state.opIds.xDelete = i++; - state.opIds.xDeleteNoWait = i++; - state.opIds.xFileSize = i++; - state.opIds.xLock = i++; - state.opIds.xOpen = i++; - state.opIds.xRead = i++; - state.opIds.xSleep = i++; - state.opIds.xSync = i++; - state.opIds.xTruncate = i++; - state.opIds.xUnlock = i++; - state.opIds.xWrite = i++; - state.opIds.mkdir = i++; - state.opIds['opfs-async-metrics'] = i++; - state.opIds['opfs-async-shutdown'] = i++; - /* The retry slot is used by the async part for wait-and-retry - semantics. Though we could hypothetically use the xSleep slot - for that, doing so might lead to undesired side effects. */ - state.opIds.retry = i++; - state.sabOP = new SharedArrayBuffer( - i * 4/* ==sizeof int32, noting that Atomics.wait() and friends - can only function on Int32Array views of an SAB. */); - opfsUtil.metrics.reset(); - } - /** - SQLITE_xxx constants to export to the async worker - counterpart... - */ - state.sq3Codes = Object.create(null); - [ - 'SQLITE_ACCESS_EXISTS', - 'SQLITE_ACCESS_READWRITE', - 'SQLITE_BUSY', - 'SQLITE_CANTOPEN', - 'SQLITE_ERROR', - 'SQLITE_IOERR', - 'SQLITE_IOERR_ACCESS', - 'SQLITE_IOERR_CLOSE', - 'SQLITE_IOERR_DELETE', - 'SQLITE_IOERR_FSYNC', - 'SQLITE_IOERR_LOCK', - 'SQLITE_IOERR_READ', - 'SQLITE_IOERR_SHORT_READ', - 'SQLITE_IOERR_TRUNCATE', - 'SQLITE_IOERR_UNLOCK', - 'SQLITE_IOERR_WRITE', - 'SQLITE_LOCK_EXCLUSIVE', - 'SQLITE_LOCK_NONE', - 'SQLITE_LOCK_PENDING', - 'SQLITE_LOCK_RESERVED', - 'SQLITE_LOCK_SHARED', - 'SQLITE_LOCKED', - 'SQLITE_MISUSE', - 'SQLITE_NOTFOUND', - 'SQLITE_OPEN_CREATE', - 'SQLITE_OPEN_DELETEONCLOSE', - 'SQLITE_OPEN_MAIN_DB', - 'SQLITE_OPEN_READONLY' - ].forEach((k)=>{ - if(undefined === (state.sq3Codes[k] = capi[k])){ - toss("Maintenance required: not found:",k); - } - }); - state.opfsFlags = Object.assign(Object.create(null),{ - /** - Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" - enables this. See defaultUnlockAsap, below. - */ - OPFS_UNLOCK_ASAP: 0x01, - /** - Flag for use with xOpen(). URI flag "delete-before-open=1" - tells the VFS to delete the db file before attempting to open - it. This can be used, e.g., to replace a db which has been - corrupted (without forcing us to expose a delete/unlink() - function in the public API). - - Failure to unlink the file is ignored but may lead to - downstream errors. An unlink can fail if, e.g., another tab - has the handle open. - - It goes without saying that deleting a file out from under another - instance results in Undefined Behavior. - */ - OPFS_UNLINK_BEFORE_OPEN: 0x02, - /** - If true, any async routine which implicitly acquires a sync - access handle (i.e. an OPFS lock) will release that lock at - the end of the call which acquires it. If false, such - "autolocks" are not released until the VFS is idle for some - brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). - */ - defaultUnlockAsap: false - }); - - /** - Runs the given operation (by name) in the async worker - counterpart, waits for its response, and returns the result - which the async worker writes to SAB[state.opIds.rc]. The - 2nd and subsequent arguments must be the arguments for the - async op. - */ - const opRun = (op,...args)=>{ - const opNdx = state.opIds[op] || toss("Invalid op ID:",op); - state.s11n.serialize(...args); - Atomics.store(state.sabOPView, state.opIds.rc, -1); - Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); - Atomics.notify(state.sabOPView, state.opIds.whichOp) - /* async thread will take over here */; - const t = performance.now(); - while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ - /* - The reason for this loop is buried in the details of a long - discussion at: - - https://github.com/sqlite/sqlite-wasm/issues/12 - - Summary: in at least one browser flavor, under high loads, - the wait()/notify() pairings can get out of sync. Calling - wait() here until it returns 'not-equal' gets them back in - sync. - */ - } - /* When the above wait() call returns 'not-equal', the async - half will have completed the operation and reported its results - in the state.opIds.rc slot of the SAB. */ - const rc = Atomics.load(state.sabOPView, state.opIds.rc); - metrics[op].wait += performance.now() - t; - if(rc && state.asyncS11nExceptions){ - const err = state.s11n.deserialize(); - if(err) error(op+"() async error:",...err); - } - return rc; - }; - - /** - Not part of the public API. Only for test/development use. - */ - opfsUtil.debug = { - asyncShutdown: ()=>{ - warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); - opRun('opfs-async-shutdown'); - }, - asyncRestart: ()=>{ - warn("Attempting to restart OPFS VFS async listener. Might work, might not."); - W.postMessage({type: 'opfs-async-restart'}); - } - }; - - const initS11n = ()=>{ - /** - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ACHTUNG: this code is 100% duplicated in the other half of - this proxy! The documentation is maintained in the - "synchronous half". - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - This proxy de/serializes cross-thread function arguments and - output-pointer values via the state.sabIO SharedArrayBuffer, - using the region defined by (state.sabS11nOffset, - state.sabS11nOffset + state.sabS11nSize]. Only one dataset is - recorded at a time. - - This is not a general-purpose format. It only supports the - range of operations, and data sizes, needed by the - sqlite3_vfs and sqlite3_io_methods operations. Serialized - data are transient and this serialization algorithm may - change at any time. - - The data format can be succinctly summarized as: - - Nt...Td...D - - Where: - - - N = number of entries (1 byte) - - - t = type ID of first argument (1 byte) - - - ...T = type IDs of the 2nd and subsequent arguments (1 byte - each). - - - d = raw bytes of first argument (per-type size). - - - ...D = raw bytes of the 2nd and subsequent arguments (per-type - size). - - All types except strings have fixed sizes. Strings are stored - using their TextEncoder/TextDecoder representations. It would - arguably make more sense to store them as Int16Arrays of - their JS character values, but how best/fastest to get that - in and out of string form is an open point. Initial - experimentation with that approach did not gain us any speed. - - Historical note: this impl was initially about 1% this size by - using using JSON.stringify/parse(), but using fit-to-purpose - serialization saves considerable runtime. - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - /* Only arguments and return values of these types may be - serialized. This covers the whole range of types needed by the - sqlite3_vfs API. */ - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - - /** - Returns an array of the deserialized state stored by the most - recent serialize() operation (from this thread or the - counterpart thread), or null if the serialization buffer is - empty. If passed a truthy argument, the serialization buffer - is cleared after deserialization. - */ - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - - /** - Serializes all arguments to the shared buffer for consumption - by the counterpart thread. - - This routine is only intended for serializing OPFS VFS - arguments and (in at least one special case) result values, - and the buffer is sized to be able to comfortably handle - those. - - If passed no arguments then it zeroes out the serialization - state. - */ - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - return state.s11n; - }/*initS11n()*/; - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(""); - /* - An alternative impl. with an unpredictable length - but much simpler: - - Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) - */ - }; - - /** - Map of sqlite3_file pointers to objects constructed by xOpen(). - */ - const __openFiles = Object.create(null); - - const opTimer = Object.create(null); - opTimer.op = undefined; - opTimer.start = undefined; - const mTimeStart = (op)=>{ - opTimer.start = performance.now(); - opTimer.op = op; - ++metrics[op].count; - }; - const mTimeEnd = ()=>( - metrics[opTimer.op].time += performance.now() - opTimer.start - ); - - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioSyncWrappers = { - xCheckReservedLock: function(pFile,pOut){ - /** - As of late 2022, only a single lock can be held on an OPFS - file. We have no way of checking whether any _other_ db - connection has a lock except by trying to obtain and (on - success) release a sync-handle for it, but doing so would - involve an inherent race condition. For the time being, - pending a better solution, we simply report whether the - given pFile is open. - - Update 2024-06-12: based on forum discussions, this - function now always sets pOut to 0 (false): - - https://sqlite.org/forum/forumpost/a2f573b00cda1372 - */ - wasm.poke(pOut, 0, 'i32'); - return 0; - }, - xClose: function(pFile){ - mTimeStart('xClose'); - let rc = 0; - const f = __openFiles[pFile]; - if(f){ - delete __openFiles[pFile]; - rc = opRun('xClose', pFile); - if(f.sq3File) f.sq3File.dispose(); - } - mTimeEnd(); - return rc; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - /*mTimeStart('xFileControl'); - mTimeEnd();*/ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - mTimeStart('xFileSize'); - let rc = opRun('xFileSize', pFile); - if(0==rc){ - try { - const sz = state.s11n.deserialize()[0]; - wasm.poke(pSz64, sz, 'i64'); - }catch(e){ - error("Unexpected error reading xFileSize() result:",e); - rc = state.sq3Codes.SQLITE_IOERR; - } - } - mTimeEnd(); - return rc; - }, - xLock: function(pFile,lockType){ - mTimeStart('xLock'); - const f = __openFiles[pFile]; - let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ - if( !f.lockType ) { - rc = opRun('xLock', pFile, lockType); - if( 0===rc ) f.lockType = lockType; - }else{ - f.lockType = lockType; - } - mTimeEnd(); - return rc; - }, - xRead: function(pFile,pDest,n,offset64){ - mTimeStart('xRead'); - const f = __openFiles[pFile]; - let rc; - try { - rc = opRun('xRead',pFile, n, Number(offset64)); - if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ - /** - Results get written to the SharedArrayBuffer f.sabView. - Because the heap is _not_ a SharedArrayBuffer, we have - to copy the results. TypedArray.set() seems to be the - fastest way to copy this. */ - wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); - } - }catch(e){ - error("xRead(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_READ; - } - mTimeEnd(); - return rc; - }, - xSync: function(pFile,flags){ - mTimeStart('xSync'); - ++metrics.xSync.count; - const rc = opRun('xSync', pFile, flags); - mTimeEnd(); - return rc; - }, - xTruncate: function(pFile,sz64){ - mTimeStart('xTruncate'); - const rc = opRun('xTruncate', pFile, Number(sz64)); - mTimeEnd(); - return rc; - }, - xUnlock: function(pFile,lockType){ - mTimeStart('xUnlock'); - const f = __openFiles[pFile]; - let rc = 0; - if( capi.SQLITE_LOCK_NONE === lockType - && f.lockType ){ - rc = opRun('xUnlock', pFile, lockType); - } + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... + */ + return opfsVfs.bindVfs(util.nu({ + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + ++metrics.xLock.count; + const f = __openFiles[pFile]; + let rc = 0; + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( f.lockType ) { + f.lockType = lockType; + }else{ + rc = opRun('xLock', pFile, lockType); if( 0===rc ) f.lockType = lockType; - mTimeEnd(); - return rc; - }, - xWrite: function(pFile,pSrc,n,offset64){ - mTimeStart('xWrite'); - const f = __openFiles[pFile]; - let rc; - try { - f.sabView.set(wasm.heap8u().subarray( - Number(pSrc), Number(pSrc) + n - )); - rc = opRun('xWrite', pFile, n, Number(offset64)); - }catch(e){ - error("xWrite(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_WRITE; - } - mTimeEnd(); - return rc; } - }/*ioSyncWrappers*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsSyncWrappers = { - xAccess: function(pVfs,zName,flags,pOut){ - mTimeStart('xAccess'); - const rc = opRun('xAccess', wasm.cstrToJs(zName)); - wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); - mTimeEnd(); - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - /* If it turns out that we need to adjust for timezone, see: - https://stackoverflow.com/a/11760121/1458521 */ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - mTimeStart('xDelete'); - const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); - mTimeEnd(); - return rc; - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return i!!v) : p; - }; - - /** - Takes the absolute path to a filesystem element. Returns an - array of [handleOfContainingDir, filename]. If the 2nd argument - is truthy then each directory element leading to the file is - created along the way. Throws if any creation or resolution - fails. - */ - opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ - const path = opfsUtil.getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = opfsUtil.rootDirectory; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; - }; - - /** - Creates the given directory name, recursively, in - the OPFS filesystem. Returns true if it succeeds or the - directory already exists, else false. - */ - opfsUtil.mkdir = async function(absDirName){ - try { - await opfsUtil.getDirForFilename(absDirName+"/filepart", true); - return true; - }catch(e){ - //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); - return false; - } - }; - /** - Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't or if an - exception is intercepted while trying to make the - determination. - */ - opfsUtil.entryExists = async function(fsEntryName){ - try { - const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); - await dh.getFileHandle(fn); - return true; - }catch(e){ - return false; - } - }; - - /** - Generates a random ASCII string, intended for use as a - temporary file name. Its argument is the length of the string, - defaulting to 16. - */ - opfsUtil.randomFilename = randomFilename; - - /** - Returns a promise which resolves to an object which represents - all files and directories in the OPFS tree. The top-most object - has two properties: `dirs` is an array of directory entries - (described below) and `files` is a list of file names for all - files in that directory. - - Traversal starts at sqlite3.opfs.rootDirectory. - - Each `dirs` entry is an object in this form: - - ``` - { name: directoryName, - dirs: [...subdirs], - files: [...file names] - } - ``` - - The `files` and `subdirs` entries are always set but may be - empty arrays. - - The returned object has the same structure but its `name` is - an empty string. All returned objects are created with - Object.create(null), so have no prototype. - - Design note: the entries do not contain more information, - e.g. file sizes, because getting such info is not only - expensive but is subject to locking-related errors. - */ - opfsUtil.treeList = async function(){ - const doDir = async function callee(dirHandle,tgt){ - tgt.name = dirHandle.name; - tgt.dirs = []; - tgt.files = []; - for await (const handle of dirHandle.values()){ - if('directory' === handle.kind){ - const subDir = Object.create(null); - tgt.dirs.push(subDir); - await callee(handle, subDir); - }else{ - tgt.files.push(handle.name); - } - } - }; - const root = Object.create(null); - await doDir(opfsUtil.rootDirectory, root); - return root; - }; - - /** - Irrevocably deletes _all_ files in the current origin's OPFS. - Obviously, this must be used with great caution. It may throw - an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which the underlying - APIs will throw are not documented (so we cannot tell you what - they are). - */ - opfsUtil.rmfr = async function(){ - const dir = opfsUtil.rootDirectory, opt = {recurse: true}; - for await (const handle of dir.values()){ - dir.removeEntry(handle.name, opt); - } - }; - - /** - Deletes the given OPFS filesystem entry. As this environment - has no notion of "current directory", the given name must be an - absolute path. If the 2nd argument is truthy, deletion is - recursive (use with caution!). - - The returned Promise resolves to true if the deletion was - successful, else false (but...). The OPFS API reports the - reason for the failure only in human-readable form, not - exceptions which can be type-checked to determine the - failure. Because of that... - - If the final argument is truthy then this function will - propagate any exception on error, rather than returning false. - */ - opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ - try { - const [hDir, filenamePart] = - await opfsUtil.getDirForFilename(fsEntryName, false); - await hDir.removeEntry(filenamePart, {recursive}); - return true; - }catch(e){ - if(throwOnError){ - throw new Error("unlink(",arguments[0],") failed: "+e.message,{ - cause: e - }); - } - return false; - } - }; - - /** - Traverses the OPFS filesystem, calling a callback for each - entry. The argument may be either a callback function or an - options object with any of the following properties: - - - `callback`: function which gets called for each filesystem - entry. It gets passed 3 arguments: 1) the - FileSystemFileHandle or FileSystemDirectoryHandle of each - entry (noting that both are instanceof FileSystemHandle). 2) - the FileSystemDirectoryHandle of the parent directory. 3) the - current depth level, with 0 being at the top of the tree - relative to the starting directory. If the callback returns a - literal false, as opposed to any other falsy value, traversal - stops without an error. Any exceptions it throws are - propagated. Results are undefined if the callback manipulate - the filesystem (e.g. removing or adding entries) because the - how OPFS iterators behave in the face of such changes is - undocumented. - - - `recursive` [bool=true]: specifies whether to recurse into - subdirectories or not. Whether recursion is depth-first or - breadth-first is unspecified! - - - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] - specifies the starting directory. - - If this function is passed a function, it is assumed to be the - callback. - - Returns a promise because it has to (by virtue of being async) - but that promise has no specific meaning: the traversal it - performs is synchronous. The promise must be used to catch any - exceptions propagated by the callback, however. - */ - opfsUtil.traverse = async function(opt){ - const defaultOpt = { - recursive: true, - directory: opfsUtil.rootDirectory - }; - if('function'===typeof opt){ - opt = {callback:opt}; - } - opt = Object.assign(defaultOpt, opt||{}); - const doDir = async function callee(dirHandle, depth){ - for await (const handle of dirHandle.values()){ - if(false === opt.callback(handle, dirHandle, depth)) return false; - else if(opt.recursive && 'directory' === handle.kind){ - if(false === await callee(handle, depth + 1)) break; - } - } - }; - doDir(opt.directory, 0); - }; - - /** - impl of importDb() when it's given a function as its second - argument. - */ - const importDbChunked = async function(filename, callback){ - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - let sah = await hFile.createSyncAccessHandle(); - let nWrote = 0, chunk, checkedHeader = false, err = false; - try{ - sah.truncate(0); - while( undefined !== (chunk = await callback()) ){ - if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); - if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ - util.affirmDbHeader(chunk); - checkedHeader = true; - } - sah.write(chunk, {at: nWrote}); - nWrote += chunk.byteLength; - } - if( nWrote < 512 || 0!==nWrote % 512 ){ - toss("Input size",nWrote,"is not correct for an SQLite database."); - } - if( !checkedHeader ){ - const header = new Uint8Array(20); - sah.read( header, {at: 0} ); - util.affirmDbHeader( header ); - } - sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); - return nWrote; - }catch(e){ - await sah.close(); - sah = undefined; - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally { - if( sah ) await sah.close(); - } - }; - - /** - Asynchronously imports the given bytes (a byte array or - ArrayBuffer) into the given database file. - - Results are undefined if the given db name refers to an opened - db. - - If passed a function for its second argument, its behaviour - changes: imports its data in chunks fed to it by the given - callback function. It calls the callback (which may be async) - repeatedly, expecting either a Uint8Array or ArrayBuffer (to - denote new input) or undefined (to denote EOF). For so long as - the callback continues to return non-undefined, it will append - incoming data to the given VFS-hosted database file. When - called this way, the resolved value of the returned Promise is - the number of bytes written to the target file. - - It very specifically requires the input to be an SQLite3 - database and throws if that's not the case. It does so in - order to prevent this function from taking on a larger scope - than it is specifically intended to. i.e. we do not want it to - become a convenience for importing arbitrary files into OPFS. - - This routine rewrites the database header bytes in the output - file (not the input array) to force disabling of WAL mode. - - On error this throws and the state of the input file is - undefined (it depends on where the exception was triggered). - - On success, resolves to the number of bytes written. - */ - opfsUtil.importDb = async function(filename, bytes){ - if( bytes instanceof Function ){ - return importDbChunked(filename, bytes); - } - if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); - util.affirmIsDb(bytes); - const n = bytes.byteLength; - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - let sah, err, nWrote = 0; - try { - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - sah = await hFile.createSyncAccessHandle(); - sah.truncate(0); - nWrote = sah.write(bytes, {at: 0}); - if(nWrote != n){ - toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); - } - sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; - return nWrote; - }catch(e){ - if( sah ){ await sah.close(); sah = undefined; } - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally{ - if( sah ) await sah.close(); - } - }; - + }), function(sqlite3, vfs){ + /* Post-VFS-registration initialization... */ if(sqlite3.oo1){ const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; + opt.vfs = vfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsDb = OpfsDb; OpfsDb.importDb = opfsUtil.importDb; - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( - opfsVfs.pointer, - function(oo1Db, sqlite3){ - /* Set a relatively high default busy-timeout handler to - help OPFS dbs deal with multi-tab/multi-worker - contention. */ - sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); - } - ); - }/*extend sqlite3.oo1*/ - - const sanityCheck = function(){ - const scope = wasm.scopedAllocPush(); - const sq3File = new sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"+randomFilename(8); - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - state.s11n.serialize("This is ä string."); - rc = state.s11n.deserialize(); - log("deserialize() says:",rc); - if("This is ä string."!==rc[0]) toss("String d13n error."); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.sabOPView[xOpen] =", - state.sabOPView[state.opIds.xOpen]); - if(0!==rc){ - error("open failed with code",rc); - return; - } - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.poke(pOut,0,'i64'); - rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.peek(pOut, 'i64')); - rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.poke(readBuf+6,0); - let jRead = wasm.cstrToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - if(vfsSyncWrappers.xSleep){ - log("xSleep()ing before close()ing..."); - vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); - log("waking up from xSleep()"); - } - rc = ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"sabOPView =",state.sabOPView); - log("Deleting file:",dbFile); - vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - warn("End of OPFS sanity checks."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); + if( true ){ + /* 2026-03-06: this was a design mis-decision and is + inconsistent with sqlite3_open() and friends, but is + retained against the risk of introducing regressions if + it's removed. */ + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + /* Set a relatively high default busy-timeout handler to + help OPFS dbs deal with multi-tab/multi-worker + contention. */ + sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); + } + ); } - }/*sanityCheck()*/; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'opfs-unavailable': - /* Async proxy has determined that OPFS is unavailable. There's - nothing more for us to do here. */ - promiseReject(new Error(data.payload.join(' '))); - break; - case 'opfs-async-loaded': - /* Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async - worker. */ - W.postMessage({type: 'opfs-async-init',args: state}); - break; - case 'opfs-async-inited': { - /* Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin... */ - if(true===promiseWasRejected){ - break /* promise was already rejected via timer */; - } - try { - sqlite3.vfs.installVfs({ - io: {struct: opfsIoMethods, methods: ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: vfsSyncWrappers} - }); - state.sabOPView = new Int32Array(state.sabOP); - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - initS11n(); - if(options.sanityChecks){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - if(thisThreadHasOPFS()){ - navigator.storage.getDirectory().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); - promiseResolve(); - }).catch(promiseReject); - }else{ - promiseResolve(); - } - }catch(e){ - error(e); - promiseReject(e); - } - break; - } - default: { - const errMsg = ( - "Unexpected message from the OPFS async worker: " + - JSON.stringify(data) - ); - error(errMsg); - promiseReject(new Error(errMsg)); - break; - } - }/*switch(data.type)*/ - }/*W.onmessage()*/; - })/*thePromise*/; - return thePromise; + }/*extend sqlite3.oo1*/ + })/*bindVfs()*/; }/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = - "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - try{ - let proxyJs = installOpfsVfs.defaultProxyUri; - if( sqlite3?.scriptInfo?.sqlite3Dir ){ - installOpfsVfs.defaultProxyUri = - sqlite3.scriptInfo.sqlite3Dir + proxyJs; - //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); - } - return installOpfsVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); - }); - }catch(e){ - sqlite3.config.error("installOpfsVfs() exception:",e); - return Promise.reject(e); - } + return installOpfsVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install 'opfs' sqlite3_vfs:",e); + }) }); }/*sqlite3ApiBootstrap.initializers.push()*/); -//#else -/* The OPFS VFS parts are elided from builds targeting node.js. */ //#endif target:node diff --git a/ext/wasm/index.html b/ext/wasm/index.html index fd65dbc339..6f22dc150b 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -115,6 +115,11 @@ 64-bit): speedtest1-worker with the OPFS VFS preselected and configured for a moderate workload. +
  • speedtest1-worker?vfs=opfs-wl + (32-bit, + 64-bit): + speedtest1-worker with the + OPFS Web Locks VFS preselected and configured for a moderate workload.
  • speedtest1-worker?vfs=opfs-sahpool (32-bit, 64-bit): diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 37f2967d88..1f60a89bb7 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -246,7 +246,7 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_ALL | F_64BIT + .flags = CP_ALL | F_64BIT | F_NOT_IN_ALL }, /* The canonical esm build. */ @@ -274,7 +274,7 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_JS | F_ESM | F_64BIT + .flags = CP_JS | F_ESM | F_64BIT | F_NOT_IN_ALL }, /* speedtest1, our primary benchmarking tool */ @@ -747,6 +747,12 @@ static void emit_api_js(char const *zBuildName){ zBuildName, zBuildName, zBuildName); pf("$(out.%s.js): $(sqlite3-api.%s.js)\n", zBuildName, zBuildName); + pf("$(sqlite3-api.%s.js):" + /* Extra deps needed by the OPFS pieces... */ + " $(dir.api)/opfs-common-shared.c-pp.js" + " $(dir.api)/opfs-common-inline.c-pp.js" + "\n", + zBuildName); } /* @@ -994,7 +1000,8 @@ static void mk_fiddle(void){ pf("$(out.%s.js): $(MAKEFILE_LIST) " "$(EXPORTED_FUNCTIONS.fiddle) " "$(fiddle.c.in) " - "$(pre-post.%s.deps)", + "$(pre-post.%s.deps) " + "$(dir.dout)/sqlite3-opfs-async-proxy.js", zBuildName, zBuildName); if( isDebug ){ pf(" $(dir.fiddle)/fiddle-worker.js" @@ -1014,10 +1021,6 @@ static void mk_fiddle(void){ pf("\t@$(call b.call.wasm-strip,%s)\n", zBuildName); pf("\t@$(call b.strip-js-emcc-bindings,$(logtag.%s))\n", zBuildName); - pf("\t@$(call b.cp," - "%s," - "$(dir.api)/sqlite3-opfs-async-proxy.js," - "$(dir $@))\n", zBuildName); if( isDebug ){ pf("\t@$(call b.cp,%s," "$(dir.fiddle)/index.html " diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index a1e5b5c344..961c8d8b62 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -66,15 +66,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const haveWasmCTests = ()=>{ return !!wasm.exports.sqlite3__wasm_test_intptr; }; - const hasOpfs = ()=>{ - return globalThis.FileSystemHandle - && globalThis.FileSystemDirectoryHandle - && globalThis.FileSystemFileHandle - && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle - && navigator?.storage?.getDirectory; - }; let SQLite3 /* populated after module load */; + const hasOpfs = ()=>!!SQLite3?.oo1?.OpfsDb; { const mapToString = (v)=>{ @@ -129,17 +123,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }; } } - const reportFinalTestStatus = function(pass){ - if(isUIThread()){ - let e = document.querySelector('#color-target'); - e.classList.add(pass ? 'tests-pass' : 'tests-fail'); - e = document.querySelector('title'); - e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText; - }else{ - postMessage({type:'test-result', payload:{pass}}); - } - TestUtil.checkHeapSize(true); - }; const log = (...args)=>{ //console.log(...args); logClass('',...args); @@ -157,6 +140,19 @@ globalThis.sqlite3InitModule = sqlite3InitModule; console.debug('tester1',...args); }; + const reportFinalTestStatus = function(pass){ + debug("Final test status:",pass); + if(isUIThread()){ + let e = document.querySelector('#color-target'); + e.classList.add(pass ? 'tests-pass' : 'tests-fail'); + e = document.querySelector('title'); + e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText; + }else{ + postMessage({type:'test-result', payload:{pass}}); + } + TestUtil.checkHeapSize(true); + }; + const toss = (...args)=>{ error(...args); throw new Error(args.join(' ')); @@ -455,9 +451,131 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert( !initDb ); tryKey('hexkey', hexFoo, 6); dbUnlink(); - }, + }; //#endif enable-see + /* Tests common to "opfs" and "opfs-wl". These tests manipulate + "this" and must run in order. + */ + T.opfsCommon = { + sanityChecks: async function(vfsName, oo1Ctor, sqlite3){ + T.assert(capi.sqlite3_vfs_find(vfsName)); + const opfs = sqlite3.opfs; + const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db'; + const fileUri = 'file://'+filename+'?delete-before-open=1'; + const initSql = [ + 'create table p(a);', + 'insert into p(a) values(1),(2),(3)' + ]; + let db = new oo1Ctor(fileUri); + try { + db.exec(initSql); + T.assert(3 === db.selectValue('select count(*) from p')); + db.close(); + db = new oo1Ctor(filename); + db.exec('insert into p(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from p')); + this.opfsDbExport = capi.sqlite3_js_db_export(db); + T.assert(this.opfsDbExport instanceof Uint8Array) + .assert(this.opfsDbExport.byteLength>0 + && 0===this.opfsDbExport.byteLength % 512); + }finally{ + db.close(); + db = null; + } + T.assert(await opfs.entryExists(filename)); + try { + db = new oo1Ctor(fileUri); + db.exec(initSql) /* will throw if delete-before-open did not work */; + T.assert(3 === db.selectValue('select count(*) from p')); + }finally{ + if(db) db.close(); + } + }, + + importer: async function(vfsName, oo1Ctor, sqlite3){ + let db; + const filename = this.opfsDbFile; + try { + const exp = this.opfsDbExport; + delete this.opfsDbExport; + this.opfsImportSize = await oo1Ctor.importDb(filename, exp); + db = new oo1Ctor(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')). + assert( this.opfsImportSize == exp.byteLength ); + db.close(); + db = null; + this.opfsUnlink = + (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink(vfsName, fn); + this.opfsUnlink(filename); + T.assert(!(await sqlite3.opfs.entryExists(filename))); + // Try again with a function as an input source: + let cursor = 0; + const blockSize = 512, end = exp.byteLength; + const reader = async function(){ + if(cursor >= exp.byteLength){ + return undefined; + } + const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize); + cursor += blockSize; + return rv; + }; + this.opfsImportSize = await oo1Ctor.importDb(filename, reader); + db = new oo1Ctor(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')). + assert( this.opfsImportSize == exp.byteLength ); + }finally{ + if(db) db.close(); + } + }, + + opfsUtil: async function(vfsName, oo1Ctor, sqlite3){ + const filename = this.opfsDbFile; + const unlink = this.opfsUnlink; + T.assert(filename && !!unlink); + delete this.opfsDbFile; + delete this.opfsUnlink; + /************************************************************** + ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended + for client-side use. It is only for this project's own + internal use. Its APIs are subject to change or removal at + any time. The sqlite3.opfs namespace is REMOVED from the + sqlite3 namespace in non-test runs of the library. + ***************************************************************/ + const opfs = sqlite3.opfs; + const fSize = this.opfsImportSize; + delete this.opfsImportSize; + let sh; + try{ + T.assert(await opfs.entryExists(filename)); + const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false); + const fh = await dirHandle.getFileHandle(filenamePart); + sh = await fh.createSyncAccessHandle(); + T.assert(fSize === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + T.assert(!(await opfs.entryExists(filename))); + }finally{ + if(sh) await sh.close(); + unlink(); + } + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } + }/*T.opfsCommon*/; + //////////////////////////////////////////////////////////////////////// // End of infrastructure setup. Now define the tests... //////////////////////////////////////////////////////////////////////// @@ -3481,7 +3599,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; test: function(sqlite3){ const JDb = sqlite3.oo1.JsStorageDb; T.seeBaseCheck(JDb, - (isInit)=>return {filename: "session"}, + (isInit)=>{ + return {filename: "session"}; + }, ()=>JDb.clearStorage('session')); } })/*kvvfs with SEE*/ @@ -3771,131 +3891,29 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('OPFS: Origin-Private File System', - (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs") - || 'requires "opfs" VFS')) + (sqlite3)=>(!!sqlite3.oo1.OpfsDb || 'requires "opfs" VFS')) .t({ name: 'OPFS db sanity checks', - test: async function(sqlite3){ - T.assert(capi.sqlite3_vfs_find('opfs')); - const opfs = sqlite3.opfs; - const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db'; - const fileUri = 'file://'+filename+'?delete-before-open=1'; - const initSql = [ - 'create table p(a);', - 'insert into p(a) values(1),(2),(3)' - ]; - let db = new sqlite3.oo1.OpfsDb(fileUri); - try { - db.exec(initSql); - T.assert(3 === db.selectValue('select count(*) from p')); - db.close(); - db = new sqlite3.oo1.OpfsDb(filename); - db.exec('insert into p(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from p')); - this.opfsDbExport = capi.sqlite3_js_db_export(db); - T.assert(this.opfsDbExport instanceof Uint8Array) - .assert(this.opfsDbExport.byteLength>0 - && 0===this.opfsDbExport.byteLength % 512); - }finally{ - db.close(); - } - T.assert(await opfs.entryExists(filename)); - try { - db = new sqlite3.oo1.OpfsDb(fileUri); - db.exec(initSql) /* will throw if delete-before-open did not work */; - T.assert(3 === db.selectValue('select count(*) from p')); - }finally{ - if(db) db.close(); - } + test: async (sqlite3)=>{ + await T.opfsCommon.sanityChecks('opfs', sqlite3.oo1.OpfsDb, sqlite3); } - }/*OPFS db sanity checks*/) + }) .t({ - name: 'OPFS import', - test: async function(sqlite3){ - let db; - const filename = this.opfsDbFile; - try { - const exp = this.opfsDbExport; - delete this.opfsDbExport; - this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp); - db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); - T.assert(6 === db.selectValue('select count(*) from p')). - assert( this.opfsImportSize == exp.byteLength ); - db.close(); - this.opfsUnlink = - (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink("opfs",fn); - this.opfsUnlink(filename); - T.assert(!(await sqlite3.opfs.entryExists(filename))); - // Try again with a function as an input source: - let cursor = 0; - const blockSize = 512, end = exp.byteLength; - const reader = async function(){ - if(cursor >= exp.byteLength){ - return undefined; - } - const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize); - cursor += blockSize; - return rv; - }; - this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader); - db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); - T.assert(6 === db.selectValue('select count(*) from p')). - assert( this.opfsImportSize == exp.byteLength ); - }finally{ - if(db) db.close(); - } + name: 'OPFS import/export', + test: async (sqlite3)=>{ + await T.opfsCommon.importer('opfs', sqlite3.oo1.OpfsDb, sqlite3); } - }/*OPFS export/import*/) + }) .t({ name: '(Internal-use) OPFS utility APIs', - test: async function(sqlite3){ - const filename = this.opfsDbFile; - const unlink = this.opfsUnlink; - T.assert(filename && !!unlink); - delete this.opfsDbFile; - delete this.opfsUnlink; - /************************************************************** - ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended - for client-side use. It is only for this project's own - internal use. Its APIs are subject to change or removal at - any time. - ***************************************************************/ - const opfs = sqlite3.opfs; - const fSize = this.opfsImportSize; - delete this.opfsImportSize; - let sh; - try{ - T.assert(await opfs.entryExists(filename)); - const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false); - const fh = await dirHandle.getFileHandle(filenamePart); - sh = await fh.createSyncAccessHandle(); - T.assert(fSize === await sh.getSize()); - await sh.close(); - sh = undefined; - unlink(); - T.assert(!(await opfs.entryExists(filename))); - }finally{ - if(sh) await sh.close(); - unlink(); - } - - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); + test: async (sqlite3)=>{ + await T.opfsCommon.opfsUtil("opfs", sqlite3.oo1.OpfsDb, sqlite3); } - }/*OPFS util sanity checks*/) + }) //#if enable-see .t({ name: 'OPFS with SEE encryption', + predicate: (sqlite3)=>!!sqlite3.oo1.OpfsDb, test: function(sqlite3){ T.seeBaseCheck( sqlite3.oo1.OpfsDb, @@ -3904,15 +3922,50 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if( isInit ) opt.filename += '?delete-before-open=1'; return opt; }, - ()=>{}); + ()=>{} + ); } - })/*OPFS with SEE*/ + }) +//#endif enable-see + ;/* end OPFS tests */ + + //////////////////////////////////////////////////////////////////////// + T.g('OPFS-WL: Origin-Private File System with Web Locks', + (sqlite3)=>(!!sqlite3.oo1.OpfsWlDb || 'requires "opfs-wl" VFS')) + .t({ + name: 'OPFS-WL db sanity checks', + test: async (sqlite3)=>{ + await T.opfsCommon.sanityChecks('opfs-wl', sqlite3.oo1.OpfsWlDb, sqlite3); + } + }) + .t({ + name: 'OPFS-WL import/export', + test: async (sqlite3)=>{ + await T.opfsCommon.importer('opfs-wl', sqlite3.oo1.OpfsWlDb, sqlite3); + } + }) +//#if enable-see + .t({ + name: 'OPFS-WL with SEE encryption', + predicate: (sqlite3)=>!!sqlite3.oo1.OpfsWlDb, + test: function(sqlite3){ + T.seeBaseCheck( + sqlite3.oo1.OpfsWlDb, + function(isInit){ + const opt = {filename: 'file:///sqlite3-see.edb'}; + if( isInit ) opt.filename += '?delete-before-open=1'; + return opt; + }, + ()=>{} + ); + } + }) //#endif enable-see ;/* end OPFS tests */ //////////////////////////////////////////////////////////////////////// T.g('OPFS SyncAccessHandle Pool VFS', - (sqlite3)=>(hasOpfs() || "requires OPFS APIs")) + (sqlite3)=>(!!sqlite3.installOpfsSAHPoolVfs || "requires OPFS SAH Pool APIs")) .t({ name: 'SAH sanity checks', test: async function(sqlite3){ @@ -4438,12 +4491,17 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// log("Loading and initializing sqlite3 WASM module..."); - if(0){ + if(1){ globalThis.sqlite3ApiConfig = { - debug: ()=>{}, - log: ()=>{}, - warn: ()=>{}, - error: ()=>{} + //debug: ()=>{}, log: ()=>{}, warn: ()=>{}, error: ()=>{}, + disable: { + vfs: { + kvvfs: false, + opfs: false, + "opfs-sahpool": false, + "opfs-wl": false + } + } } } //#if not target:es6-module diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index 54ed04a4f6..c5e929b010 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -25,15 +25,14 @@ the workers=N URL flag. Set the time between each workload with interval=N (milliseconds). Set the number of worker iterations with iterations=N. - Enable OPFS VFS verbosity with verbose=1-3 (output - goes to the dev console). Disable/enable "unlock ASAP" mode - (higher concurrency, lower speed) with unlock-asap=0-1. + Disable/enable "unlock ASAP" mode (potentially higher + concurrency but lower speed) with unlock-asap=0-1.

    Achtung: if it does not start to do anything within a couple of seconds, check the dev console: Chrome sometimes fails to load - the wasm module due to "cannot allocate WasmMemory." Closing and - re-opening the tab usually resolves it, but sometimes restarting - the browser is required. + the wasm module due to "cannot allocate WasmMemory" when + reloading the page. Closing and re-opening the tab usually + resolves it, but sometimes restarting the browser is required.

    Links for various testing options:

    diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 1848901afe..a73bc2e47d 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -8,6 +8,7 @@ return ''+v; default: break; } + if( v instanceof Date ) return v.toString(); if(null===v) return 'null'; if(v instanceof Error){ v = { @@ -55,11 +56,11 @@ const options = Object.create(null); options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); options.workerCount = ( - urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 - ) || 4; + urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 0 + ) || 3; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 - ) || 1; + ) || 0; options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 1000 ) || 1000; @@ -69,47 +70,60 @@ options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; + options.vfs = urlArgsHtml.get('vfs') || 'opfs'; options.noUnlink = !!urlArgsHtml.has('no-unlink'); const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); }; workers.counts = {loaded: 0, passed: 0, failed: 0}; + let timeStart; + const calcTime = (endDate)=>{ + return endDate.getTime() - timeStart?.getTime?.(); + }; const checkFinished = function(){ if(workers.counts.passed + workers.counts.failed !== workers.length){ return; } if(workers.counts.failed>0){ - logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s) in", + calcTime(new Date()),"ms"); }else{ - logCss('tests-pass',"All",workers.length,"workers finished."); + logCss('tests-pass',"All",workers.length,"workers finished in", + calcTime(new Date()),"ms"); } }; + workers.onmessage = function(msg){ msg = msg.data; const prefix = 'Worker #'+msg.worker+':'; switch(msg.type){ - case 'loaded': - stdout(prefix,"loaded"); - if(++workers.counts.loaded === workers.length){ - stdout("All",workers.length,"workers loaded. Telling them to run..."); - workers.post('run'); - } - break; - case 'stdout': stdout(prefix,...msg.payload); break; - case 'stderr': stderr(prefix,...msg.payload); break; - case 'error': stderr(prefix,"ERROR:",...msg.payload); break; - case 'finished': - ++workers.counts.passed; - logCss('tests-pass',prefix,...msg.payload); - checkFinished(); - break; - case 'failed': - ++workers.counts.failed; - logCss('tests-fail',prefix,"FAILED:",...msg.payload); - checkFinished(); - break; - default: logCss('error',"Unhandled message type:",msg); break; + case 'loaded': + stdout(prefix,"loaded"); + if(++workers.counts.loaded === workers.length){ + timeStart = new Date(); + stdout(timeStart,"All",workers.length,"workers loaded. Telling them to run..."); + workers.post('run'); + } + break; + case 'stdout': stdout(prefix,...msg.payload); break; + case 'stderr': stderr(prefix,...msg.payload); break; + case 'error': stderr(prefix,"ERROR:",...msg.payload); break; + case 'finished': { + ++workers.counts.passed; + const timeEnd = new Date(); + logCss('tests-pass',prefix,timeEnd,"("+calcTime(timeEnd)+"ms) ", ...msg.payload); + checkFinished(); + break; + } + case 'failed': { + ++workers.counts.failed; + const timeEnd = new Date(); + logCss('tests-fail',prefix,timeEnd,"("+calcTime(timeEnd)+"ms) FAILED:",...msg.payload); + checkFinished(); + break; + } + default: logCss('error',"Unhandled message type:",msg); break; } }; @@ -118,7 +132,10 @@ const eTestLinks = document.querySelector('#testlinks'); const optArgs = function(obj){ const li = []; - for(const k of ['interval','iterations','workers','verbose','unlock-asap']){ + for(const k of [ + 'interval', 'iterations', 'unlock-asap', + 'opfsVerbose', 'vfs', 'workers' + ]){ if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]); } return li.join('&'); @@ -127,7 +144,10 @@ {interval: 1000, workers: 5, iterations: 30}, {interval: 500, workers: 5, iterations: 30}, {interval: 250, workers: 3, iterations: 30}, - {interval: 600, workers: 5, iterations: 100} + {interval: 600, workers: 5, iterations: 100}, + {interval: 500, workers: 1, iterations: 5, vfs: 'opfs-wl'}, + {interval: 500, workers: 2, iterations: 10, vfs: 'opfs-wl'}, + {interval: 500, workers: 5, iterations: 20, vfs: 'opfs-wl'}, ]){ const li = document.createElement('li'); eTestLinks.appendChild(li); @@ -142,13 +162,14 @@ workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + + '&vfs='+options.vfs + '&interval='+options.interval + '&iterations='+options.iterations + '&opfs-verbose='+options.opfsVerbose + '&opfs-unlock-asap='+options.unlockAsap ); for(let i = 0; i < options.workerCount; ++i){ - stdout("Launching worker..."); + stdout("Launching worker...", i, ); workers.push(new Worker( workers.uri+'&workerId='+(i+1)+( (i || options.noUnlink) ? '' : '&unlink-db' @@ -158,4 +179,4 @@ // Have to delay onmessage assignment until after the loop // to avoid that early workers get an undue head start. workers.forEach((w)=>w.onmessage = workers.onmessage); -})(self); +})(globalThis); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 5d28bedee0..fb603cf795 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -1,19 +1,24 @@ -importScripts( - (new URL(self.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' -); -self.sqlite3InitModule().then(async function(sqlite3){ - const urlArgs = new URL(self.location.href).searchParams; - const options = { - workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), - unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ - }; +'use strict'; +const urlArgs = new URL(globalThis.location.href).searchParams; +const options = { + workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0, + vfs: urlArgs.get('vfs') +}; +const jsSqlite = urlArgs.get('sqlite3.dir') + +'/sqlite3.js?opfs-async-proxy-id=' + +options.workerName; +importScripts(jsSqlite)/*Sigh - URL args are not propagated this way*/; +//const sqlite3InitModule = (await import(jsSqlite)).default; +globalThis.sqlite3InitModule.__isUnderTest = true; +globalThis.sqlite3InitModule().then(async function(sqlite3){ const wPost = (type,...payload)=>{ postMessage({type, worker: options.workerName, payload}); }; const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); if(!sqlite3.opfs){ - stderr("OPFS support not detected. Aborting."); + stderr("This code requires the (private) sqlite3.opfs object. Aborting."); return; } @@ -34,80 +39,107 @@ self.sqlite3InitModule().then(async function(sqlite3){ count: 0 }); const finish = ()=>{ - if(db){ - if(!db.pointer) return; + if(db?.pointer){ db.close(); } if(interval.error){ - wPost('failed',"Ending work after interval #"+interval.count, + stderr("Ending work at interval #"+interval.count, + "due to error:", interval.error); + wPost('failed', "at interval #"+interval.count, "due to error:",interval.error); }else{ wPost('finished',"Ending work after",interval.count,"intervals."); } }; const run = async function(){ - db = new sqlite3.oo1.OpfsDb({ - filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c' + const Ctors = Object.assign(Object.create(null),{ + opfs: sqlite3.oo1.OpfsDb, + 'opfs-wl': sqlite3.oo1.OpfsWlDb }); + const ctor = Ctors[options.vfs]; + if( !ctor ){ + stderr("Invalid VFS name:",vfs); + return; + } + stdout("ctor:", options.vfs); + while(true){ + try{ + db = new ctor({ + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, + flags: 'c' + }); + break; + }catch(e){ + if(e instanceof sqlite3.SQLite3Error + && sqlite3.capi.SQLITE_BUSY===e.resultCode){ + stderr("Retrying open() for BUSY:",e.message); + continue; + } + throw e; + } + } sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); - db.transaction((db)=>{ - db.exec([ - "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", - "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" - ]); + sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{ + db.transaction((db)=>{ + db.exec([ + "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", + "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" + ]); + }); }); const maxIterations = urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; - stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); + stdout("Starting",maxIterations,"interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); ++interval.count; const prefix = "v(#"+interval.count+")"; stdout("Setting",prefix,"=",tm); try{ - db.exec({ - sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", - bind: [options.workerName, new Date().getTime()] + sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{ + db.exec({ + sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", + bind: [options.workerName, new Date().getTime()] + }); + }, (n)=>{ + stderr("Attempt #",n,"for interval #",interval.count,"due to BUSY"); }); - //stdout("Set",prefix); }catch(e){ + stderr("Error: ",e.message); interval.error = e; + throw e; } + //stdout("doWork()",prefix,"error ",interval.error); }; - if(1){/*use setInterval()*/ - setTimeout(async function timer(){ - await doWork(); - if(interval.error || maxIterations === interval.count){ - finish(); - }else{ - setTimeout(timer, interval.delay); - } - }, interval.delay); - }else{ - /*This approach provides no concurrency whatsoever: each worker - is run to completion before any others can work.*/ - let i; - for(i = 0; i < maxIterations; ++i){ - await doWork(); - if(interval.error) break; - await wait(interval.ms); + setTimeout(async function timer(){ + await doWork(); + if(interval.error || maxIterations === interval.count){ + finish(); + }else{ + const jitter = (Math.random()>=0.5) + ? ((Math.random()*100 - Math.random()*100) | 0) + : 0; + const n = interval.delay + jitter; + setTimeout(timer, n<50 ? interval.delay : n); } - finish(); - } + }, interval.delay); }/*run()*/; - self.onmessage = function({data}){ + globalThis.onmessage = function({data}){ switch(data.type){ - case 'run': run().catch((e)=>{ + case 'run': + run().catch((e)=>{ if(!interval.error) interval.error = e; + }).catch(e=>{ + /* Don't do this in finally() - it fires too soon. */ finish(); + throw e; }); - break; - default: - stderr("Unhandled message type '"+data.type+"'."); - break; + break; + default: + stderr("Unhandled message type '"+data.type+"'."); + break; } }; }); diff --git a/manifest b/manifest index 9b6f4841be..7f650c37c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sparens\saround\san\sexpression\sin\sthe\sCLI\sshell\sto\saccount\sfor\soperator\sprecedence,\sas\sreported\sin\s[forum:856ff7a2f1|forum\spost\s856ff7a2f1]. -D 2026-03-08T07:02:46.194 +C Merge\sopfs-wl\sinto\strunk,\snot\sso\smuch\sfor\sthe\s"opfs-wl"\sVFS,\swhich\sis\sstil\stentative\s(working\sfine\sbut\swith\slittle\sperceived\sbenefit),\sbut\sfor\sthe\s"opfs"\sVFS\scleanups\swhich\snecessarily\shappened\saround\sit. +D 2026-03-08T08:32:22.078 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -572,7 +572,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6 -F ext/wasm/GNUmakefile 79236447d750609aa6beda30feec1314180c5462a493ad94214122887232bfd4 +F ext/wasm/GNUmakefile 4496669ca5785c7bfc7af76565f1dd45e8f9ec7529b58c485fd374088162bef1 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -584,19 +584,22 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 6ad5ace0a16b3300e8d40d4e596eee1c773424e22 F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 +F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 +F ext/wasm/api/opfs-common-shared.c-pp.js 49f4b044dfb3becd8c926ef42b7494da680e5ac878c6d10ef510b66f74ad55a1 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b -F ext/wasm/api/post-js-header.js d24bd0d065f3489c8b78ddf3ead6321e5d047187a162cd503c41700e03dd1f06 +F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f F ext/wasm/api/sqlite3-api-glue.c-pp.js 7afb3da3510facafd94ce31133ec847d0d4db5b2b5e4325941803fd3bca07c16 F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa -F ext/wasm/api/sqlite3-api-prologue.js 2eac3d601c8a824fda1529f9eb894518672d3e575b49c0601094afda442b294f +F ext/wasm/api/sqlite3-api-prologue.js db1ba33ca7092f12d5bb6cb615c78c457432be4a1d26785686993a8909950d72 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.js 92d6d327a862f1627ff3e88e60fdfea9def06ad539d98929ba46490e64372736 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js c2236c9037ccac368be6eeed3e5f6ed3cefed0d20539c9092139f27a76aaf1ba w ext/wasm/api/sqlite3-opfs-async-proxy.js F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js f907fa712d1c9d98a3c08393f3e97a664857782da11670e129d39e383f45f968 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js cf5fba74978d3e0983f68505d9695ee7836853855590a92ccc5a96e27db5350b +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 743650cfefed0df900171971065c2d618e5652177a98844ddb6695c70149968e +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4b02d7063d00e2d4f42d4ca79a8a0bca1b275257e135bc154d35b701ed1d234a F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c ddf9d435b2e901eaceb805ff694e9609c2f32b5cf89a4f164e734a6fa303fdd2 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -621,11 +624,11 @@ F ext/wasm/fiddle/fiddle-worker.js 6c72acac2d381480bc9f5eb538e3f2faf2c1f72dd4fcb F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893 F ext/wasm/fiddle/index.c-pp.html 72c7e5517217960b3809648429ea396a7cbad0ffb2c92f6a2f5703abecb27317 F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290 -F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb5f1756 +F ext/wasm/index.html 5bf6cf1b0a3c8b9f5f54d77f2219d7ae87a15162055ce308109c49b1dcab4239 F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x -F ext/wasm/mkwasmbuilds.c 0e9198eb90acae4bcf57cf62d7186f6af5aaac02efdb075a1aded33614b3805a +F ext/wasm/mkwasmbuilds.c d4479af2774a43104b819d9286961b7c09793ca39440e20191e693ceff21f911 F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -641,10 +644,10 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js 1c81b2b3f6c02adbb50ff48e07645946cd066cd215c0b8b500574fefecacf5c8 -F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e -F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 -F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 +F ext/wasm/tester1.c-pp.js e26e78fe450375b43a88a5da41946b8997612201a592770f19ec827b2bf96cc0 +F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226 +F ext/wasm/tests/opfs/concurrency/test.js 46c772bc18abb0fcbb058d57b5aaee9e7938f948ecdd802c6ca0850ad3519f92 +F ext/wasm/tests/opfs/concurrency/worker.js 4edeec8d88f14c9603c9d4e32ce379ce1d5394f3c5e820776e26a603d600b3e3 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2189,8 +2192,9 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 0c5d4fe0bd322b4c8dbd0800001317a21cd511b00244de74b4db3fd051153879 -R d722ffeb4d76b155994400ff67e06b8f +P da3ad170aebf57f2202e56673ac149bc51077d126ab5928f1db16492e403d5b9 616d32c8380ef47c5a7c2eb765e7b6fbec9500d4e16c837f4ee11d71c239a55f +R 7fa9d9bc620afcdc086430a72b380676 +T +closed 616d32c8380ef47c5a7c2eb765e7b6fbec9500d4e16c837f4ee11d71c239a55f Closed\sby\sintegrate-merge. U stephan -Z f680be3a93bb03be191e09867e313221 +Z eb8fc8dc10da8c389057ea48d94a3dd6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 56e3347d8a..1542198500 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da3ad170aebf57f2202e56673ac149bc51077d126ab5928f1db16492e403d5b9 +bcaa876c6dafee0995d239f59b6ce7810b6f5faa219b9c392966211ae3dec17a