From 8c6bf948b79e04fbc17daba8697138df6a8e81ed Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 27 Feb 2026 03:11:40 +0000 Subject: [PATCH 01/32] An initial copy/paste stub for experimenting with using WebLocks for a new copy of the OPFS vfs. This is not yet functional, or even loading, but needs stashing to avoid potential loss. FossilOrigin-Name: 372f18ae909bafd7167b70051935832e3107ede72aaeccbf5c042db7ba791912 --- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 1471 ++++++++++++++++++++++ manifest | 16 +- manifest.tags | 4 +- manifest.uuid | 2 +- 4 files changed, 1484 insertions(+), 9 deletions(-) create mode 100644 ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js new file mode 100644 index 0000000000..9fe09269cf --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -0,0 +1,1471 @@ +//#if not target:node +/* + 2026-02-20 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is a placeholder for a reimplementation of the "opfs" VFS + (as distinct from "opfs-sahpool") which uses WebLocks instead + locking based on a bespoke custom Atomics.wait()/notify() + protocol. This file holds the "synchronous half" of the VFS, whereas + it shares the "asynchronous half" of the "opfs" VFS. + + This file is intended to be appended to the main sqlite3 JS + deliverable somewhere after sqlite3-api-oo1.js. +*/ +'use strict'; +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +/** + installOpfsVfs() returns a Promise which, on success, installs an + sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs + which accept a VFS. It is intended to be called via + sqlite3ApiBootstrap.initializers or an equivalent mechanism. + + The installed VFS uses the Origin-Private FileSystem API for + all file storage. On error it is rejected with an exception + explaining the problem. Reasons for rejection include, but are + not limited to: + + - The counterpart Worker (see below) could not be loaded. + + - The environment does not support OPFS. That includes when + this function is called from the main window thread. + + Significant notes and limitations: + + - The OPFS features used here are only available in dedicated Worker + threads. This file tries to detect that case, resulting in a + rejected Promise if those features do not seem to be available. + + - It requires the SharedArrayBuffer and Atomics classes, and the + former is only available if the HTTP server emits the so-called + COOP and COEP response headers. These features are required for + proxying OPFS's synchronous API via the synchronous interface + required by the sqlite3_vfs API. + + - This function may only be called a single time. When called, this + function removes itself from the sqlite3 object. + + All arguments to this function are for internal/development purposes + only. They do not constitute a public API and may change at any + time. + + The argument may optionally be a plain object with the following + configuration options: + + - proxyUri: name of the async proxy JS file. + + - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables + logging of errors. 2 enables logging of warnings and errors. 3 + additionally enables debugging info. Logging is performed + via the sqlite3.config.{log|warn|error}() functions. + + - sanityChecks (=false): if true, some basic sanity tests are run on + the OPFS VFS API after it's initialized, before the returned + Promise resolves. This is only intended for testing and + development of the VFS, not client-side use. + + On success, the Promise resolves to the top-most sqlite3 namespace + object and that object gets a new object installed in its + `opfs` property, containing several OPFS-specific utilities. +*/ +const installOpfsVfs = function callee(options){ + if(!globalThis.SharedArrayBuffer + || !globalThis.Atomics){ + return Promise.reject( + new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ + "The server must emit the COOP/COEP response headers to enable those. "+ + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") + ); + }else if('undefined'===typeof WorkerGlobalScope){ + return Promise.reject( + new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ + "because it requires Atomics.wait().") + ); + }else if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ + return Promise.reject( + new Error("Missing required OPFS APIs.") + ); + } + const nu = (...obj)=>Object.assign(Object.create(null),...obj); + options = nu(options); + const urlParams = new URL(globalThis.location.href).searchParams; + if(urlParams.has('opfs-disable')){ + //sqlite3.config.warn('Explicitly not installing 'opfs-wl' VFS due to opfs-disable flag.'); + return Promise.resolve(sqlite3); + } + if(undefined===options.verbose){ + options.verbose = urlParams.has('opfs-verbose') + ? (+urlParams.get('opfs-verbose') || 2) : 1; + } + if(undefined===options.sanityChecks){ + options.sanityChecks = urlParams.has('opfs-sanity-check'); + } + if(undefined===options.proxyUri){ + options.proxyUri = callee.defaultProxyUri; + } + + //sqlite3.config.warn("OPFS options =",options,globalThis.location); + + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } + const thePromise = new Promise(function(promiseResolve_, promiseReject_){ + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const toss = sqlite3.util.toss; + const capi = sqlite3.capi; + const util = sqlite3.util; + const wasm = sqlite3.wasm; + const sqlite3_vfs = capi.sqlite3_vfs; + const sqlite3_file = capi.sqlite3_file; + const sqlite3_io_methods = capi.sqlite3_io_methods; + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + + ACHTUNG: do not rely on these APIs in client code. They are + experimental and subject to change or removal as the + OPFS-specific sqlite3_vfs evolves. + */ + const opfsUtil = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + const thisThreadHasOPFS = ()=>{ + return globalThis.FileSystemHandle && + globalThis.FileSystemDirectoryHandle && + globalThis.FileSystemFileHandle && + globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator?.storage?.getDirectory; + }; + + /** + Not part of the public API. Solely for internal/development + use. + */ + opfsUtil.metrics = { + dump: function(){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; + } + sqlite3.config.log(globalThis.location.href, + "metrics for",globalThis.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); + sqlite3.config.log("Serialization metrics:",metrics.s11n); + W.postMessage({type:'opfs-async-metrics'}); + }, + reset: function(){ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); + } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + } + }/*metrics*/; + const opfsIoMethods = new sqlite3_io_methods(); + const opfsVfs = new sqlite3_vfs() + .addOnDispose( ()=>opfsIoMethods.dispose()); + let promiseWasRejected = undefined; + const promiseReject = (err)=>{ + promiseWasRejected = true; + opfsVfs.dispose(); + return promiseReject_(err); + }; + const promiseResolve = ()=>{ + promiseWasRejected = false; + return promiseResolve_(sqlite3); + }; + const W = +//#if target:es6-bundler-friendly + new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url)); +//#elif target:es6-module + new Worker(new URL(options.proxyUri, import.meta.url)); +//#else + new Worker(options.proxyUri); +//#endif + setTimeout(()=>{ + /* At attempt to work around a browser-specific quirk in which + the Worker load is failing in such a way that we neither + resolve nor reject it. This workaround gives that resolve/reject + a time limit and rejects if that timer expires. Discussion: + https://sqlite.org/forum/forumpost/a708c98dcb3ef */ + if(undefined===promiseWasRejected){ + promiseReject( + new Error("Timeout while waiting for OPFS async proxy worker.") + ); + } + }, 4000); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit + is undocumented/unspecified. */; + opfsVfs.$zName = wasm.allocCString('opfs-wl'); + // All C-side memory of opfsVfs is zeroed out, but just to be explicit: + opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) + ); + /** + Pedantic sidebar about opfsVfs.ondispose: the entries in that array + are items to clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the opfsVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + /** + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. + + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. + + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. + */ + const state = Object.create(null); + state.verbose = options.verbose; + state.littleEndian = (()=>{ + const buffer = new ArrayBuffer(2); + new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); + // Int16Array uses the platform's endianness. + return new Int16Array(buffer)[0] === 256; + })(); + /** + asyncIdleWaitTime is how long (ms) to wait, in the async proxy, + for each Atomics.wait() when waiting on inbound VFS API calls. + We need to wake up periodically to give the thread a chance to + do other things. If this is too high (e.g. 500ms) then even two + workers/tabs can easily run into locking errors. Some multiple + of this value is also used for determining how long to wait on + lock contention to free up. + */ + state.asyncIdleWaitTime = 150; + + /** + Whether the async counterpart should log exceptions to + the serialization channel. That produces a great deal of + noise for seemingly innocuous things like xAccess() checks + for missing files, so this option may have one of 3 values: + + 0 = no exception logging. + + 1 = only log exceptions for "significant" ops like xOpen(), + xRead(), and xWrite(). + + 2 = log all exceptions. + */ + state.asyncS11nExceptions = 1; + /* Size of file I/O buffer block. 64k = max sqlite3 page size, and + xRead/xWrite() will never deal in blocks larger than that. */ + state.fileBufferSize = 1024 * 64; + state.sabS11nOffset = state.fileBufferSize; + /** + The size of the block in our SAB for serializing arguments and + result values. Needs to be large enough to hold serialized + values of any of the proxied APIs. Filenames are the largest + part but are limited to opfsVfs.$mxPathname bytes. We also + store exceptions there, so it needs to be long enough to hold + a reasonably long exception string. + */ + state.sabS11nSize = opfsVfs.$mxPathname * 2; + /** + The SAB used for all data I/O between the synchronous and + async halves (file i/o and arg/result s11n). + */ + state.sabIO = new SharedArrayBuffer( + state.fileBufferSize/* file i/o block */ + + state.sabS11nSize/* argument/result serialization block */ + ); + state.opIds = Object.create(null); + const metrics = Object.create(null); + { + /* Indexes for use in our SharedArrayBuffer... */ + let i = 0; + /* SAB slot used to communicate which operation is desired + between both workers. This worker writes to it and the other + listens for changes. */ + state.opIds.whichOp = i++; + /* Slot for storing return values. This worker listens to that + slot and the other worker writes to it. */ + state.opIds.rc = i++; + /* Each function gets an ID which this worker writes to + the whichOp slot. The async-api worker uses Atomic.wait() + on the whichOp slot to figure out which operation to run + next. */ + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xDeleteNoWait = i++; + state.opIds.xFileSize = i++; + state.opIds.xLock = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xUnlock = i++; + state.opIds.xWrite = i++; + state.opIds.mkdir = i++; + state.opIds.lockControl = i++ /* we signal the intent to lock here */; + state.opIds.lockType + /** Internal signals which are used only during development and + testing via the dev console. */ + state.opIds['opfs-async-metrics'] = i++; + state.opIds['opfs-async-shutdown'] = i++; + /* The retry slot is used by the async part for wait-and-retry + semantics. Though we could hypothetically use the xSleep slot + for that, doing so might lead to undesired side effects. */ + state.opIds.retry = i++; + + /* Slots for submitting the lock type and receiving its acknowledgement. */ + state.lock = nu({ + type: i++, + atomicsHandshake: i++ + }); + state.sabOP = new SharedArrayBuffer( + i * 4/* ==sizeof int32, noting that Atomics.wait() and friends + can only function on Int32Array views of an SAB. */); + opfsUtil.metrics.reset(); + } + /** + SQLITE_xxx constants to export to the async worker + counterpart... + */ + state.sq3Codes = Object.create(null); + [ + 'SQLITE_ACCESS_EXISTS', + 'SQLITE_ACCESS_READWRITE', + 'SQLITE_BUSY', + 'SQLITE_CANTOPEN', + 'SQLITE_ERROR', + 'SQLITE_IOERR', + 'SQLITE_IOERR_ACCESS', + 'SQLITE_IOERR_CLOSE', + 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_LOCK', + 'SQLITE_IOERR_READ', + 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_TRUNCATE', + 'SQLITE_IOERR_UNLOCK', + 'SQLITE_IOERR_WRITE', + 'SQLITE_LOCK_EXCLUSIVE', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCKED', + 'SQLITE_MISUSE', + 'SQLITE_NOTFOUND', + 'SQLITE_OPEN_CREATE', + 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', + 'SQLITE_OPEN_READONLY' + ].forEach((k)=>{ + if(undefined === (state.sq3Codes[k] = capi[k])){ + toss("Maintenance required: not found:",k); + } + }); + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" + enables this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + Flag for use with xOpen(). URI flag "delete-before-open=1" + tells the VFS to delete the db file before attempting to open + it. This can be used, e.g., to replace a db which has been + corrupted (without forcing us to expose a delete/unlink() + function in the public API). + + Failure to unlink the file is ignored but may lead to + downstream errors. An unlink can fail if, e.g., another tab + has the handle open. + + It goes without saying that deleting a file out from under another + instance results in Undefined Behavior. + */ + OPFS_UNLINK_BEFORE_OPEN: 0x02, + /** + If true, any async routine which implicitly acquires a sync + access handle (i.e. an OPFS lock) will release that lock at + the end of the call which acquires it. If false, such + "autolocks" are not released until the VFS is idle for some + brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). + */ + defaultUnlockAsap: false + }); + + /** + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The + 2nd and subsequent arguments must be the arguments for the + async op. + */ + const opRun = (op,...args)=>{ + const opNdx = state.opIds[op] || toss("Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, state.opIds.rc, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + /* async thread will take over here */; + const t = performance.now(); + while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ + /* + The reason for this loop is buried in the details of a long + discussion at: + + https://github.com/sqlite/sqlite-wasm/issues/12 + + Summary: in at least one browser flavor, under high loads, + the wait()/notify() pairings can get out of sync. Calling + wait() here until it returns 'not-equal' gets them back in + sync. + */ + } + /* When the above wait() call returns 'not-equal', the async + half will have completed the operation and reported its results + in the state.opIds.rc slot of the SAB. */ + const rc = Atomics.load(state.sabOPView, state.opIds.rc); + metrics[op].wait += performance.now() - t; + if(rc && state.asyncS11nExceptions){ + const err = state.s11n.deserialize(); + if(err) error(op+"() async error:",...err); + } + return rc; + }; + + /** + Not part of the public API. Only for test/development use. + */ + opfsUtil.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; + + const initS11n = ()=>{ + /** + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ACHTUNG: this code is 100% duplicated in the other half of + this proxy! The documentation is maintained in the + "synchronous half". + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset + state.sabS11nSize]. Only one dataset is + recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from this thread or the + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. + */ + state.s11n.deserialize = function(clear=false){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + return state.s11n; + }/*initS11n()*/; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(""); + /* + An alternative impl. with an unpredictable length + but much simpler: + + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) + */ + }; + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = Object.create(null); + + const opTimer = Object.create(null); + opTimer.op = undefined; + opTimer.start = undefined; + const mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + ++metrics[op].count; + }; + const mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start + ); + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + /** + As of late 2022, only a single lock can be held on an OPFS + file. We have no way of checking whether any _other_ db + connection has a lock except by trying to obtain and (on + success) release a sync-handle for it, but doing so would + involve an inherent race condition. For the time being, + pending a better solution, we simply report whether the + given pFile is open. + + Update 2024-06-12: based on forum discussions, this + function now always sets pOut to 0 (false): + + https://sqlite.org/forum/forumpost/a2f573b00cda1372 + */ + wasm.poke(pOut, 0, 'i32'); + return 0; + }, + xClose: function(pFile){ + mTimeStart('xClose'); + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + mTimeEnd(); + return rc; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + /*mTimeStart('xFileControl'); + mTimeEnd();*/ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + mTimeStart('xFileSize'); + let rc = opRun('xFileSize', pFile); + if(0==rc){ + try { + const sz = state.s11n.deserialize()[0]; + wasm.poke(pSz64, sz, 'i64'); + }catch(e){ + error("Unexpected error reading xFileSize() result:",e); + rc = state.sq3Codes.SQLITE_IOERR; + } + } + mTimeEnd(); + return rc; + }, + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + const f = __openFiles[pFile]; + let rc = 0; + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( !f.lockType ) { + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; + }else{ + f.lockType = lockType; + } + mTimeEnd(); + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + mTimeStart('xRead'); + const f = __openFiles[pFile]; + let rc; + try { + rc = opRun('xRead',pFile, n, Number(offset64)); + if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ + /** + Results get written to the SharedArrayBuffer f.sabView. + Because the heap is _not_ a SharedArrayBuffer, we have + to copy the results. TypedArray.set() seems to be the + fastest way to copy this. */ + wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); + } + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + mTimeEnd(); + return rc; + }, + xSync: function(pFile,flags){ + mTimeStart('xSync'); + ++metrics.xSync.count; + const rc = opRun('xSync', pFile, flags); + mTimeEnd(); + return rc; + }, + xTruncate: function(pFile,sz64){ + mTimeStart('xTruncate'); + const rc = opRun('xTruncate', pFile, Number(sz64)); + mTimeEnd(); + return rc; + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === lockType + && f.lockType ){ + rc = opRun('xUnlock', pFile, lockType); + } + if( 0===rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + mTimeStart('xWrite'); + const f = __openFiles[pFile]; + let rc; + try { + f.sabView.set(wasm.heap8u().subarray( + Number(pSrc), Number(pSrc) + n + )); + rc = opRun('xWrite', pFile, n, Number(offset64)); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_WRITE; + } + mTimeEnd(); + return rc; + } + }/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + mTimeStart('xAccess'); + const rc = opRun('xAccess', wasm.cstrToJs(zName)); + wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); + mTimeEnd(); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + /* If it turns out that we need to adjust for timezone, see: + https://stackoverflow.com/a/11760121/1458521 */ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + mTimeStart('xDelete'); + const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); + mTimeEnd(); + return rc; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = opfsUtil.rootDirectory; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't or if an + exception is intercepted while trying to make the + determination. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string, intended for use as a + temporary file name. Its argument is the length of the string, + defaulting to 16. + */ + opfsUtil.randomFilename = randomFilename; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + await doDir(opfsUtil.rootDirectory, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). + */ + opfsUtil.rmfr = async function(){ + const dir = opfsUtil.rootDirectory, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each + entry. The argument may be either a callback function or an + options object with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: opfsUtil.rootDirectory + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + /** + impl of importDb() when it's given a function as its second + argument. + */ + const importDbChunked = async function(filename, callback){ + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + let sah = await hFile.createSyncAccessHandle(); + let nWrote = 0, chunk, checkedHeader = false, err = false; + try{ + sah.truncate(0); + while( undefined !== (chunk = await callback()) ){ + if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); + if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ + util.affirmDbHeader(chunk); + checkedHeader = true; + } + sah.write(chunk, {at: nWrote}); + nWrote += chunk.byteLength; + } + if( nWrote < 512 || 0!==nWrote % 512 ){ + toss("Input size",nWrote,"is not correct for an SQLite database."); + } + if( !checkedHeader ){ + const header = new Uint8Array(20); + sah.read( header, {at: 0} ); + util.affirmDbHeader( header ); + } + sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); + return nWrote; + }catch(e){ + await sah.close(); + sah = undefined; + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally { + if( sah ) await sah.close(); + } + }; + + /** + Asynchronously imports the given bytes (a byte array or + ArrayBuffer) into the given database file. + + Results are undefined if the given db name refers to an opened + db. + + If passed a function for its second argument, its behaviour + changes: imports its data in chunks fed to it by the given + callback function. It calls the callback (which may be async) + repeatedly, expecting either a Uint8Array or ArrayBuffer (to + denote new input) or undefined (to denote EOF). For so long as + the callback continues to return non-undefined, it will append + incoming data to the given VFS-hosted database file. When + called this way, the resolved value of the returned Promise is + the number of bytes written to the target file. + + It very specifically requires the input to be an SQLite3 + database and throws if that's not the case. It does so in + order to prevent this function from taking on a larger scope + than it is specifically intended to. i.e. we do not want it to + become a convenience for importing arbitrary files into OPFS. + + This routine rewrites the database header bytes in the output + file (not the input array) to force disabling of WAL mode. + + On error this throws and the state of the input file is + undefined (it depends on where the exception was triggered). + + On success, resolves to the number of bytes written. + */ + opfsUtil.importDb = async function(filename, bytes){ + if( bytes instanceof Function ){ + return importDbChunked(filename, bytes); + } + if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); + util.affirmIsDb(bytes); + const n = bytes.byteLength; + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + let sah, err, nWrote = 0; + try { + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + sah = await hFile.createSyncAccessHandle(); + sah.truncate(0); + nWrote = sah.write(bytes, {at: 0}); + if(nWrote != n){ + toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); + } + sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; + return nWrote; + }catch(e){ + if( sah ){ await sah.close(); sah = undefined; } + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally{ + if( sah ) await sah.close(); + } + }; + + if(sqlite3.oo1){ + const OpfsDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = opfsVfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsDb = OpfsDb; + OpfsDb.importDb = opfsUtil.importDb; + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + /* Set a relatively high default busy-timeout handler to + help OPFS dbs deal with multi-tab/multi-worker + contention. */ + sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); + } + ); + }/*extend sqlite3.oo1*/ + + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.poke(pOut,0,'i64'); + rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.peek(pOut, 'i64')); + rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.poke(readBuf+6,0); + let jRead = wasm.cstrToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //log("Worker.onmessage:",data); + switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; + case 'opfs-async-loaded': + /* Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async + worker. */ + W.postMessage({type: 'opfs-async-init',args: state}); + break; + case 'opfs-async-inited': { + /* Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin... */ + if(true===promiseWasRejected){ + break /* promise was already rejected via timer */; + } + try { + sqlite3.vfs.installVfs({ + io: {struct: opfsIoMethods, methods: ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: vfsSyncWrappers} + }); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + initS11n(); + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + if(thisThreadHasOPFS()){ + navigator.storage.getDirectory().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + sqlite3.opfs = opfsUtil; + opfsUtil.rootDirectory = d; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(); + }).catch(promiseReject); + }else{ + promiseResolve(); + } + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + default: { + const errMsg = ( + "Unexpected message from the OPFS async worker: " + + JSON.stringify(data) + ); + error(errMsg); + promiseReject(new Error(errMsg)); + break; + } + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; +}/*installOpfsVfs()*/; +installOpfsVfs.defaultProxyUri = + "sqlite3-opfs-async-proxy.js"; +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + try{ + let proxyJs = installOpfsVfs.defaultProxyUri; + if( sqlite3?.scriptInfo?.sqlite3Dir ){ + installOpfsVfs.defaultProxyUri = + sqlite3.scriptInfo.sqlite3Dir + proxyJs; + //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); + } + return installOpfsVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); + }); + }catch(e){ + sqlite3.config.error("installOpfsVfs() exception:",e); + return Promise.reject(e); + } +}); +}/*sqlite3ApiBootstrap.initializers.push()*/); +//#else +/* The OPFS VFS parts are elided from builds targeting node.js. */ +//#endif target:node diff --git a/manifest b/manifest index 387c312553..52ed36f06c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\sthe\sSQLITE_PREPARE_FROM_DDL\sdocumentation. -D 2026-02-26T16:19:21.240 +C An\sinitial\scopy/paste\sstub\sfor\sexperimenting\swith\susing\sWebLocks\sfor\sa\snew\scopy\sof\sthe\sOPFS\svfs.\sThis\sis\snot\syet\sfunctional,\sor\seven\sloading,\sbut\sneeds\sstashing\sto\savoid\spotential\sloss. +D 2026-02-27T03:11:40.358 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -605,6 +605,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 92d6d327a862f1627ff3e88e60fdfea9def06 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 17b16db29e2e0c75d00f675c17710539269e30f853da6bac9044879e298c270f F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 38484644c21b21b97f60979dddd620e47bdba628bde4ae62b6ce8859870e4f85 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -2195,8 +2196,11 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P c3288f16848866a2c846221c33631785f7b39938078bb95c61895f789395aa1d -R dcd62261fc7d31c8a65dd3cc5d14fd76 -U drh -Z de1179cfcfccba649912303467a12236 +P 9174e7dacf867c1f80e73ccf5e0ee7b9e84fbe9f2e53559d06b72206b1cde3c6 +R b825f5f5f6b51df161046dc5b5c20e5a +T *branch * opfs-wl +T *sym-opfs-wl * +T -sym-trunk * Cancelled\sby\sbranch. +U stephan +Z 7cdbae53834e92d0f15d611fc35ab500 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.tags b/manifest.tags index bec971799f..e8a777ecb7 100644 --- a/manifest.tags +++ b/manifest.tags @@ -1,2 +1,2 @@ -branch trunk -tag trunk +branch opfs-wl +tag opfs-wl diff --git a/manifest.uuid b/manifest.uuid index 1a18fe1258..f746549ef5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9174e7dacf867c1f80e73ccf5e0ee7b9e84fbe9f2e53559d06b72206b1cde3c6 +372f18ae909bafd7167b70051935832e3107ede72aaeccbf5c042db7ba791912 From e2309eebd74193efe65014896ec264a743026810 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 3 Mar 2026 21:07:46 +0000 Subject: [PATCH 02/32] Implementations of WebLock-based opfs::xLock() and xUnlock(). Still completely untested and not yet integrated into the build. FossilOrigin-Name: 3343e3aabe465f4ab91dd148dfc5a60346e9afb560c10d1f92aeae98763ec3ce --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 45 ++++++++++++++++++++-- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 48 ++++++++++++++++++------ manifest | 14 +++---- manifest.uuid | 2 +- 4 files changed, 86 insertions(+), 23 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 79fc47393e..ede2bc7cd4 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -318,7 +318,7 @@ const installAsyncProxy = function(){ } log("Got",opName+"() sync handle for",fh.filenameAbs, 'in',performance.now() - t,'ms'); - if(!fh.xLock){ + if(!fh.xLock && !state.lock/*set by opfs-wl*/){ __implicitLocks.add(fh.fid); log("Acquired implicit lock for",opName+"()",fh.fid,fh.filenameAbs); } @@ -703,6 +703,39 @@ const installAsyncProxy = function(){ return state.s11n; }/*initS11n()*/; + /** + Starts a new WebLock request. + */ + const handleLockRequest = async function(){ + const args = state.s11n.deserialize(true) + || toss("Expecting a filename argument from the proxy xLock()"); + const view = state.sabOPView; + const slock = state.lock; + const lockType = Atomics.load(view, slock.type); + + await navigator.locks.request('sqlite3-vfs-opfs:'+args[0], { + mode: (lockType===state.sq3Codes.SQLITE_LOCK_EXCLUSIVE) + ? 'exclusive' : 'shared' + }, async (wl)=>{ + /** + A. Tell the C-side we have the browser lock. We use the same + handshake slot, but a specific 'Granted' value. + */ + Atomics.store(view, slock.atomicsHandshake, 2); + Atomics.notify(view, slock.atomicsHandshake); + /** + B. Sit here and keep the lock room occupied until the + Receptionist receives 'unlockControl'. + */ + while( 1!==Atomics.load(view, slock.atomicsHandshake) ){ + Atomics.wait(view. slock.atomicsHandshake, 2); + } + /* C. Reset the handshake slot. */ + Atomics.store(view, slock.atomicsHandshake, 0); + Atomics.notify(view, lock.atomicsHandshake); + }); + }; + const waitLoop = async function f(){ const opHandlers = Object.create(null); for(let k of Object.keys(state.opIds)){ @@ -713,10 +746,11 @@ const installAsyncProxy = function(){ o.key = k; o.f = vi; } + const opIds = state.opIds; while(!flagAsyncShutdown){ try { if('not-equal'!==Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, state.asyncIdleWaitTime + state.sabOPView, opIds.whichOp, 0, state.asyncIdleWaitTime )){ /* Maintenance note: we compare against 'not-equal' because @@ -736,8 +770,11 @@ const installAsyncProxy = function(){ await releaseImplicitLocks(); continue; } - const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); - Atomics.store(state.sabOPView, state.opIds.whichOp, 0); + const opId = Atomics.exchange(state.sabOPView, opIds.whichOp, 0); + if( opId===opIds.lockControl ){ + handleLockRequest(); + continue; + } const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 9fe09269cf..99e176c396 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -372,7 +372,6 @@ const installOpfsVfs = function callee(options){ state.opIds.xWrite = i++; state.opIds.mkdir = i++; state.opIds.lockControl = i++ /* we signal the intent to lock here */; - state.opIds.lockType /** Internal signals which are used only during development and testing via the dev console. */ state.opIds['opfs-async-metrics'] = i++; @@ -384,8 +383,8 @@ const installOpfsVfs = function callee(options){ /* Slots for submitting the lock type and receiving its acknowledgement. */ state.lock = nu({ - type: i++, - atomicsHandshake: i++ + type: i++ /* SQLITE_LOCK_xyz value */, + atomicsHandshake: i++ /* 1=release, 2=granted */ }); state.sabOP = new SharedArrayBuffer( i * 4/* ==sizeof int32, noting that Atomics.wait() and friends @@ -425,7 +424,12 @@ const installOpfsVfs = function callee(options){ 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE', 'SQLITE_OPEN_MAIN_DB', - 'SQLITE_OPEN_READONLY' + 'SQLITE_OPEN_READONLY', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_EXCLUSIVE' ].forEach((k)=>{ if(undefined === (state.sq3Codes[k] = capi[k])){ toss("Maintenance required: not found:",k); @@ -787,7 +791,7 @@ const installOpfsVfs = function callee(options){ mTimeEnd(); return rc; }, - xLock: function(pFile,lockType){ + xLock: function(pFile, lockType){ mTimeStart('xLock'); const f = __openFiles[pFile]; let rc = 0; @@ -796,8 +800,23 @@ const installOpfsVfs = function callee(options){ type. If no lock is active, have the async counterpart lock the file. */ if( !f.lockType ) { - rc = opRun('xLock', pFile, lockType); - if( 0===rc ) f.lockType = lockType; + try{ + const view = state.sabOPView; + /* We need to pass pFile's name through so that the other + side can create the WebLock name. */ + state.s11n.serialize(f.filename) + Atomics.store(view, state.lock.type, lockType); + Atomics.store(view, state.opIds.whichOp, state.opIds.lockControl); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + while('not-equal'!==Atomics.wait(view, state.lock.atomicsHandshake, 0)){ + /* Loop is a workaround for environment-specific quirks. See + notes in similar loops. */ + } + f.lockType = lockType; + }catch(e){ + error("xLock(",arguments,") failed", e, f); + rc = capi.SQLITE_IOERR_LOCK; + } }else{ f.lockType = lockType; } @@ -842,9 +861,16 @@ const installOpfsVfs = function callee(options){ mTimeStart('xUnlock'); const f = __openFiles[pFile]; let rc = 0; - if( capi.SQLITE_LOCK_NONE === lockType - && f.lockType ){ - rc = opRun('xUnlock', pFile, lockType); + if( lockType < f.lockType ){ + try{ + const view = state.sabOPView; + Atomics.store(view, state.lock.atomicsHandshake, 1); + Atomics.notify(view, state.lock.atomicsHandshake); + Atomics.wait(view, state.lock.atomicsHandshake, 1); + }catch(e){ + error("xUnlock(",pFile,lockType,") failed",e, f); + rc = capi.SQLITE_IOERR_LOCK; + } } if( 0===rc ) f.lockType = lockType; mTimeEnd(); @@ -932,7 +958,7 @@ const installOpfsVfs = function callee(options){ } const fh = Object.create(null); fh.fid = pFile; - fh.filename = zName; + fh.filename = wasm.cstrToJs(zName); fh.sab = new SharedArrayBuffer(state.fileBufferSize); fh.flags = flags; fh.readOnly = !(capi.SQLITE_OPEN_CREATE & flags) diff --git a/manifest b/manifest index 7e05ddc666..1ef5d804a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sinto\sthe\sopfs-wl\sbranch. -D 2026-03-03T18:34:59.705 +C Implementations\sof\sWebLock-based\sopfs::xLock()\sand\sxUnlock().\sStill\scompletely\suntested\sand\snot\syet\sintegrated\sinto\sthe\sbuild. +D 2026-03-03T21:07:46.412 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -592,11 +592,11 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 1fefd40ab21e3dbf46f43b6fafb07f13eb13cc157a884f7c1134caf631ddb3f2 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.js 92d6d327a862f1627ff3e88e60fdfea9def06ad539d98929ba46490e64372736 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 912ddb17627e933eb9596d393227f7ea47b1136fc2fb0d957d9979e71de59e81 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 38484644c21b21b97f60979dddd620e47bdba628bde4ae62b6ce8859870e4f85 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js c3e453a5736ee1bb9b08247c6e97c1433933a320c7548bedd20481eae7922a48 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -2189,8 +2189,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 372f18ae909bafd7167b70051935832e3107ede72aaeccbf5c042db7ba791912 88dce64242552e7443d9fb496f6f3ad71dc5b4a882ce21b7ef1d5ea4e26f1e61 -R 75fb07a85007c6410c69c66cd4391b75 +P 81bc4b3b6abc19f98d1f3b1065c4b39a42620a0d7abebe98605dca62dd2d6f9a +R f9d18b4e8da91823211b005773e17736 U stephan -Z 42a2789071a18d2bcb63e8ba8c976a45 +Z e389fe2e9ad12fa6369004db9e3f215f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cb69ee9652..976e29a881 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -81bc4b3b6abc19f98d1f3b1065c4b39a42620a0d7abebe98605dca62dd2d6f9a +3343e3aabe465f4ab91dd148dfc5a60346e9afb560c10d1f92aeae98763ec3ce From 0b7b802d8331cda0f5d9ee4714de078d1c4c3eaa Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 3 Mar 2026 21:46:44 +0000 Subject: [PATCH 03/32] Add a missing reset of the lock handshake SharedArrayBuffer slot. FossilOrigin-Name: 5910ed7ac843aa1fdb103ddcfaf6270d12cb39199205fc0b9f721fbf7fa2f851 --- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 99e176c396..0184bd480e 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -805,6 +805,7 @@ const installOpfsVfs = function callee(options){ /* We need to pass pFile's name through so that the other side can create the WebLock name. */ state.s11n.serialize(f.filename) + Atomics.store(view, state.lock.atomicsHandshake, 0); Atomics.store(view, state.lock.type, lockType); Atomics.store(view, state.opIds.whichOp, state.opIds.lockControl); Atomics.notify(state.sabOPView, state.opIds.whichOp) diff --git a/manifest b/manifest index 1ef5d804a8..b8efc4b0b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implementations\sof\sWebLock-based\sopfs::xLock()\sand\sxUnlock().\sStill\scompletely\suntested\sand\snot\syet\sintegrated\sinto\sthe\sbuild. -D 2026-03-03T21:07:46.412 +C Add\sa\smissing\sreset\sof\sthe\slock\shandshake\sSharedArrayBuffer\sslot. +D 2026-03-03T21:46:44.640 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -596,7 +596,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 912ddb17627e933eb9596d393227f7ea47b11 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js c3e453a5736ee1bb9b08247c6e97c1433933a320c7548bedd20481eae7922a48 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js deea0f903e8265f58fe57315c35f62170fd65d4730bece7978772c28cead1402 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -2189,8 +2189,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 81bc4b3b6abc19f98d1f3b1065c4b39a42620a0d7abebe98605dca62dd2d6f9a -R f9d18b4e8da91823211b005773e17736 +P 3343e3aabe465f4ab91dd148dfc5a60346e9afb560c10d1f92aeae98763ec3ce +R f490861a3a61c6b12505aa6e9bec9268 U stephan -Z e389fe2e9ad12fa6369004db9e3f215f +Z 0dd9a22951092896d0ba74d53d905c49 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 976e29a881..757a353e6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3343e3aabe465f4ab91dd148dfc5a60346e9afb560c10d1f92aeae98763ec3ce +5910ed7ac843aa1fdb103ddcfaf6270d12cb39199205fc0b9f721fbf7fa2f851 From a2d3a54f80b890a25e82877bcf2b612a64919255 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 3 Mar 2026 23:43:40 +0000 Subject: [PATCH 04/32] An initial attempt to plugging opfs-wl into the build but its initial handshake with the async half collides with the opfs VFS's handshake, causing bootstrapping to fail miserably. We'll need to either devise a handshake which can differentiate between the two instances or we'll need to preprocess sqlite3-opfs-async-proxy into two copies. Move the (now) three copies of some common code shared by the opfs pieces into a preprocessor #include. FossilOrigin-Name: 1e0b72631aecb0bb72f4089116da221e6c4abf962db589de08132cd52c2be0e2 --- ext/wasm/GNUmakefile | 11 +- ext/wasm/api/opfs-common.c-pp.js | 200 ++++++++++++++ ext/wasm/api/post-js-header.js | 1 + ext/wasm/api/sqlite3-api-prologue.js | 3 +- ...xy.js => sqlite3-opfs-async-proxy.c-pp.js} | 101 +------ ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 251 +++--------------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 170 +----------- ext/wasm/mkwasmbuilds.c | 7 +- ext/wasm/tests/opfs/concurrency/worker.js | 9 +- manifest | 27 +- manifest.uuid | 2 +- 11 files changed, 274 insertions(+), 508 deletions(-) create mode 100644 ext/wasm/api/opfs-common.c-pp.js rename ext/wasm/api/{sqlite3-opfs-async-proxy.js => sqlite3-opfs-async-proxy.c-pp.js} (88%) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index dc3c2d2555..371bec6bdd 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -913,6 +913,7 @@ endif sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js +#sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js # Parallel builds can fail if $(sqlite3-license-version.js) is not # created early enough, so make all files in $(sqlite-api.jses) except @@ -1182,13 +1183,13 @@ all: demos ####################################################################### # -# "SOAP" is a static file which is not part of the amalgamation but -# gets copied into the build output folder and into each of the fiddle -# builds. +# "SOAP" is not part of the amalgamation but gets copied into the +# build output folder and into each of the fiddle builds. # sqlite3.ext.js += $(dir.dout)/sqlite3-opfs-async-proxy.js -$(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js - @$(call b.cp,@,$<,$@) +$(eval $(call b.c-pp.target,soap,\ + $(dir.api)/sqlite3-opfs-async-proxy.c-pp.js,\ + $(dir.dout)/sqlite3-opfs-async-proxy.js)) # # Add a dep of $(sqlite3.ext.js) on every individual build's JS file. diff --git a/ext/wasm/api/opfs-common.c-pp.js b/ext/wasm/api/opfs-common.c-pp.js new file mode 100644 index 0000000000..1a7818927a --- /dev/null +++ b/ext/wasm/api/opfs-common.c-pp.js @@ -0,0 +1,200 @@ +//#if nope +/** + This file is for preprocessor #include into the "opfs" and + "opfs-wl" impls, as well as their async-proxy part. +*/ +//#endif + +//#if not defined opfs-async-proxy +/** + TODO: move the sqlite3.opfs (private/internal) namespace object + init from sqlite3-vfs-opfs*.js into here. That namespace gets + removed from the sqlite3 namespace in the final stages of library + bootstrapping except in test runs, where it's retained so that + tests can clean up OPFS so their test cases work (the down-side of + them being persistent). +*/ +//#endif not defined opfs-async-proxy + +// This function won't work as-is if we #include it, but the +// missing elements have not yet been identified. +const initS11n = function(){ + /* This function is needed by the "opfs" and "opfs-wl" VFSes + and sqlite-opfs-async-proxy.js (used by those of those). */ + /** + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset + state.sabS11nSize]. Only one dataset is + recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from this thread or the + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. + */ + state.s11n.deserialize = function(clear=false){ +//#if defined opfs-has-metrics + ++metrics.s11n.deserialize.count; +//#endif + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); +//#if defined opfs-has-metrics + metrics.s11n.deserialize.time += performance.now() - t; +//#endif + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); +//#if defined opfs-has-metrics + ++metrics.s11n.serialize.count; +//#endif + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } +//#if defined opfs-has-metrics + metrics.s11n.serialize.time += performance.now() - t; +//#endif + }; + +//#if defined opfs-async-proxy + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; +//#endif + + return state.s11n; +}/*initS11n()*/; diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index 670051bd86..348f80ea0b 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -34,6 +34,7 @@ Module.runSQLite3PostLoadInit = async function( - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls - sqlite3-vfs-opfs.c-pp.js => OPFS VFS - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS + - sqlite3-vfs-opfs-wl.c-pp.js => WebLock-using OPFS VFS - post-js-footer.js => this file's epilogue And all of that gets sandwiched between extern-pre-js.js and diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index c53acee769..3ee44d33b0 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -2002,6 +2002,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( so that we can add tests for them. */ delete sqlite3.util; delete sqlite3.StructBinder; + delete sqlite3.opfs; } return sqlite3; }; @@ -2034,7 +2035,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ scriptInfo: undefined }; - if( ('undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */) ){ + if( 'undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */ ){ sqlite3.__isUnderTest = !!sqlite3IsUnderTest; } try{ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js similarity index 88% rename from ext/wasm/api/sqlite3-opfs-async-proxy.js rename to ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index ede2bc7cd4..294e7687be 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -603,105 +603,8 @@ const installAsyncProxy = function(){ } }/*vfsAsyncImpls*/; - const initS11n = ()=>{ - /** - ACHTUNG: this code is 100% duplicated in the other half of this - proxy! The documentation is maintained in the "synchronous half". - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - state.s11n.deserialize = function(clear=false){ - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - return rc; - }; - state.s11n.serialize = function(...args){ - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - }; - - state.s11n.storeException = state.asyncS11nExceptions - ? ((priority,e)=>{ - if(priority<=state.asyncS11nExceptions){ - state.s11n.serialize([e.name,': ',e.message].join("")); - } - }) - : ()=>{}; - - return state.s11n; - }/*initS11n()*/; +//#define opfs-async-proxy +//#include api/opfs-common.c-pp.js /** Starts a new WebLock request. diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 0184bd480e..dfa44af5e1 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -11,54 +11,51 @@ *********************************************************************** - This file is a placeholder for a reimplementation of the "opfs" VFS - (as distinct from "opfs-sahpool") which uses WebLocks instead - locking based on a bespoke custom Atomics.wait()/notify() - protocol. This file holds the "synchronous half" of the VFS, whereas - it shares the "asynchronous half" of the "opfs" VFS. + This file is a reimplementation of the "opfs" VFS (as distinct from + "opfs-sahpool") which uses WebLocks for locking instead of a bespoke + custom Atomics.wait()/notify() protocol. This file holds the + "synchronous half" of the VFS, whereas it shares the "asynchronous + half" of the "opfs" VFS. This file is intended to be appended to the main sqlite3 JS deliverable somewhere after sqlite3-api-oo1.js. + + TODOs (2026-0303): + + - Move pieces of this file which are common to it, + sqlite3-vfs-opfs.c-pp.js, and/or sqlite3-opfs-async-proxy.js into + separate files and #include them in each using the preprocessor. + e.g. the s11n namespace object is duplicated in all three files. + + - For purposes of tester1.js we need to figure out which of these + VFSes will install the (internal-use-only) sqlite3.opfs utility code + namespace. We need that in order to clean up OPFS files during test + runs. Alternately, move those into their own + sqlite3ApiBootstrap.initializers entry which precedes both of the + VFSes, so they'll have access to it during bootstrapping. The + sqlite3.opfs namespace is removed at the end of bootstrapping unless + the library is told to run in testing mode (which is not a + documented feature). */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** installOpfsVfs() returns a Promise which, on success, installs an - sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs + sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via sqlite3ApiBootstrap.initializers or an equivalent mechanism. - The installed VFS uses the Origin-Private FileSystem API for - all file storage. On error it is rejected with an exception - explaining the problem. Reasons for rejection include, but are - not limited to: + This VFS is essentially a copy of the "opfs" VFS but uses + WebLocks for its xLock() and xUnlock() implementations. - - The counterpart Worker (see below) could not be loaded. + Quirks specific to this VFS: - - The environment does not support OPFS. That includes when - this function is called from the main window thread. + - Because WebLocks effectively block until they return, they will + effectively hang on locks rather than returning SQLITE_BUSY. - Significant notes and limitations: - - The OPFS features used here are only available in dedicated Worker - threads. This file tries to detect that case, resulting in a - rejected Promise if those features do not seem to be available. - - - It requires the SharedArrayBuffer and Atomics classes, and the - former is only available if the HTTP server emits the so-called - COOP and COEP response headers. These features are required for - proxying OPFS's synchronous API via the synchronous interface - required by the sqlite3_vfs API. - - - This function may only be called a single time. When called, this - function removes itself from the sqlite3 object. - - All arguments to this function are for internal/development purposes - only. They do not constitute a public API and may change at any - time. - - The argument may optionally be a plain object with the following - configuration options: + The argument may optionally be a plain object with the following + configuration options: - proxyUri: name of the async proxy JS file. @@ -67,14 +64,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ additionally enables debugging info. Logging is performed via the sqlite3.config.{log|warn|error}() functions. - - sanityChecks (=false): if true, some basic sanity tests are run on - the OPFS VFS API after it's initialized, before the returned - Promise resolves. This is only intended for testing and - development of the VFS, not client-side use. - On success, the Promise resolves to the top-most sqlite3 namespace - object and that object gets a new object installed in its - `opfs` property, containing several OPFS-specific utilities. + object. */ const installOpfsVfs = function callee(options){ if(!globalThis.SharedArrayBuffer @@ -115,9 +106,6 @@ const installOpfsVfs = function callee(options){ if(undefined===options.proxyUri){ options.proxyUri = callee.defaultProxyUri; } - - //sqlite3.config.warn("OPFS options =",options,globalThis.location); - if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } @@ -524,171 +512,8 @@ const installOpfsVfs = function callee(options){ } }; - const initS11n = ()=>{ - /** - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ACHTUNG: this code is 100% duplicated in the other half of - this proxy! The documentation is maintained in the - "synchronous half". - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - This proxy de/serializes cross-thread function arguments and - output-pointer values via the state.sabIO SharedArrayBuffer, - using the region defined by (state.sabS11nOffset, - state.sabS11nOffset + state.sabS11nSize]. Only one dataset is - recorded at a time. - - This is not a general-purpose format. It only supports the - range of operations, and data sizes, needed by the - sqlite3_vfs and sqlite3_io_methods operations. Serialized - data are transient and this serialization algorithm may - change at any time. - - The data format can be succinctly summarized as: - - Nt...Td...D - - Where: - - - N = number of entries (1 byte) - - - t = type ID of first argument (1 byte) - - - ...T = type IDs of the 2nd and subsequent arguments (1 byte - each). - - - d = raw bytes of first argument (per-type size). - - - ...D = raw bytes of the 2nd and subsequent arguments (per-type - size). - - All types except strings have fixed sizes. Strings are stored - using their TextEncoder/TextDecoder representations. It would - arguably make more sense to store them as Int16Arrays of - their JS character values, but how best/fastest to get that - in and out of string form is an open point. Initial - experimentation with that approach did not gain us any speed. - - Historical note: this impl was initially about 1% this size by - using using JSON.stringify/parse(), but using fit-to-purpose - serialization saves considerable runtime. - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - /* Only arguments and return values of these types may be - serialized. This covers the whole range of types needed by the - sqlite3_vfs API. */ - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - - /** - Returns an array of the deserialized state stored by the most - recent serialize() operation (from this thread or the - counterpart thread), or null if the serialization buffer is - empty. If passed a truthy argument, the serialization buffer - is cleared after deserialization. - */ - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - - /** - Serializes all arguments to the shared buffer for consumption - by the counterpart thread. - - This routine is only intended for serializing OPFS VFS - arguments and (in at least one special case) result values, - and the buffer is sized to be able to comfortably handle - those. - - If passed no arguments then it zeroes out the serialization - state. - */ - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - return state.s11n; - }/*initS11n()*/; +//#define opfs-has-metrics +//#include api/opfs-common.c-pp.js /** Generates a random ASCII string len characters long, intended for @@ -1323,14 +1148,14 @@ const installOpfsVfs = function callee(options){ }; if(sqlite3.oo1){ - const OpfsDb = function(...args){ + const OpfsWlDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); opt.vfs = opfsVfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; - OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); - sqlite3.oo1.OpfsDb = OpfsDb; - OpfsDb.importDb = opfsUtil.importDb; + OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsWlDb = OpfsWlDb; + OpfsWlDb.importDb = opfsUtil.importDb; sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( opfsVfs.pointer, function(oo1Db, sqlite3){ @@ -1446,7 +1271,7 @@ const installOpfsVfs = function callee(options){ navigator.storage.getDirectory().then((d)=>{ W.onerror = W._originalOnError; delete W._originalOnError; - sqlite3.opfs = opfsUtil; + //sqlite3.opfs = opfsUtil; opfsUtil.rootDirectory = d; log("End of OPFS sqlite3_vfs setup.", opfsVfs); promiseResolve(); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index ffa90ed068..dd1b4ba90b 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -71,8 +71,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ development of the VFS, not client-side use. On success, the Promise resolves to the top-most sqlite3 namespace - object and that object gets a new object installed in its - `opfs` property, containing several OPFS-specific utilities. + object. */ const installOpfsVfs = function callee(options){ if(!globalThis.SharedArrayBuffer @@ -509,171 +508,8 @@ const installOpfsVfs = function callee(options){ } }; - const initS11n = ()=>{ - /** - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ACHTUNG: this code is 100% duplicated in the other half of - this proxy! The documentation is maintained in the - "synchronous half". - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - This proxy de/serializes cross-thread function arguments and - output-pointer values via the state.sabIO SharedArrayBuffer, - using the region defined by (state.sabS11nOffset, - state.sabS11nOffset + state.sabS11nSize]. Only one dataset is - recorded at a time. - - This is not a general-purpose format. It only supports the - range of operations, and data sizes, needed by the - sqlite3_vfs and sqlite3_io_methods operations. Serialized - data are transient and this serialization algorithm may - change at any time. - - The data format can be succinctly summarized as: - - Nt...Td...D - - Where: - - - N = number of entries (1 byte) - - - t = type ID of first argument (1 byte) - - - ...T = type IDs of the 2nd and subsequent arguments (1 byte - each). - - - d = raw bytes of first argument (per-type size). - - - ...D = raw bytes of the 2nd and subsequent arguments (per-type - size). - - All types except strings have fixed sizes. Strings are stored - using their TextEncoder/TextDecoder representations. It would - arguably make more sense to store them as Int16Arrays of - their JS character values, but how best/fastest to get that - in and out of string form is an open point. Initial - experimentation with that approach did not gain us any speed. - - Historical note: this impl was initially about 1% this size by - using using JSON.stringify/parse(), but using fit-to-purpose - serialization saves considerable runtime. - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - /* Only arguments and return values of these types may be - serialized. This covers the whole range of types needed by the - sqlite3_vfs API. */ - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - - /** - Returns an array of the deserialized state stored by the most - recent serialize() operation (from this thread or the - counterpart thread), or null if the serialization buffer is - empty. If passed a truthy argument, the serialization buffer - is cleared after deserialization. - */ - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - - /** - Serializes all arguments to the shared buffer for consumption - by the counterpart thread. - - This routine is only intended for serializing OPFS VFS - arguments and (in at least one special case) result values, - and the buffer is sized to be able to comfortably handle - those. - - If passed no arguments then it zeroes out the serialization - state. - */ - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - return state.s11n; - }/*initS11n()*/; +//#define opfs-has-metrics +//#include api/opfs-common.c-pp.js /** Generates a random ASCII string len characters long, intended for diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 37f2967d88..748594f4ca 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -994,7 +994,8 @@ static void mk_fiddle(void){ pf("$(out.%s.js): $(MAKEFILE_LIST) " "$(EXPORTED_FUNCTIONS.fiddle) " "$(fiddle.c.in) " - "$(pre-post.%s.deps)", + "$(pre-post.%s.deps) " + "$(dir.dout)/sqlite3-opfs-async-proxy.js", zBuildName, zBuildName); if( isDebug ){ pf(" $(dir.fiddle)/fiddle-worker.js" @@ -1014,10 +1015,6 @@ static void mk_fiddle(void){ pf("\t@$(call b.call.wasm-strip,%s)\n", zBuildName); pf("\t@$(call b.strip-js-emcc-bindings,$(logtag.%s))\n", zBuildName); - pf("\t@$(call b.cp," - "%s," - "$(dir.api)/sqlite3-opfs-async-proxy.js," - "$(dir $@))\n", zBuildName); if( isDebug ){ pf("\t@$(call b.cp,%s," "$(dir.fiddle)/index.html " diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 5d28bedee0..a28d80f202 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -1,8 +1,9 @@ importScripts( - (new URL(self.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' + (new URL(globalThis.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' ); -self.sqlite3InitModule().then(async function(sqlite3){ - const urlArgs = new URL(self.location.href).searchParams; +globalThis.sqlite3InitModule.__isUnderTest = true; +globalThis.sqlite3InitModule().then(async function(sqlite3){ + const urlArgs = new URL(globalThis.location.href).searchParams; const options = { workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ @@ -98,7 +99,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }/*run()*/; - self.onmessage = function({data}){ + globalThis.onmessage = function({data}){ switch(data.type){ case 'run': run().catch((e)=>{ if(!interval.error) interval.error = e; diff --git a/manifest b/manifest index b8efc4b0b2..a23506fd3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\sreset\sof\sthe\slock\shandshake\sSharedArrayBuffer\sslot. -D 2026-03-03T21:46:44.640 +C An\sinitial\sattempt\sto\splugging\sopfs-wl\sinto\sthe\sbuild\sbut\sits\sinitial\shandshake\swith\sthe\sasync\shalf\scollides\swith\sthe\sopfs\sVFS's\shandshake,\scausing\sbootstrapping\sto\sfail\smiserably.\sWe'll\sneed\sto\seither\sdevise\sa\shandshake\swhich\scan\sdifferentiate\sbetween\sthe\stwo\sinstances\sor\swe'll\sneed\sto\spreprocess\ssqlite3-opfs-async-proxy\sinto\stwo\scopies.\sMove\sthe\s(now)\sthree\scopies\sof\ssome\scommon\scode\sshared\sby\sthe\sopfs\spieces\sinto\sa\spreprocessor\s#include. +D 2026-03-03T23:43:40.326 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -572,7 +572,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6 -F ext/wasm/GNUmakefile 79236447d750609aa6beda30feec1314180c5462a493ad94214122887232bfd4 +F ext/wasm/GNUmakefile a64605e68d4561f440bdc538e01364210f75052469d6ea21bf2843f1539144e9 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -584,20 +584,21 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 7ba933e8f1290cc65459dd371c0c9a031d96bdf14 F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 +F ext/wasm/api/opfs-common.c-pp.js 5540c58aee8fbbf2c67984e0a35e2b680a7603052f2cc442d407da3cc81c4819 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b -F ext/wasm/api/post-js-header.js d24bd0d065f3489c8b78ddf3ead6321e5d047187a162cd503c41700e03dd1f06 +F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa -F ext/wasm/api/sqlite3-api-prologue.js 1fefd40ab21e3dbf46f43b6fafb07f13eb13cc157a884f7c1134caf631ddb3f2 +F ext/wasm/api/sqlite3-api-prologue.js ccd8ece4b4580d2a70996218f28e810d70a86f5e2795f4d4a75f0603af24aef6 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.js 912ddb17627e933eb9596d393227f7ea47b1136fc2fb0d957d9979e71de59e81 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 0ec326d4c66c7af5a4c79a1be65c07c1e1844e641f133ba026a4a01eb1a37c56 w ext/wasm/api/sqlite3-opfs-async-proxy.js F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js deea0f903e8265f58fe57315c35f62170fd65d4730bece7978772c28cead1402 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 1abe919013d759b311e7e3f5c4a326f9027b1b6a0fbe89db5f303e2ff109d5d4 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js ab2a73f41a2e109f25fd3159531e801ca14c782c297b2fc1836e6a02b795321b F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -626,7 +627,7 @@ F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x -F ext/wasm/mkwasmbuilds.c 0e9198eb90acae4bcf57cf62d7186f6af5aaac02efdb075a1aded33614b3805a +F ext/wasm/mkwasmbuilds.c b1ed20dfba178e6ccae91b4594ac9439992fd0cb783f202c639ca761f1b1d2a3 F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -645,7 +646,7 @@ F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3 F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 -F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 +F ext/wasm/tests/opfs/concurrency/worker.js 3e29c07bdd11f4ff9ee50e641e39c254d39243bbb83fb20a255755ee92739d12 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2189,8 +2190,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 3343e3aabe465f4ab91dd148dfc5a60346e9afb560c10d1f92aeae98763ec3ce -R f490861a3a61c6b12505aa6e9bec9268 +P 5910ed7ac843aa1fdb103ddcfaf6270d12cb39199205fc0b9f721fbf7fa2f851 +R d999a6bf319e44a9103bde13f0ab87af U stephan -Z 0dd9a22951092896d0ba74d53d905c49 +Z 097c558a7425489e055f6ff0be38c909 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 757a353e6f..9be206dbe2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5910ed7ac843aa1fdb103ddcfaf6270d12cb39199205fc0b9f721fbf7fa2f851 +1e0b72631aecb0bb72f4089116da221e6c4abf962db589de08132cd52c2be0e2 From 9ac95d9ce1324d2acce9877b4c7c389c223d7a39 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 00:37:55 +0000 Subject: [PATCH 05/32] opfs-wl is now loading and registering but it's still untested. FossilOrigin-Name: 9a471f7491a371052bf1785098ba966dd0d03503e7d8b9fbcd65f07b038e5021 --- ext/wasm/GNUmakefile | 2 +- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 8 ++++++++ ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 9 ++++++--- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 7 ++++--- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 29 insertions(+), 17 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 371bec6bdd..7e48113ed2 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -913,7 +913,7 @@ endif sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js -#sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js # Parallel builds can fail if $(sqlite3-license-version.js) is not # created early enough, so make all files in $(sqlite-api.jses) except diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 294e7687be..740be8938b 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -51,6 +51,14 @@ */ "use strict"; const wPost = (type,...args)=>postMessage({type, payload:args}); +//#if nope +const urlParams = new URL(globalThis.location.href).searchParams; +if( !urlParams.has('vfs') ){ + throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); +} +const isWebLocker = 'opfs-wl'===urlParams.get('vfs'); +const msgKeyPrefix = 'opfs-'; //isWebLocker ? 'opfs-wl-' : 'opfs-'; +//#endif const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; if(globalThis.window === globalThis){ diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index dfa44af5e1..c53bebdfe1 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -197,13 +197,14 @@ const installOpfsVfs = function callee(options){ promiseWasRejected = false; return promiseResolve_(sqlite3); }; + const workerArgs = '?vfs=opfs-wl'; const W = //#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url)); + new Worker(new URL("sqlite3-opfs-async-proxy.js"+workerArgs, import.meta.url)); //#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); + new Worker(new URL(options.proxyUri+workerArgs, import.meta.url)); //#else - new Worker(options.proxyUri); + new Worker(options.proxyUri+workerArgs); //#endif setTimeout(()=>{ /* At attempt to work around a browser-specific quirk in which @@ -1156,6 +1157,7 @@ const installOpfsVfs = function callee(options){ OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsWlDb = OpfsWlDb; OpfsWlDb.importDb = opfsUtil.importDb; +//#if nope sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( opfsVfs.pointer, function(oo1Db, sqlite3){ @@ -1165,6 +1167,7 @@ const installOpfsVfs = function callee(options){ sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); } ); +//#endif }/*extend sqlite3.oo1*/ const sanityCheck = function(){ diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index dd1b4ba90b..934564c5ea 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -207,13 +207,14 @@ const installOpfsVfs = function callee(options){ promiseWasRejected = false; return promiseResolve_(sqlite3); }; + const workerArgs = '?vfs=opfs'; const W = //#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url)); + new Worker(new URL("sqlite3-opfs-async-proxy.js"+workerArgs, import.meta.url)); //#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); + new Worker(new URL(options.proxyUri+workerArgs, import.meta.url)); //#else - new Worker(options.proxyUri); + new Worker(options.proxyUri+workerArgs); //#endif setTimeout(()=>{ /* At attempt to work around a browser-specific quirk in which diff --git a/manifest b/manifest index a23506fd3a..2f56d2c256 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C An\sinitial\sattempt\sto\splugging\sopfs-wl\sinto\sthe\sbuild\sbut\sits\sinitial\shandshake\swith\sthe\sasync\shalf\scollides\swith\sthe\sopfs\sVFS's\shandshake,\scausing\sbootstrapping\sto\sfail\smiserably.\sWe'll\sneed\sto\seither\sdevise\sa\shandshake\swhich\scan\sdifferentiate\sbetween\sthe\stwo\sinstances\sor\swe'll\sneed\sto\spreprocess\ssqlite3-opfs-async-proxy\sinto\stwo\scopies.\sMove\sthe\s(now)\sthree\scopies\sof\ssome\scommon\scode\sshared\sby\sthe\sopfs\spieces\sinto\sa\spreprocessor\s#include. -D 2026-03-03T23:43:40.326 +C opfs-wl\sis\snow\sloading\sand\sregistering\sbut\sit's\sstill\suntested. +D 2026-03-04T00:37:55.997 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -572,7 +572,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6 -F ext/wasm/GNUmakefile a64605e68d4561f440bdc538e01364210f75052469d6ea21bf2843f1539144e9 +F ext/wasm/GNUmakefile a0afdb3e6306997de1e123e8b8270affb744519c9e384dbc277d0af6a6ba1c91 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -593,12 +593,12 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js ccd8ece4b4580d2a70996218f28e810d70a86f5e2795f4d4a75f0603af24aef6 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 0ec326d4c66c7af5a4c79a1be65c07c1e1844e641f133ba026a4a01eb1a37c56 w ext/wasm/api/sqlite3-opfs-async-proxy.js +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 041f24cd61d470072303b0cd1bb73dd2928bf2c96c0fbf3d3f4a39024c4e9ae7 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 1abe919013d759b311e7e3f5c4a326f9027b1b6a0fbe89db5f303e2ff109d5d4 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js ab2a73f41a2e109f25fd3159531e801ca14c782c297b2fc1836e6a02b795321b +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 072abf5bd30e5a611f9d15039bb41795fb517b5ade1b1973cbdd70c171a1f822 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e44649cf72b303b857ce4d7e33b13b88a1f97009514b1b84d2e9c2ecf511fca1 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -2190,8 +2190,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 5910ed7ac843aa1fdb103ddcfaf6270d12cb39199205fc0b9f721fbf7fa2f851 -R d999a6bf319e44a9103bde13f0ab87af +P 1e0b72631aecb0bb72f4089116da221e6c4abf962db589de08132cd52c2be0e2 +R 32d905549a7b0f1874fdf6ef84b0e8af U stephan -Z 097c558a7425489e055f6ff0be38c909 +Z 466cb2ea030b3f6732fe2aeb3d4128c0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9be206dbe2..558d2665e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e0b72631aecb0bb72f4089116da221e6c4abf962db589de08132cd52c2be0e2 +9a471f7491a371052bf1785098ba966dd0d03503e7d8b9fbcd65f07b038e5021 From 6d6bf30735240776ecbfc6ebd86a267e47f06fe0 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 00:48:03 +0000 Subject: [PATCH 06/32] A potential fix for the probable breakage of bundler-friendly builds in the previous check-in. Pending review from someone who uses those tools. FossilOrigin-Name: 8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec --- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 2 +- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index c53bebdfe1..6113657021 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -200,7 +200,7 @@ const installOpfsVfs = function callee(options){ const workerArgs = '?vfs=opfs-wl'; const W = //#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js"+workerArgs, import.meta.url)); + new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); //#elif target:es6-module new Worker(new URL(options.proxyUri+workerArgs, import.meta.url)); //#else diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 934564c5ea..216cd83a2a 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -210,7 +210,7 @@ const installOpfsVfs = function callee(options){ const workerArgs = '?vfs=opfs'; const W = //#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js"+workerArgs, import.meta.url)); + new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); //#elif target:es6-module new Worker(new URL(options.proxyUri+workerArgs, import.meta.url)); //#else diff --git a/manifest b/manifest index 2f56d2c256..037a053c93 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C opfs-wl\sis\snow\sloading\sand\sregistering\sbut\sit's\sstill\suntested. -D 2026-03-04T00:37:55.997 +C A\spotential\sfix\sfor\sthe\sprobable\sbreakage\sof\sbundler-friendly\sbuilds\sin\sthe\sprevious\scheck-in.\sPending\sreview\sfrom\ssomeone\swho\suses\sthose\stools. +D 2026-03-04T00:48:03.758 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -597,8 +597,8 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 041f24cd61d470072303b0cd1bb73dd2 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 072abf5bd30e5a611f9d15039bb41795fb517b5ade1b1973cbdd70c171a1f822 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e44649cf72b303b857ce4d7e33b13b88a1f97009514b1b84d2e9c2ecf511fca1 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 8c79bf3ea89102772225e6539c030787ea85203b6cc0354c545df69d81a2d910 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b2b4c7570ec816c49e7b3293f451992bd7fb7154dcd4cec6fb05d032c6cedf52 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -2190,8 +2190,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 1e0b72631aecb0bb72f4089116da221e6c4abf962db589de08132cd52c2be0e2 -R 32d905549a7b0f1874fdf6ef84b0e8af +P 9a471f7491a371052bf1785098ba966dd0d03503e7d8b9fbcd65f07b038e5021 +R 8aa8acc5e3e3610acfaa4dc12ceb0c5d U stephan -Z 466cb2ea030b3f6732fe2aeb3d4128c0 +Z dc633fdeffefdb99fdb4ea173110ea63 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 558d2665e6..3a2f818401 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a471f7491a371052bf1785098ba966dd0d03503e7d8b9fbcd65f07b038e5021 +8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec From 96daad10faab55aca79d53fe78bd5d1bcaaf18df Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 11:37:39 +0000 Subject: [PATCH 07/32] Consolidate much of the OPFS utility code into a new file for use by two of the OPFS VFSes. FossilOrigin-Name: db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7 --- ext/wasm/GNUmakefile | 1 + ...mon.c-pp.js => opfs-common-inline.c-pp.js} | 22 +- ext/wasm/api/opfs-common-shared.c-pp.js | 430 ++++++++++++++++ ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 13 +- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 481 ++---------------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 466 ++--------------- manifest | 21 +- manifest.uuid | 2 +- 8 files changed, 544 insertions(+), 892 deletions(-) rename ext/wasm/api/{opfs-common.c-pp.js => opfs-common-inline.c-pp.js} (89%) create mode 100644 ext/wasm/api/opfs-common-shared.c-pp.js diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 7e48113ed2..1b7c5269e8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -911,6 +911,7 @@ ifeq (0,$(wasm-bare-bones)) sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js endif sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js +sqlite3-api.jses += $(dir.api)/opfs-common-shared.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js diff --git a/ext/wasm/api/opfs-common.c-pp.js b/ext/wasm/api/opfs-common-inline.c-pp.js similarity index 89% rename from ext/wasm/api/opfs-common.c-pp.js rename to ext/wasm/api/opfs-common-inline.c-pp.js index 1a7818927a..dcff8a05bd 100644 --- a/ext/wasm/api/opfs-common.c-pp.js +++ b/ext/wasm/api/opfs-common-inline.c-pp.js @@ -1,26 +1,14 @@ //#if nope /** This file is for preprocessor #include into the "opfs" and - "opfs-wl" impls, as well as their async-proxy part. + "opfs-wl" impls, as well as their async-proxy part. It must be + inlined in those files, as opposed to being a shared copy in the + library, because (A) the async proxy does not load the library and + (B) it references an object which is local to each of those files + but which has a 99% identical structure for each. */ //#endif - -//#if not defined opfs-async-proxy -/** - TODO: move the sqlite3.opfs (private/internal) namespace object - init from sqlite3-vfs-opfs*.js into here. That namespace gets - removed from the sqlite3 namespace in the final stages of library - bootstrapping except in test runs, where it's retained so that - tests can clean up OPFS so their test cases work (the down-side of - them being persistent). -*/ -//#endif not defined opfs-async-proxy - -// This function won't work as-is if we #include it, but the -// missing elements have not yet been identified. const initS11n = function(){ - /* This function is needed by the "opfs" and "opfs-wl" VFSes - and sqlite-opfs-async-proxy.js (used by those of those). */ /** This proxy de/serializes cross-thread function arguments and output-pointer values via the state.sabIO SharedArrayBuffer, diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js new file mode 100644 index 0000000000..56af5585bc --- /dev/null +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -0,0 +1,430 @@ +//#if not target:node +/* + 2026-03-04 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js. It + creates a private/internal sqlite3.opfs namespace common to the two + and used (only) by them and the test framework. It is not part of + the public API. +*/ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + 'use strict'; + const toss = sqlite3.util.toss; + const toss3 = sqlite3.util.toss3; + const capi = sqlite3.capi; + const util = sqlite3.util; + const wasm = sqlite3.wasm; + + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + + This is an internal/private namespace intended for use solely + by the OPFS VFSes and test code for them. The library bootstrapping + process removes this object in non-testing contexts. + + */ + const opfsUtil = sqlite3.opfs = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + opfsUtil.thisThreadHasOPFS = ()=>{ + return globalThis.FileSystemHandle && + globalThis.FileSystemDirectoryHandle && + globalThis.FileSystemFileHandle && + globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator?.storage?.getDirectory; + }; + + /** + Must be called by the OPFS VFSes immediately after they determine + whether OPFS is available by calling + thisThreadHasOPFS(). Resolves to the OPFS storage root directory + and sets opfsUtil.rootDirectory to that value. + */ + opfsUtil.getRootDir = async function f(){ + return f.promise ??= navigator.storage.getDirectory().then(d=>{ + opfsUtil.rootDirectory = d; + return d; + }).catch(e=>{ + delete f.promise; + throw e; + }); + }; + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg + is true, the result is returned as an array of path elements, + else an absolute path string is returned. + */ + opfsUtil.getResolvedPath = function(filename,splitIt){ + const p = new URL(filename, "file://irrelevant").pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = await opfsUtil.getRootDir(); + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't or if an + exception is intercepted while trying to make the + determination. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + opfsUtil.randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(""); + /* + An alternative impl. with an unpredictable length + but much simpler: + + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) + */ + }; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + const dir = await opfsUtil.getRootDir(); + await doDir(dir, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). + */ + opfsUtil.rmfr = async function(){ + const rd = await opfsUtil.getRootDir(); + const dir = rd, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each + entry. The argument may be either a callback function or an + options object with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: await opfsUtil.getRootDir() + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + /** + Impl of opfsUtil.importDb() when it's given a function as its + second argument. + */ + const importDbChunked = async function(filename, callback){ + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + let sah = await hFile.createSyncAccessHandle(); + let nWrote = 0, chunk, checkedHeader = false, err = false; + try{ + sah.truncate(0); + while( undefined !== (chunk = await callback()) ){ + if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); + if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ + util.affirmDbHeader(chunk); + checkedHeader = true; + } + sah.write(chunk, {at: nWrote}); + nWrote += chunk.byteLength; + } + if( nWrote < 512 || 0!==nWrote % 512 ){ + toss("Input size",nWrote,"is not correct for an SQLite database."); + } + if( !checkedHeader ){ + const header = new Uint8Array(20); + sah.read( header, {at: 0} ); + util.affirmDbHeader( header ); + } + sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); + return nWrote; + }catch(e){ + await sah.close(); + sah = undefined; + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally { + if( sah ) await sah.close(); + } + }; + + /** + Asynchronously imports the given bytes (a byte array or + ArrayBuffer) into the given database file. + + Results are undefined if the given db name refers to an opened + db. + + If passed a function for its second argument, its behaviour + changes: imports its data in chunks fed to it by the given + callback function. It calls the callback (which may be async) + repeatedly, expecting either a Uint8Array or ArrayBuffer (to + denote new input) or undefined (to denote EOF). For so long as + the callback continues to return non-undefined, it will append + incoming data to the given VFS-hosted database file. When + called this way, the resolved value of the returned Promise is + the number of bytes written to the target file. + + It very specifically requires the input to be an SQLite3 + database and throws if that's not the case. It does so in + order to prevent this function from taking on a larger scope + than it is specifically intended to. i.e. we do not want it to + become a convenience for importing arbitrary files into OPFS. + + This routine rewrites the database header bytes in the output + file (not the input array) to force disabling of WAL mode. + + On error this throws and the state of the input file is + undefined (it depends on where the exception was triggered). + + On success, resolves to the number of bytes written. + */ + opfsUtil.importDb = async function(filename, bytes){ + if( bytes instanceof Function ){ + return importDbChunked(filename, bytes); + } + if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); + util.affirmIsDb(bytes); + const n = bytes.byteLength; + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + let sah, err, nWrote = 0; + try { + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + sah = await hFile.createSyncAccessHandle(); + sah.truncate(0); + nWrote = sah.write(bytes, {at: 0}); + if(nWrote != n){ + toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); + } + sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; + return nWrote; + }catch(e){ + if( sah ){ await sah.close(); sah = undefined; } + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally{ + if( sah ) await sah.close(); + } + }; + + /** + Checks for features required for OPFS VFSes and throws with a + descriptive error message if they're not found. This is intended + to be run as part of async VFS installation steps. + */ + opfsUtil.vfsInstallationFeatureCheck = function(){ + if(!globalThis.SharedArrayBuffer + || !globalThis.Atomics){ + throw new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ + "The server must emit the COOP/COEP response headers to enable those. "+ + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep"); + }else if('undefined'===typeof WorkerGlobalScope){ + throw new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ + "because it requires Atomics.wait()."); + }else if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ + throw new newError("Missing required OPFS APIs."); + } + }; + +}/*sqlite3ApiBootstrap.initializers*/); +//#else +/* + The OPFS SAH Pool parts are elided from builds targeting node.js. +*/ +//#endif target:node diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 740be8938b..98eeb1639b 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -46,19 +46,16 @@ theFunc().then(...) is not compatible with the change to synchronous, but we do do not use those APIs that way. i.e. we don't _need_ to change anything for this, but at some point (after Chrome - versions (approximately) 104-107 are extinct) should change our + versions (approximately) 104-107 are extinct) we should change our usage of those methods to remove the "await". */ "use strict"; const wPost = (type,...args)=>postMessage({type, payload:args}); -//#if nope const urlParams = new URL(globalThis.location.href).searchParams; if( !urlParams.has('vfs') ){ throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); } const isWebLocker = 'opfs-wl'===urlParams.get('vfs'); -const msgKeyPrefix = 'opfs-'; //isWebLocker ? 'opfs-wl-' : 'opfs-'; -//#endif const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; if(globalThis.window === globalThis){ @@ -73,6 +70,9 @@ const installAsyncProxy = function(){ this API. */ const state = Object.create(null); +//#define opfs-async-proxy +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-async-proxy /** verbose: @@ -326,7 +326,7 @@ const installAsyncProxy = function(){ } log("Got",opName+"() sync handle for",fh.filenameAbs, 'in',performance.now() - t,'ms'); - if(!fh.xLock && !state.lock/*set by opfs-wl*/){ + if(!isWebLocker && !fh.xLock){ __implicitLocks.add(fh.fid); log("Acquired implicit lock for",opName+"()",fh.fid,fh.filenameAbs); } @@ -611,9 +611,6 @@ const installAsyncProxy = function(){ } }/*vfsAsyncImpls*/; -//#define opfs-async-proxy -//#include api/opfs-common.c-pp.js - /** Starts a new WebLock request. */ diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 6113657021..e2f4ff2f93 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -20,12 +20,7 @@ This file is intended to be appended to the main sqlite3 JS deliverable somewhere after sqlite3-api-oo1.js. - TODOs (2026-0303): - - - Move pieces of this file which are common to it, - sqlite3-vfs-opfs.c-pp.js, and/or sqlite3-opfs-async-proxy.js into - separate files and #include them in each using the preprocessor. - e.g. the s11n namespace object is duplicated in all three files. + TODOs (2026-03-03): - For purposes of tester1.js we need to figure out which of these VFSes will install the (internal-use-only) sqlite3.opfs utility code @@ -39,8 +34,11 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") + /* Gets removed from sqlite3 during bootstrap, so we need an + early reference to it. */; /** - installOpfsVfs() returns a Promise which, on success, installs an + installOpfsWlVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via sqlite3ApiBootstrap.initializers or an equivalent mechanism. @@ -67,27 +65,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ On success, the Promise resolves to the top-most sqlite3 namespace object. */ -const installOpfsVfs = function callee(options){ - if(!globalThis.SharedArrayBuffer - || !globalThis.Atomics){ - return Promise.reject( - new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ - "The server must emit the COOP/COEP response headers to enable those. "+ - "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") - ); - }else if('undefined'===typeof WorkerGlobalScope){ - return Promise.reject( - new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ - "because it requires Atomics.wait().") - ); - }else if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - return Promise.reject( - new Error("Missing required OPFS APIs.") - ); +const installOpfsWlVfs = function callee(options){ + try{ + opfsUtil.vfsInstallationFeatureCheck(); + }catch(e){ + return Promise.reject(e); } const nu = (...obj)=>Object.assign(Object.create(null),...obj); options = nu(options); @@ -129,31 +111,34 @@ const installOpfsVfs = function callee(options){ const sqlite3_file = capi.sqlite3_file; const sqlite3_io_methods = capi.sqlite3_io_methods; /** - Generic utilities for working with OPFS. This will get filled out - by the Promise setup and, on success, installed as sqlite3.opfs. + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. - ACHTUNG: do not rely on these APIs in client code. They are - experimental and subject to change or removal as the - OPFS-specific sqlite3_vfs evolves. - */ - const opfsUtil = Object.create(null); + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. - /** - Returns true if _this_ thread has access to the OPFS APIs. + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. */ - const thisThreadHasOPFS = ()=>{ - return globalThis.FileSystemHandle && - globalThis.FileSystemDirectoryHandle && - globalThis.FileSystemFileHandle && - globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator?.storage?.getDirectory; - }; + const state = Object.create(null); + const metrics = Object.create(null); +//#define opfs-has-metrics +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-has-metrics - /** - Not part of the public API. Solely for internal/development - use. - */ - opfsUtil.metrics = { + const vfsMetrics = { dump: function(){ let k, n = 0, t = 0, w = 0; for(k in state.opIds){ @@ -254,29 +239,7 @@ const installOpfsVfs = function callee(options){ environment or the other when sqlite3_os_end() is called (_if_ it gets called at all in a wasm build, which is undefined). */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); state.verbose = options.verbose; state.littleEndian = (()=>{ const buffer = new ArrayBuffer(2); @@ -331,7 +294,6 @@ const installOpfsVfs = function callee(options){ + state.sabS11nSize/* argument/result serialization block */ ); state.opIds = Object.create(null); - const metrics = Object.create(null); { /* Indexes for use in our SharedArrayBuffer... */ let i = 0; @@ -378,7 +340,7 @@ const installOpfsVfs = function callee(options){ state.sabOP = new SharedArrayBuffer( i * 4/* ==sizeof int32, noting that Atomics.wait() and friends can only function on Int32Array views of an SAB. */); - opfsUtil.metrics.reset(); + vfsMetrics.reset(); } /** SQLITE_xxx constants to export to the async worker @@ -499,6 +461,7 @@ const installOpfsVfs = function callee(options){ return rc; }; +//#if nope /** Not part of the public API. Only for test/development use. */ @@ -512,35 +475,7 @@ const installOpfsVfs = function callee(options){ W.postMessage({type: 'opfs-async-restart'}); } }; - -//#define opfs-has-metrics -//#include api/opfs-common.c-pp.js - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(""); - /* - An alternative impl. with an unpredictable length - but much simpler: - - Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) - */ - }; +//#endif /** Map of sqlite3_file pointers to objects constructed by xOpen(). @@ -770,7 +705,7 @@ const installOpfsVfs = function callee(options){ mTimeStart('xOpen'); let opfsFlags = 0; if(0===zName){ - zName = randomFilename(); + zName = opfsUtil.randomFilename(); }else if(wasm.isPtr(zName)){ if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ /* -----------------------^^^^^ MUST pass the untranslated @@ -832,322 +767,6 @@ const installOpfsVfs = function callee(options){ }; } - /** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg - is true, the result is returned as an array of path elements, - else an absolute path string is returned. - */ - opfsUtil.getResolvedPath = function(filename,splitIt){ - const p = new URL(filename, "file://irrelevant").pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; - }; - - /** - Takes the absolute path to a filesystem element. Returns an - array of [handleOfContainingDir, filename]. If the 2nd argument - is truthy then each directory element leading to the file is - created along the way. Throws if any creation or resolution - fails. - */ - opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ - const path = opfsUtil.getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = opfsUtil.rootDirectory; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; - }; - - /** - Creates the given directory name, recursively, in - the OPFS filesystem. Returns true if it succeeds or the - directory already exists, else false. - */ - opfsUtil.mkdir = async function(absDirName){ - try { - await opfsUtil.getDirForFilename(absDirName+"/filepart", true); - return true; - }catch(e){ - //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); - return false; - } - }; - /** - Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't or if an - exception is intercepted while trying to make the - determination. - */ - opfsUtil.entryExists = async function(fsEntryName){ - try { - const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); - await dh.getFileHandle(fn); - return true; - }catch(e){ - return false; - } - }; - - /** - Generates a random ASCII string, intended for use as a - temporary file name. Its argument is the length of the string, - defaulting to 16. - */ - opfsUtil.randomFilename = randomFilename; - - /** - Returns a promise which resolves to an object which represents - all files and directories in the OPFS tree. The top-most object - has two properties: `dirs` is an array of directory entries - (described below) and `files` is a list of file names for all - files in that directory. - - Traversal starts at sqlite3.opfs.rootDirectory. - - Each `dirs` entry is an object in this form: - - ``` - { name: directoryName, - dirs: [...subdirs], - files: [...file names] - } - ``` - - The `files` and `subdirs` entries are always set but may be - empty arrays. - - The returned object has the same structure but its `name` is - an empty string. All returned objects are created with - Object.create(null), so have no prototype. - - Design note: the entries do not contain more information, - e.g. file sizes, because getting such info is not only - expensive but is subject to locking-related errors. - */ - opfsUtil.treeList = async function(){ - const doDir = async function callee(dirHandle,tgt){ - tgt.name = dirHandle.name; - tgt.dirs = []; - tgt.files = []; - for await (const handle of dirHandle.values()){ - if('directory' === handle.kind){ - const subDir = Object.create(null); - tgt.dirs.push(subDir); - await callee(handle, subDir); - }else{ - tgt.files.push(handle.name); - } - } - }; - const root = Object.create(null); - await doDir(opfsUtil.rootDirectory, root); - return root; - }; - - /** - Irrevocably deletes _all_ files in the current origin's OPFS. - Obviously, this must be used with great caution. It may throw - an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which the underlying - APIs will throw are not documented (so we cannot tell you what - they are). - */ - opfsUtil.rmfr = async function(){ - const dir = opfsUtil.rootDirectory, opt = {recurse: true}; - for await (const handle of dir.values()){ - dir.removeEntry(handle.name, opt); - } - }; - - /** - Deletes the given OPFS filesystem entry. As this environment - has no notion of "current directory", the given name must be an - absolute path. If the 2nd argument is truthy, deletion is - recursive (use with caution!). - - The returned Promise resolves to true if the deletion was - successful, else false (but...). The OPFS API reports the - reason for the failure only in human-readable form, not - exceptions which can be type-checked to determine the - failure. Because of that... - - If the final argument is truthy then this function will - propagate any exception on error, rather than returning false. - */ - opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ - try { - const [hDir, filenamePart] = - await opfsUtil.getDirForFilename(fsEntryName, false); - await hDir.removeEntry(filenamePart, {recursive}); - return true; - }catch(e){ - if(throwOnError){ - throw new Error("unlink(",arguments[0],") failed: "+e.message,{ - cause: e - }); - } - return false; - } - }; - - /** - Traverses the OPFS filesystem, calling a callback for each - entry. The argument may be either a callback function or an - options object with any of the following properties: - - - `callback`: function which gets called for each filesystem - entry. It gets passed 3 arguments: 1) the - FileSystemFileHandle or FileSystemDirectoryHandle of each - entry (noting that both are instanceof FileSystemHandle). 2) - the FileSystemDirectoryHandle of the parent directory. 3) the - current depth level, with 0 being at the top of the tree - relative to the starting directory. If the callback returns a - literal false, as opposed to any other falsy value, traversal - stops without an error. Any exceptions it throws are - propagated. Results are undefined if the callback manipulate - the filesystem (e.g. removing or adding entries) because the - how OPFS iterators behave in the face of such changes is - undocumented. - - - `recursive` [bool=true]: specifies whether to recurse into - subdirectories or not. Whether recursion is depth-first or - breadth-first is unspecified! - - - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] - specifies the starting directory. - - If this function is passed a function, it is assumed to be the - callback. - - Returns a promise because it has to (by virtue of being async) - but that promise has no specific meaning: the traversal it - performs is synchronous. The promise must be used to catch any - exceptions propagated by the callback, however. - */ - opfsUtil.traverse = async function(opt){ - const defaultOpt = { - recursive: true, - directory: opfsUtil.rootDirectory - }; - if('function'===typeof opt){ - opt = {callback:opt}; - } - opt = Object.assign(defaultOpt, opt||{}); - const doDir = async function callee(dirHandle, depth){ - for await (const handle of dirHandle.values()){ - if(false === opt.callback(handle, dirHandle, depth)) return false; - else if(opt.recursive && 'directory' === handle.kind){ - if(false === await callee(handle, depth + 1)) break; - } - } - }; - doDir(opt.directory, 0); - }; - - /** - impl of importDb() when it's given a function as its second - argument. - */ - const importDbChunked = async function(filename, callback){ - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - let sah = await hFile.createSyncAccessHandle(); - let nWrote = 0, chunk, checkedHeader = false, err = false; - try{ - sah.truncate(0); - while( undefined !== (chunk = await callback()) ){ - if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); - if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ - util.affirmDbHeader(chunk); - checkedHeader = true; - } - sah.write(chunk, {at: nWrote}); - nWrote += chunk.byteLength; - } - if( nWrote < 512 || 0!==nWrote % 512 ){ - toss("Input size",nWrote,"is not correct for an SQLite database."); - } - if( !checkedHeader ){ - const header = new Uint8Array(20); - sah.read( header, {at: 0} ); - util.affirmDbHeader( header ); - } - sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); - return nWrote; - }catch(e){ - await sah.close(); - sah = undefined; - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally { - if( sah ) await sah.close(); - } - }; - - /** - Asynchronously imports the given bytes (a byte array or - ArrayBuffer) into the given database file. - - Results are undefined if the given db name refers to an opened - db. - - If passed a function for its second argument, its behaviour - changes: imports its data in chunks fed to it by the given - callback function. It calls the callback (which may be async) - repeatedly, expecting either a Uint8Array or ArrayBuffer (to - denote new input) or undefined (to denote EOF). For so long as - the callback continues to return non-undefined, it will append - incoming data to the given VFS-hosted database file. When - called this way, the resolved value of the returned Promise is - the number of bytes written to the target file. - - It very specifically requires the input to be an SQLite3 - database and throws if that's not the case. It does so in - order to prevent this function from taking on a larger scope - than it is specifically intended to. i.e. we do not want it to - become a convenience for importing arbitrary files into OPFS. - - This routine rewrites the database header bytes in the output - file (not the input array) to force disabling of WAL mode. - - On error this throws and the state of the input file is - undefined (it depends on where the exception was triggered). - - On success, resolves to the number of bytes written. - */ - opfsUtil.importDb = async function(filename, bytes){ - if( bytes instanceof Function ){ - return importDbChunked(filename, bytes); - } - if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); - util.affirmIsDb(bytes); - const n = bytes.byteLength; - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - let sah, err, nWrote = 0; - try { - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - sah = await hFile.createSyncAccessHandle(); - sah.truncate(0); - nWrote = sah.write(bytes, {at: 0}); - if(nWrote != n){ - toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); - } - sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; - return nWrote; - }catch(e){ - if( sah ){ await sah.close(); sah = undefined; } - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally{ - if( sah ) await sah.close(); - } - }; - if(sqlite3.oo1){ const OpfsWlDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -1270,13 +889,11 @@ const installOpfsVfs = function callee(options){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); } - if(thisThreadHasOPFS()){ - navigator.storage.getDirectory().then((d)=>{ + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ W.onerror = W._originalOnError; delete W._originalOnError; - //sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); + log("End of OPFS-WL sqlite3_vfs setup.", opfsVfs); promiseResolve(); }).catch(promiseReject); }else{ @@ -1301,22 +918,22 @@ const installOpfsVfs = function callee(options){ }/*W.onmessage()*/; })/*thePromise*/; return thePromise; -}/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = +}/*installOpfsWlVfs()*/; +installOpfsWlVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ - let proxyJs = installOpfsVfs.defaultProxyUri; + let proxyJs = installOpfsWlVfs.defaultProxyUri; if( sqlite3?.scriptInfo?.sqlite3Dir ){ - installOpfsVfs.defaultProxyUri = + installOpfsWlVfs.defaultProxyUri = sqlite3.scriptInfo.sqlite3Dir + proxyJs; - //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); + //sqlite3.config.warn("installOpfsWlVfs.defaultProxyUri =",installOpfsWlVfs.defaultProxyUri); } - return installOpfsVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); + return installOpfsWlVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install OPFS-WL sqlite3_vfs:",e.message); }); }catch(e){ - sqlite3.config.error("installOpfsVfs() exception:",e); + sqlite3.config.error("installOpfsWlVfs() exception:",e); return Promise.reject(e); } }); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 216cd83a2a..7ba7427d0a 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -20,7 +20,10 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ -/** + const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") + /* Gets removed from sqlite3 during bootstrap, so we need an + early reference to it. */; + /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via @@ -74,30 +77,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ object. */ const installOpfsVfs = function callee(options){ - if(!globalThis.SharedArrayBuffer - || !globalThis.Atomics){ - return Promise.reject( - new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ - "The server must emit the COOP/COEP response headers to enable those. "+ - "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") - ); - }else if('undefined'===typeof WorkerGlobalScope){ - return Promise.reject( - new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ - "because it requires Atomics.wait().") - ); - }else if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - return Promise.reject( - new Error("Missing required OPFS APIs.") - ); - } - if(!options || 'object'!==typeof options){ - options = Object.create(null); + try{ + opfsUtil.vfsInstallationFeatureCheck(); + }catch(e){ + return Promise.reject(e); } + options = Object.assign(Object.create(null), options); const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('opfs-disable')){ //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); @@ -113,12 +98,10 @@ const installOpfsVfs = function callee(options){ if(undefined===options.proxyUri){ options.proxyUri = callee.defaultProxyUri; } - - //sqlite3.config.warn("OPFS options =",options,globalThis.location); - if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } + //sqlite3.config.warn("OPFS options =",options,globalThis.location); const thePromise = new Promise(function(promiseResolve_, promiseReject_){ const loggers = [ sqlite3.config.error, @@ -138,32 +121,36 @@ const installOpfsVfs = function callee(options){ const sqlite3_vfs = capi.sqlite3_vfs; const sqlite3_file = capi.sqlite3_file; const sqlite3_io_methods = capi.sqlite3_io_methods; - /** - Generic utilities for working with OPFS. This will get filled out - by the Promise setup and, on success, installed as sqlite3.opfs. - - ACHTUNG: do not rely on these APIs in client code. They are - experimental and subject to change or removal as the - OPFS-specific sqlite3_vfs evolves. - */ - const opfsUtil = Object.create(null); /** - Returns true if _this_ thread has access to the OPFS APIs. - */ - const thisThreadHasOPFS = ()=>{ - return globalThis.FileSystemHandle && - globalThis.FileSystemDirectoryHandle && - globalThis.FileSystemFileHandle && - globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator?.storage?.getDirectory; - }; + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. - /** - Not part of the public API. Solely for internal/development - use. + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. + + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. */ - opfsUtil.metrics = { + const state = Object.create(null); + const metrics = Object.create(null); +//#define opfs-has-metrics +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-has-metrics + + const vfsMetrics = { dump: function(){ let k, n = 0, t = 0, w = 0; for(k in state.opIds){ @@ -264,29 +251,6 @@ const installOpfsVfs = function callee(options){ environment or the other when sqlite3_os_end() is called (_if_ it gets called at all in a wasm build, which is undefined). */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); state.verbose = options.verbose; state.littleEndian = (()=>{ const buffer = new ArrayBuffer(2); @@ -341,7 +305,6 @@ const installOpfsVfs = function callee(options){ + state.sabS11nSize/* argument/result serialization block */ ); state.opIds = Object.create(null); - const metrics = Object.create(null); { /* Indexes for use in our SharedArrayBuffer... */ let i = 0; @@ -379,7 +342,7 @@ const installOpfsVfs = function callee(options){ state.sabOP = new SharedArrayBuffer( i * 4/* ==sizeof int32, noting that Atomics.wait() and friends can only function on Int32Array views of an SAB. */); - opfsUtil.metrics.reset(); + vfsMetrics.reset(); } /** SQLITE_xxx constants to export to the async worker @@ -495,6 +458,7 @@ const installOpfsVfs = function callee(options){ return rc; }; +//#if nope /** Not part of the public API. Only for test/development use. */ @@ -508,35 +472,7 @@ const installOpfsVfs = function callee(options){ W.postMessage({type: 'opfs-async-restart'}); } }; - -//#define opfs-has-metrics -//#include api/opfs-common.c-pp.js - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(""); - /* - An alternative impl. with an unpredictable length - but much simpler: - - Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) - */ - }; +//#endif /** Map of sqlite3_file pointers to objects constructed by xOpen(). @@ -743,7 +679,7 @@ const installOpfsVfs = function callee(options){ mTimeStart('xOpen'); let opfsFlags = 0; if(0===zName){ - zName = randomFilename(); + zName = opfsUtil.randomFilename(); }else if(wasm.isPtr(zName)){ if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ /* -----------------------^^^^^ MUST pass the untranslated @@ -805,322 +741,6 @@ const installOpfsVfs = function callee(options){ }; } - /** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg - is true, the result is returned as an array of path elements, - else an absolute path string is returned. - */ - opfsUtil.getResolvedPath = function(filename,splitIt){ - const p = new URL(filename, "file://irrelevant").pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; - }; - - /** - Takes the absolute path to a filesystem element. Returns an - array of [handleOfContainingDir, filename]. If the 2nd argument - is truthy then each directory element leading to the file is - created along the way. Throws if any creation or resolution - fails. - */ - opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ - const path = opfsUtil.getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = opfsUtil.rootDirectory; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; - }; - - /** - Creates the given directory name, recursively, in - the OPFS filesystem. Returns true if it succeeds or the - directory already exists, else false. - */ - opfsUtil.mkdir = async function(absDirName){ - try { - await opfsUtil.getDirForFilename(absDirName+"/filepart", true); - return true; - }catch(e){ - //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); - return false; - } - }; - /** - Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't or if an - exception is intercepted while trying to make the - determination. - */ - opfsUtil.entryExists = async function(fsEntryName){ - try { - const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); - await dh.getFileHandle(fn); - return true; - }catch(e){ - return false; - } - }; - - /** - Generates a random ASCII string, intended for use as a - temporary file name. Its argument is the length of the string, - defaulting to 16. - */ - opfsUtil.randomFilename = randomFilename; - - /** - Returns a promise which resolves to an object which represents - all files and directories in the OPFS tree. The top-most object - has two properties: `dirs` is an array of directory entries - (described below) and `files` is a list of file names for all - files in that directory. - - Traversal starts at sqlite3.opfs.rootDirectory. - - Each `dirs` entry is an object in this form: - - ``` - { name: directoryName, - dirs: [...subdirs], - files: [...file names] - } - ``` - - The `files` and `subdirs` entries are always set but may be - empty arrays. - - The returned object has the same structure but its `name` is - an empty string. All returned objects are created with - Object.create(null), so have no prototype. - - Design note: the entries do not contain more information, - e.g. file sizes, because getting such info is not only - expensive but is subject to locking-related errors. - */ - opfsUtil.treeList = async function(){ - const doDir = async function callee(dirHandle,tgt){ - tgt.name = dirHandle.name; - tgt.dirs = []; - tgt.files = []; - for await (const handle of dirHandle.values()){ - if('directory' === handle.kind){ - const subDir = Object.create(null); - tgt.dirs.push(subDir); - await callee(handle, subDir); - }else{ - tgt.files.push(handle.name); - } - } - }; - const root = Object.create(null); - await doDir(opfsUtil.rootDirectory, root); - return root; - }; - - /** - Irrevocably deletes _all_ files in the current origin's OPFS. - Obviously, this must be used with great caution. It may throw - an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which the underlying - APIs will throw are not documented (so we cannot tell you what - they are). - */ - opfsUtil.rmfr = async function(){ - const dir = opfsUtil.rootDirectory, opt = {recurse: true}; - for await (const handle of dir.values()){ - dir.removeEntry(handle.name, opt); - } - }; - - /** - Deletes the given OPFS filesystem entry. As this environment - has no notion of "current directory", the given name must be an - absolute path. If the 2nd argument is truthy, deletion is - recursive (use with caution!). - - The returned Promise resolves to true if the deletion was - successful, else false (but...). The OPFS API reports the - reason for the failure only in human-readable form, not - exceptions which can be type-checked to determine the - failure. Because of that... - - If the final argument is truthy then this function will - propagate any exception on error, rather than returning false. - */ - opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ - try { - const [hDir, filenamePart] = - await opfsUtil.getDirForFilename(fsEntryName, false); - await hDir.removeEntry(filenamePart, {recursive}); - return true; - }catch(e){ - if(throwOnError){ - throw new Error("unlink(",arguments[0],") failed: "+e.message,{ - cause: e - }); - } - return false; - } - }; - - /** - Traverses the OPFS filesystem, calling a callback for each - entry. The argument may be either a callback function or an - options object with any of the following properties: - - - `callback`: function which gets called for each filesystem - entry. It gets passed 3 arguments: 1) the - FileSystemFileHandle or FileSystemDirectoryHandle of each - entry (noting that both are instanceof FileSystemHandle). 2) - the FileSystemDirectoryHandle of the parent directory. 3) the - current depth level, with 0 being at the top of the tree - relative to the starting directory. If the callback returns a - literal false, as opposed to any other falsy value, traversal - stops without an error. Any exceptions it throws are - propagated. Results are undefined if the callback manipulate - the filesystem (e.g. removing or adding entries) because the - how OPFS iterators behave in the face of such changes is - undocumented. - - - `recursive` [bool=true]: specifies whether to recurse into - subdirectories or not. Whether recursion is depth-first or - breadth-first is unspecified! - - - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] - specifies the starting directory. - - If this function is passed a function, it is assumed to be the - callback. - - Returns a promise because it has to (by virtue of being async) - but that promise has no specific meaning: the traversal it - performs is synchronous. The promise must be used to catch any - exceptions propagated by the callback, however. - */ - opfsUtil.traverse = async function(opt){ - const defaultOpt = { - recursive: true, - directory: opfsUtil.rootDirectory - }; - if('function'===typeof opt){ - opt = {callback:opt}; - } - opt = Object.assign(defaultOpt, opt||{}); - const doDir = async function callee(dirHandle, depth){ - for await (const handle of dirHandle.values()){ - if(false === opt.callback(handle, dirHandle, depth)) return false; - else if(opt.recursive && 'directory' === handle.kind){ - if(false === await callee(handle, depth + 1)) break; - } - } - }; - doDir(opt.directory, 0); - }; - - /** - impl of importDb() when it's given a function as its second - argument. - */ - const importDbChunked = async function(filename, callback){ - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - let sah = await hFile.createSyncAccessHandle(); - let nWrote = 0, chunk, checkedHeader = false, err = false; - try{ - sah.truncate(0); - while( undefined !== (chunk = await callback()) ){ - if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); - if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ - util.affirmDbHeader(chunk); - checkedHeader = true; - } - sah.write(chunk, {at: nWrote}); - nWrote += chunk.byteLength; - } - if( nWrote < 512 || 0!==nWrote % 512 ){ - toss("Input size",nWrote,"is not correct for an SQLite database."); - } - if( !checkedHeader ){ - const header = new Uint8Array(20); - sah.read( header, {at: 0} ); - util.affirmDbHeader( header ); - } - sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); - return nWrote; - }catch(e){ - await sah.close(); - sah = undefined; - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally { - if( sah ) await sah.close(); - } - }; - - /** - Asynchronously imports the given bytes (a byte array or - ArrayBuffer) into the given database file. - - Results are undefined if the given db name refers to an opened - db. - - If passed a function for its second argument, its behaviour - changes: imports its data in chunks fed to it by the given - callback function. It calls the callback (which may be async) - repeatedly, expecting either a Uint8Array or ArrayBuffer (to - denote new input) or undefined (to denote EOF). For so long as - the callback continues to return non-undefined, it will append - incoming data to the given VFS-hosted database file. When - called this way, the resolved value of the returned Promise is - the number of bytes written to the target file. - - It very specifically requires the input to be an SQLite3 - database and throws if that's not the case. It does so in - order to prevent this function from taking on a larger scope - than it is specifically intended to. i.e. we do not want it to - become a convenience for importing arbitrary files into OPFS. - - This routine rewrites the database header bytes in the output - file (not the input array) to force disabling of WAL mode. - - On error this throws and the state of the input file is - undefined (it depends on where the exception was triggered). - - On success, resolves to the number of bytes written. - */ - opfsUtil.importDb = async function(filename, bytes){ - if( bytes instanceof Function ){ - return importDbChunked(filename, bytes); - } - if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); - util.affirmIsDb(bytes); - const n = bytes.byteLength; - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - let sah, err, nWrote = 0; - try { - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - sah = await hFile.createSyncAccessHandle(); - sah.truncate(0); - nWrote = sah.write(bytes, {at: 0}); - if(nWrote != n){ - toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); - } - sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; - return nWrote; - }catch(e){ - if( sah ){ await sah.close(); sah = undefined; } - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally{ - if( sah ) await sah.close(); - } - }; - if(sqlite3.oo1){ const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -1241,12 +861,10 @@ const installOpfsVfs = function callee(options){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); } - if(thisThreadHasOPFS()){ - navigator.storage.getDirectory().then((d)=>{ + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ W.onerror = W._originalOnError; delete W._originalOnError; - sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; log("End of OPFS sqlite3_vfs setup.", opfsVfs); promiseResolve(); }).catch(promiseReject); diff --git a/manifest b/manifest index 037a053c93..f334f15c5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\spotential\sfix\sfor\sthe\sprobable\sbreakage\sof\sbundler-friendly\sbuilds\sin\sthe\sprevious\scheck-in.\sPending\sreview\sfrom\ssomeone\swho\suses\sthose\stools. -D 2026-03-04T00:48:03.758 +C Consolidate\smuch\sof\sthe\sOPFS\sutility\scode\sinto\sa\snew\sfile\sfor\suse\sby\stwo\sof\sthe\sOPFS\sVFSes. +D 2026-03-04T11:37:39.256 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -572,7 +572,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6 -F ext/wasm/GNUmakefile a0afdb3e6306997de1e123e8b8270affb744519c9e384dbc277d0af6a6ba1c91 +F ext/wasm/GNUmakefile 4496669ca5785c7bfc7af76565f1dd45e8f9ec7529b58c485fd374088162bef1 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -584,7 +584,8 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 7ba933e8f1290cc65459dd371c0c9a031d96bdf14 F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 -F ext/wasm/api/opfs-common.c-pp.js 5540c58aee8fbbf2c67984e0a35e2b680a7603052f2cc442d407da3cc81c4819 +F ext/wasm/api/opfs-common-inline.c-pp.js b9c4e080698792cbc04ce9dd9dda7d8316c6db0262f74820706f98b352b949d5 w ext/wasm/api/opfs-common.c-pp.js +F ext/wasm/api/opfs-common-shared.c-pp.js 3f8f3f2ab4790fdd1e6d1d9224e232cef07f1c8753827c18bbba965dbe98795f F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -593,12 +594,12 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js ccd8ece4b4580d2a70996218f28e810d70a86f5e2795f4d4a75f0603af24aef6 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 041f24cd61d470072303b0cd1bb73dd2928bf2c96c0fbf3d3f4a39024c4e9ae7 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 25b856508ac94336419133c6ec10594f576b469f85cc69cde4c09cfa06a8e1c7 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 8c79bf3ea89102772225e6539c030787ea85203b6cc0354c545df69d81a2d910 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b2b4c7570ec816c49e7b3293f451992bd7fb7154dcd4cec6fb05d032c6cedf52 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 54f71e563dda30af73ed84ff9de03441537b2e8fb8d2ae2a0b0c8187f51db67a +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 2e2a72a40e2ad6ea92f52eb7adbd925d4acd874ffeecaa00b85234ad49862655 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -2190,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 9a471f7491a371052bf1785098ba966dd0d03503e7d8b9fbcd65f07b038e5021 -R 8aa8acc5e3e3610acfaa4dc12ceb0c5d +P 8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec +R 82203270087631af34878be94f4d1594 U stephan -Z dc633fdeffefdb99fdb4ea173110ea63 +Z 1548da853f7a79cab3a86616c1bf35cc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a2f818401..65aadb1877 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec +db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7 From 6ff9fcfad25f5eaab13c0e2a7a73fd4eecadf737 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 14:33:33 +0000 Subject: [PATCH 08/32] Baby steps in consolidating common OPFS VFS code. FossilOrigin-Name: b0dd23299e97ff975f213cb3a8b051f4d7b785b29def82e01f53427fdf77ecb6 --- ext/wasm/api/opfs-common-inline.c-pp.js | 11 +- ext/wasm/api/opfs-common-shared.c-pp.js | 232 ++++++++++++- ext/wasm/api/sqlite3-api-prologue.js | 1 + ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 2 +- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 304 +++--------------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 239 +++----------- manifest | 22 +- manifest.uuid | 2 +- 8 files changed, 335 insertions(+), 478 deletions(-) diff --git a/ext/wasm/api/opfs-common-inline.c-pp.js b/ext/wasm/api/opfs-common-inline.c-pp.js index dcff8a05bd..04ae17d555 100644 --- a/ext/wasm/api/opfs-common-inline.c-pp.js +++ b/ext/wasm/api/opfs-common-inline.c-pp.js @@ -8,6 +8,8 @@ but which has a 99% identical structure for each. */ //#endif +//#// vfs.metrics.enable is a refactoring crutch. +//#define vfs.metrics.enable=0 const initS11n = function(){ /** This proxy de/serializes cross-thread function arguments and @@ -88,7 +90,7 @@ const initS11n = function(){ is cleared after deserialization. */ state.s11n.deserialize = function(clear=false){ -//#if defined opfs-has-metrics +//#if vfs.metrics.enable ++metrics.s11n.deserialize.count; //#endif const t = performance.now(); @@ -116,7 +118,7 @@ const initS11n = function(){ } if(clear) viewU8[0] = 0; //log("deserialize:",argc, rc); -//#if defined opfs-has-metrics +//#if vfs.metrics.enable metrics.s11n.deserialize.time += performance.now() - t; //#endif return rc; @@ -136,7 +138,7 @@ const initS11n = function(){ */ state.s11n.serialize = function(...args){ const t = performance.now(); -//#if defined opfs-has-metrics +//#if vfs.metrics.enable ++metrics.s11n.serialize.count; //#endif if(args.length){ @@ -169,7 +171,7 @@ const initS11n = function(){ }else{ viewU8[0] = 0; } -//#if defined opfs-has-metrics +//#if vfs.metrics.enable metrics.s11n.serialize.time += performance.now() - t; //#endif }; @@ -185,4 +187,5 @@ const initS11n = function(){ //#endif return state.s11n; +//#undef vfs.metrics.enable }/*initS11n()*/; diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 56af5585bc..077f4c5a6a 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -19,7 +19,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; const toss = sqlite3.util.toss; - const toss3 = sqlite3.util.toss3; const capi = sqlite3.capi; const util = sqlite3.util; const wasm = sqlite3.wasm; @@ -422,9 +421,232 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; + /** + Populates the main state object used by "opfs" and "opfs-wl", and + transfered from those to their async counterpart. + + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. + + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. + + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. + */ + opfsUtil.createVfsStateObject = function(opfsVfs){ + if( !(opfsVfs instanceof capi.sqlite3_vfs) ){ + toss("Expecting a sqlite3_vfs instance"); + } + const vfsName = wasm.cstrToJs(opfsVfs.$zName); + const isWebLocker = 'opfs-wl'===vfsName; + const state = util.nu(); + /** + asyncIdleWaitTime is how long (ms) to wait, in the async proxy, + for each Atomics.wait() when waiting on inbound VFS API calls. + We need to wake up periodically to give the thread a chance to + do other things. If this is too high (e.g. 500ms) then even two + workers/tabs can easily run into locking errors. Some multiple + of this value is also used for determining how long to wait on + lock contention to free up. + */ + state.asyncIdleWaitTime = isWebLocker ? 100 : 150; + + /** + Whether the async counterpart should log exceptions to + the serialization channel. That produces a great deal of + noise for seemingly innocuous things like xAccess() checks + for missing files, so this option may have one of 3 values: + + 0 = no exception logging. + + 1 = only log exceptions for "significant" ops like xOpen(), + xRead(), and xWrite(). + + 2 = log all exceptions. + */ + state.asyncS11nExceptions = 1; + /* Size of file I/O buffer block. 64k = max sqlite3 page size, and + xRead/xWrite() will never deal in blocks larger than that. */ + state.fileBufferSize = 1024 * 64; + state.sabS11nOffset = state.fileBufferSize; + /** + The size of the block in our SAB for serializing arguments and + result values. Needs to be large enough to hold serialized + values of any of the proxied APIs. Filenames are the largest + part but are limited to opfsVfs.$mxPathname bytes. We also + store exceptions there, so it needs to be long enough to hold + a reasonably long exception string. + */ + state.sabS11nSize = opfsVfs.$mxPathname * 2; + /** + The SAB used for all data I/O between the synchronous and + async halves (file i/o and arg/result s11n). + */ + state.sabIO = new SharedArrayBuffer( + state.fileBufferSize/* file i/o block */ + + state.sabS11nSize/* argument/result serialization block */ + ); + state.opIds = Object.create(null); + { + /* + Maintenance reminder: + + Some of these fields are only for use by the "opfs-wl" VFS, + but they must also be set up for the "ofps" VFS so that the + sizes and offsets calculated here are consistent in the async + proxy. Hypothetically they could differ and it would cope + but... why invite disaster over eliding a few superfluous (for + "opfs') properties? + */ + /* Indexes for use in our SharedArrayBuffer... */ + let i = 0; + /* SAB slot used to communicate which operation is desired + between both workers. This worker writes to it and the other + listens for changes. */ + state.opIds.whichOp = i++; + /* Slot for storing return values. This worker listens to that + slot and the other worker writes to it. */ + state.opIds.rc = i++; + /* Each function gets an ID which this worker writes to + the whichOp slot. The async-api worker uses Atomic.wait() + on the whichOp slot to figure out which operation to run + next. */ + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xDeleteNoWait = i++; + state.opIds.xFileSize = i++; + state.opIds.xLock = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xUnlock = i++; + state.opIds.xWrite = i++; + state.opIds.mkdir = i++; + state.opIds.lockControl = i++ /* opfs-wl signals the intent to lock here */; + /** Internal signals which are used only during development and + testing via the dev console. */ + state.opIds['opfs-async-metrics'] = i++; + state.opIds['opfs-async-shutdown'] = i++; + /* The retry slot is used by the async part for wait-and-retry + semantics. Though we could hypothetically use the xSleep slot + for that, doing so might lead to undesired side effects. */ + state.opIds.retry = i++; + + /* Slots for submitting the lock type and receiving its acknowledgement. + Only used by "opfs-wl". */ + state.lock = util.nu({ + type: i++ /* SQLITE_LOCK_xyz value */, + atomicsHandshake: i++ /* 0=pending, 1=release, 2=granted */ + }); + state.sabOP = new SharedArrayBuffer( + i * 4/* ==sizeof int32, noting that Atomics.wait() and friends + can only function on Int32Array views of an SAB. */); + } + /** + SQLITE_xxx constants to export to the async worker + counterpart... + */ + state.sq3Codes = Object.create(null); + [ + 'SQLITE_ACCESS_EXISTS', + 'SQLITE_ACCESS_READWRITE', + 'SQLITE_BUSY', + 'SQLITE_CANTOPEN', + 'SQLITE_ERROR', + 'SQLITE_IOERR', + 'SQLITE_IOERR_ACCESS', + 'SQLITE_IOERR_CLOSE', + 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_LOCK', + 'SQLITE_IOERR_READ', + 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_TRUNCATE', + 'SQLITE_IOERR_UNLOCK', + 'SQLITE_IOERR_WRITE', + 'SQLITE_LOCK_EXCLUSIVE', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCKED', + 'SQLITE_MISUSE', + 'SQLITE_NOTFOUND', + 'SQLITE_OPEN_CREATE', + 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', + 'SQLITE_OPEN_READONLY', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_EXCLUSIVE' + ].forEach((k)=>{ + if(undefined === (state.sq3Codes[k] = capi[k])){ + toss("Maintenance required: not found:",k); + } + }); + + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" + enables this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + Flag for use with xOpen(). URI flag "delete-before-open=1" + tells the VFS to delete the db file before attempting to open + it. This can be used, e.g., to replace a db which has been + corrupted (without forcing us to expose a delete/unlink() + function in the public API). + + Failure to unlink the file is ignored but may lead to + downstream errors. An unlink can fail if, e.g., another tab + has the handle open. + + It goes without saying that deleting a file out from under another + instance results in Undefined Behavior. + */ + OPFS_UNLINK_BEFORE_OPEN: 0x02, + /** + If true, any async routine which implicitly acquires a sync + access handle (i.e. an OPFS lock) will release that lock at + the end of the call which acquires it. If false, such + "autolocks" are not released until the VFS is idle for some + brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). + */ + defaultUnlockAsap: false + }); + +//#if nope +/* does not yet work this way */ +//#define opfs-has-metrics +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-has-metrics + state.initS11n = initS11n; +//#endif + return state; + }/*createVfsStateObject()*/; + }/*sqlite3ApiBootstrap.initializers*/); -//#else -/* - The OPFS SAH Pool parts are elided from builds targeting node.js. -*/ //#endif target:node diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 3ee44d33b0..7bb31dc60b 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -752,6 +752,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( toss: function(...args){throw new Error(args.join(' '))}, toss3, typedArrayPart: wasm.typedArrayPart, + nu: (...obj)=>Object.assign(Object.create(null),...obj), assert: function(arg,msg){ if( !arg ){ util.toss("Assertion failed:",msg); diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 98eeb1639b..c9bd620fa5 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -50,12 +50,12 @@ usage of those methods to remove the "await". */ "use strict"; -const wPost = (type,...args)=>postMessage({type, payload:args}); const urlParams = new URL(globalThis.location.href).searchParams; if( !urlParams.has('vfs') ){ throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); } const isWebLocker = 'opfs-wl'===urlParams.get('vfs'); +const wPost = (type,...args)=>postMessage({type, payload:args}); const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; if(globalThis.window === globalThis){ diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index e2f4ff2f93..dee1f6d190 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -34,9 +34,11 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") - /* Gets removed from sqlite3 during bootstrap, so we need an - early reference to it. */; + /* These get removed from sqlite3 during bootstrap, so we need an + early reference to it. */ + const util = sqlite3.util; + const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs"); + /** installOpfsWlVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs @@ -64,6 +66,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ On success, the Promise resolves to the top-most sqlite3 namespace object. + + Code-diver notes: this file is particularly sparse on documentation + because much of it is identical to the code in + sqlite3-vfs-opfs.c-pp.js. See that file for more details. */ const installOpfsWlVfs = function callee(options){ try{ @@ -71,8 +77,7 @@ const installOpfsWlVfs = function callee(options){ }catch(e){ return Promise.reject(e); } - const nu = (...obj)=>Object.assign(Object.create(null),...obj); - options = nu(options); + options = util.nu(options); const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('opfs-disable')){ //sqlite3.config.warn('Explicitly not installing 'opfs-wl' VFS due to opfs-disable flag.'); @@ -105,34 +110,30 @@ const installOpfsWlVfs = function callee(options){ const error = (...args)=>logImpl(0, ...args); const toss = sqlite3.util.toss; const capi = sqlite3.capi; - const util = sqlite3.util; const wasm = sqlite3.wasm; const sqlite3_vfs = capi.sqlite3_vfs; const sqlite3_file = capi.sqlite3_file; const sqlite3_io_methods = capi.sqlite3_io_methods; - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + const opfsIoMethods = new sqlite3_io_methods(); + const opfsVfs = new sqlite3_vfs() + .addOnDispose( ()=>opfsIoMethods.dispose()); + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit + is undocumented/unspecified. */; + opfsVfs.$zName = wasm.allocCString('opfs-wl'); + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) + ); + const state = opfsUtil.createVfsStateObject(opfsVfs); + state.verbose = options.verbose; const metrics = Object.create(null); //#define opfs-has-metrics //#include api/opfs-common-inline.c-pp.js @@ -168,10 +169,8 @@ const installOpfsWlVfs = function callee(options){ s = metrics.s11n.deserialize = Object.create(null); s.count = s.time = 0; } - }/*metrics*/; - const opfsIoMethods = new sqlite3_io_methods(); - const opfsVfs = new sqlite3_vfs() - .addOnDispose( ()=>opfsIoMethods.dispose()); + }/*vfsMetrics*/; + vfsMetrics.reset(); let promiseWasRejected = undefined; const promiseReject = (err)=>{ promiseWasRejected = true; @@ -182,14 +181,14 @@ const installOpfsWlVfs = function callee(options){ promiseWasRejected = false; return promiseResolve_(sqlite3); }; - const workerArgs = '?vfs=opfs-wl'; + options.proxyUri += '?vfs=opfs-wl'; const W = //#if target:es6-bundler-friendly new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); //#elif target:es6-module - new Worker(new URL(options.proxyUri+workerArgs, import.meta.url)); + new Worker(new URL(options.proxyUri, import.meta.url)); //#else - new Worker(options.proxyUri+workerArgs); + new Worker(options.proxyUri); //#endif setTimeout(()=>{ /* At attempt to work around a browser-specific quirk in which @@ -210,216 +209,6 @@ const installOpfsWlVfs = function callee(options){ error("Error initializing OPFS asyncer:",err); promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); }; - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit - is undocumented/unspecified. */; - opfsVfs.$zName = wasm.allocCString('opfs-wl'); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.addOnDispose( - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) - ); - /** - Pedantic sidebar about opfsVfs.ondispose: the entries in that array - are items to clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the opfsVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ - - state.verbose = options.verbose; - state.littleEndian = (()=>{ - const buffer = new ArrayBuffer(2); - new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); - // Int16Array uses the platform's endianness. - return new Int16Array(buffer)[0] === 256; - })(); - /** - asyncIdleWaitTime is how long (ms) to wait, in the async proxy, - for each Atomics.wait() when waiting on inbound VFS API calls. - We need to wake up periodically to give the thread a chance to - do other things. If this is too high (e.g. 500ms) then even two - workers/tabs can easily run into locking errors. Some multiple - of this value is also used for determining how long to wait on - lock contention to free up. - */ - state.asyncIdleWaitTime = 150; - - /** - Whether the async counterpart should log exceptions to - the serialization channel. That produces a great deal of - noise for seemingly innocuous things like xAccess() checks - for missing files, so this option may have one of 3 values: - - 0 = no exception logging. - - 1 = only log exceptions for "significant" ops like xOpen(), - xRead(), and xWrite(). - - 2 = log all exceptions. - */ - state.asyncS11nExceptions = 1; - /* Size of file I/O buffer block. 64k = max sqlite3 page size, and - xRead/xWrite() will never deal in blocks larger than that. */ - state.fileBufferSize = 1024 * 64; - state.sabS11nOffset = state.fileBufferSize; - /** - The size of the block in our SAB for serializing arguments and - result values. Needs to be large enough to hold serialized - values of any of the proxied APIs. Filenames are the largest - part but are limited to opfsVfs.$mxPathname bytes. We also - store exceptions there, so it needs to be long enough to hold - a reasonably long exception string. - */ - state.sabS11nSize = opfsVfs.$mxPathname * 2; - /** - The SAB used for all data I/O between the synchronous and - async halves (file i/o and arg/result s11n). - */ - state.sabIO = new SharedArrayBuffer( - state.fileBufferSize/* file i/o block */ - + state.sabS11nSize/* argument/result serialization block */ - ); - state.opIds = Object.create(null); - { - /* Indexes for use in our SharedArrayBuffer... */ - let i = 0; - /* SAB slot used to communicate which operation is desired - between both workers. This worker writes to it and the other - listens for changes. */ - state.opIds.whichOp = i++; - /* Slot for storing return values. This worker listens to that - slot and the other worker writes to it. */ - state.opIds.rc = i++; - /* Each function gets an ID which this worker writes to - the whichOp slot. The async-api worker uses Atomic.wait() - on the whichOp slot to figure out which operation to run - next. */ - state.opIds.xAccess = i++; - state.opIds.xClose = i++; - state.opIds.xDelete = i++; - state.opIds.xDeleteNoWait = i++; - state.opIds.xFileSize = i++; - state.opIds.xLock = i++; - state.opIds.xOpen = i++; - state.opIds.xRead = i++; - state.opIds.xSleep = i++; - state.opIds.xSync = i++; - state.opIds.xTruncate = i++; - state.opIds.xUnlock = i++; - state.opIds.xWrite = i++; - state.opIds.mkdir = i++; - state.opIds.lockControl = i++ /* we signal the intent to lock here */; - /** Internal signals which are used only during development and - testing via the dev console. */ - state.opIds['opfs-async-metrics'] = i++; - state.opIds['opfs-async-shutdown'] = i++; - /* The retry slot is used by the async part for wait-and-retry - semantics. Though we could hypothetically use the xSleep slot - for that, doing so might lead to undesired side effects. */ - state.opIds.retry = i++; - - /* Slots for submitting the lock type and receiving its acknowledgement. */ - state.lock = nu({ - type: i++ /* SQLITE_LOCK_xyz value */, - atomicsHandshake: i++ /* 1=release, 2=granted */ - }); - state.sabOP = new SharedArrayBuffer( - i * 4/* ==sizeof int32, noting that Atomics.wait() and friends - can only function on Int32Array views of an SAB. */); - vfsMetrics.reset(); - } - /** - SQLITE_xxx constants to export to the async worker - counterpart... - */ - state.sq3Codes = Object.create(null); - [ - 'SQLITE_ACCESS_EXISTS', - 'SQLITE_ACCESS_READWRITE', - 'SQLITE_BUSY', - 'SQLITE_CANTOPEN', - 'SQLITE_ERROR', - 'SQLITE_IOERR', - 'SQLITE_IOERR_ACCESS', - 'SQLITE_IOERR_CLOSE', - 'SQLITE_IOERR_DELETE', - 'SQLITE_IOERR_FSYNC', - 'SQLITE_IOERR_LOCK', - 'SQLITE_IOERR_READ', - 'SQLITE_IOERR_SHORT_READ', - 'SQLITE_IOERR_TRUNCATE', - 'SQLITE_IOERR_UNLOCK', - 'SQLITE_IOERR_WRITE', - 'SQLITE_LOCK_EXCLUSIVE', - 'SQLITE_LOCK_NONE', - 'SQLITE_LOCK_PENDING', - 'SQLITE_LOCK_RESERVED', - 'SQLITE_LOCK_SHARED', - 'SQLITE_LOCKED', - 'SQLITE_MISUSE', - 'SQLITE_NOTFOUND', - 'SQLITE_OPEN_CREATE', - 'SQLITE_OPEN_DELETEONCLOSE', - 'SQLITE_OPEN_MAIN_DB', - 'SQLITE_OPEN_READONLY', - 'SQLITE_LOCK_NONE', - 'SQLITE_LOCK_SHARED', - 'SQLITE_LOCK_RESERVED', - 'SQLITE_LOCK_PENDING', - 'SQLITE_LOCK_EXCLUSIVE' - ].forEach((k)=>{ - if(undefined === (state.sq3Codes[k] = capi[k])){ - toss("Maintenance required: not found:",k); - } - }); - state.opfsFlags = Object.assign(Object.create(null),{ - /** - Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" - enables this. See defaultUnlockAsap, below. - */ - OPFS_UNLOCK_ASAP: 0x01, - /** - Flag for use with xOpen(). URI flag "delete-before-open=1" - tells the VFS to delete the db file before attempting to open - it. This can be used, e.g., to replace a db which has been - corrupted (without forcing us to expose a delete/unlink() - function in the public API). - - Failure to unlink the file is ignored but may lead to - downstream errors. An unlink can fail if, e.g., another tab - has the handle open. - - It goes without saying that deleting a file out from under another - instance results in Undefined Behavior. - */ - OPFS_UNLINK_BEFORE_OPEN: 0x02, - /** - If true, any async routine which implicitly acquires a sync - access handle (i.e. an OPFS lock) will release that lock at - the end of the call which acquires it. If false, such - "autolocks" are not released until the VFS is idle for some - brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). - */ - defaultUnlockAsap: false - }); /** Runs the given operation (by name) in the async worker @@ -501,18 +290,17 @@ const installOpfsWlVfs = function callee(options){ const ioSyncWrappers = { xCheckReservedLock: function(pFile,pOut){ /** - As of late 2022, only a single lock can be held on an OPFS - file. We have no way of checking whether any _other_ db - connection has a lock except by trying to obtain and (on - success) release a sync-handle for it, but doing so would - involve an inherent race condition. For the time being, - pending a better solution, we simply report whether the - given pFile is open. + After consultation with a topic expert: "opfs-wl" will + continue to use the same no-op impl which "opfs" does + because: - Update 2024-06-12: based on forum discussions, this - function now always sets pOut to 0 (false): + - xCheckReservedLock() is just a hint. If SQLite needs to + lock, it's still going to try to lock. - https://sqlite.org/forum/forumpost/a2f573b00cda1372 + - We cannot do this check synchronously in "opfs-wl", + so would need to pass it to the async proxy. That would + make it inordinately expensive considering that it's + just a hint. */ wasm.poke(pOut, 0, 'i32'); return 0; @@ -855,6 +643,8 @@ const installOpfsWlVfs = function callee(options){ } }/*sanityCheck()*/; + //const initS11n = state.initS11n || toss("Missing state.initS11n()"); + //delete state.initS11n; W.onmessage = function({data}){ //log("Worker.onmessage:",data); switch(data.type){ @@ -867,7 +657,7 @@ const installOpfsWlVfs = function callee(options){ /* Arrives as soon as the asyc proxy finishes loading. Pass our config and shared state on to the async worker. */ - W.postMessage({type: 'opfs-async-init',args: state}); + W.postMessage({type: 'opfs-async-init', args: util.nu(state)}); break; case 'opfs-async-inited': { /* Indicates that the async partner has received the 'init' diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 7ba7427d0a..b294163236 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -122,6 +122,41 @@ const installOpfsVfs = function callee(options){ const sqlite3_file = capi.sqlite3_file; const sqlite3_io_methods = capi.sqlite3_io_methods; + const opfsIoMethods = new sqlite3_io_methods(); + const opfsVfs = new sqlite3_vfs() + .addOnDispose( ()=>opfsIoMethods.dispose()); + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit + is undocumented/unspecified. */; + opfsVfs.$zName = wasm.allocCString("opfs"); + // All C-side memory of opfsVfs is zeroed out, but just to be explicit: + opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) + /** + Pedantic sidebar: the entries in this array are items to + clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown + order of the wasm engine and the JS one are undefined so + there is no guaranty that the opfsVfs instance would be + available in one environment or the other when + sqlite3_os_end() is called (_if_ it gets called at all in a + wasm build, which is undefined). + */ + ); + /** State which we send to the async-api Worker or share with it. This object must initially contain only cloneable or sharable @@ -144,7 +179,8 @@ const installOpfsVfs = function callee(options){ postMessage() to communicate its SharedArrayBuffer, and mixing that approach with Atomics.wait/notify() gets a bit messy. */ - const state = Object.create(null); + const state = opfsUtil.createVfsStateObject(opfsVfs); + state.verbose = options.verbose; const metrics = Object.create(null); //#define opfs-has-metrics //#include api/opfs-common-inline.c-pp.js @@ -181,9 +217,7 @@ const installOpfsVfs = function callee(options){ s.count = s.time = 0; } }/*metrics*/; - const opfsIoMethods = new sqlite3_io_methods(); - const opfsVfs = new sqlite3_vfs() - .addOnDispose( ()=>opfsIoMethods.dispose()); + vfsMetrics.reset(); let promiseWasRejected = undefined; const promiseReject = (err)=>{ promiseWasRejected = true; @@ -222,201 +256,6 @@ const installOpfsVfs = function callee(options){ error("Error initializing OPFS asyncer:",err); promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); }; - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit - is undocumented/unspecified. */; - opfsVfs.$zName = wasm.allocCString("opfs"); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.addOnDispose( - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) - ); - /** - Pedantic sidebar about opfsVfs.ondispose: the entries in that array - are items to clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the opfsVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ - state.verbose = options.verbose; - state.littleEndian = (()=>{ - const buffer = new ArrayBuffer(2); - new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); - // Int16Array uses the platform's endianness. - return new Int16Array(buffer)[0] === 256; - })(); - /** - asyncIdleWaitTime is how long (ms) to wait, in the async proxy, - for each Atomics.wait() when waiting on inbound VFS API calls. - We need to wake up periodically to give the thread a chance to - do other things. If this is too high (e.g. 500ms) then even two - workers/tabs can easily run into locking errors. Some multiple - of this value is also used for determining how long to wait on - lock contention to free up. - */ - state.asyncIdleWaitTime = 150; - - /** - Whether the async counterpart should log exceptions to - the serialization channel. That produces a great deal of - noise for seemingly innocuous things like xAccess() checks - for missing files, so this option may have one of 3 values: - - 0 = no exception logging. - - 1 = only log exceptions for "significant" ops like xOpen(), - xRead(), and xWrite(). - - 2 = log all exceptions. - */ - state.asyncS11nExceptions = 1; - /* Size of file I/O buffer block. 64k = max sqlite3 page size, and - xRead/xWrite() will never deal in blocks larger than that. */ - state.fileBufferSize = 1024 * 64; - state.sabS11nOffset = state.fileBufferSize; - /** - The size of the block in our SAB for serializing arguments and - result values. Needs to be large enough to hold serialized - values of any of the proxied APIs. Filenames are the largest - part but are limited to opfsVfs.$mxPathname bytes. We also - store exceptions there, so it needs to be long enough to hold - a reasonably long exception string. - */ - state.sabS11nSize = opfsVfs.$mxPathname * 2; - /** - The SAB used for all data I/O between the synchronous and - async halves (file i/o and arg/result s11n). - */ - state.sabIO = new SharedArrayBuffer( - state.fileBufferSize/* file i/o block */ - + state.sabS11nSize/* argument/result serialization block */ - ); - state.opIds = Object.create(null); - { - /* Indexes for use in our SharedArrayBuffer... */ - let i = 0; - /* SAB slot used to communicate which operation is desired - between both workers. This worker writes to it and the other - listens for changes. */ - state.opIds.whichOp = i++; - /* Slot for storing return values. This worker listens to that - slot and the other worker writes to it. */ - state.opIds.rc = i++; - /* Each function gets an ID which this worker writes to - the whichOp slot. The async-api worker uses Atomic.wait() - on the whichOp slot to figure out which operation to run - next. */ - state.opIds.xAccess = i++; - state.opIds.xClose = i++; - state.opIds.xDelete = i++; - state.opIds.xDeleteNoWait = i++; - state.opIds.xFileSize = i++; - state.opIds.xLock = i++; - state.opIds.xOpen = i++; - state.opIds.xRead = i++; - state.opIds.xSleep = i++; - state.opIds.xSync = i++; - state.opIds.xTruncate = i++; - state.opIds.xUnlock = i++; - state.opIds.xWrite = i++; - state.opIds.mkdir = i++; - state.opIds['opfs-async-metrics'] = i++; - state.opIds['opfs-async-shutdown'] = i++; - /* The retry slot is used by the async part for wait-and-retry - semantics. Though we could hypothetically use the xSleep slot - for that, doing so might lead to undesired side effects. */ - state.opIds.retry = i++; - state.sabOP = new SharedArrayBuffer( - i * 4/* ==sizeof int32, noting that Atomics.wait() and friends - can only function on Int32Array views of an SAB. */); - vfsMetrics.reset(); - } - /** - SQLITE_xxx constants to export to the async worker - counterpart... - */ - state.sq3Codes = Object.create(null); - [ - 'SQLITE_ACCESS_EXISTS', - 'SQLITE_ACCESS_READWRITE', - 'SQLITE_BUSY', - 'SQLITE_CANTOPEN', - 'SQLITE_ERROR', - 'SQLITE_IOERR', - 'SQLITE_IOERR_ACCESS', - 'SQLITE_IOERR_CLOSE', - 'SQLITE_IOERR_DELETE', - 'SQLITE_IOERR_FSYNC', - 'SQLITE_IOERR_LOCK', - 'SQLITE_IOERR_READ', - 'SQLITE_IOERR_SHORT_READ', - 'SQLITE_IOERR_TRUNCATE', - 'SQLITE_IOERR_UNLOCK', - 'SQLITE_IOERR_WRITE', - 'SQLITE_LOCK_EXCLUSIVE', - 'SQLITE_LOCK_NONE', - 'SQLITE_LOCK_PENDING', - 'SQLITE_LOCK_RESERVED', - 'SQLITE_LOCK_SHARED', - 'SQLITE_LOCKED', - 'SQLITE_MISUSE', - 'SQLITE_NOTFOUND', - 'SQLITE_OPEN_CREATE', - 'SQLITE_OPEN_DELETEONCLOSE', - 'SQLITE_OPEN_MAIN_DB', - 'SQLITE_OPEN_READONLY' - ].forEach((k)=>{ - if(undefined === (state.sq3Codes[k] = capi[k])){ - toss("Maintenance required: not found:",k); - } - }); - state.opfsFlags = Object.assign(Object.create(null),{ - /** - Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" - enables this. See defaultUnlockAsap, below. - */ - OPFS_UNLOCK_ASAP: 0x01, - /** - Flag for use with xOpen(). URI flag "delete-before-open=1" - tells the VFS to delete the db file before attempting to open - it. This can be used, e.g., to replace a db which has been - corrupted (without forcing us to expose a delete/unlink() - function in the public API). - - Failure to unlink the file is ignored but may lead to - downstream errors. An unlink can fail if, e.g., another tab - has the handle open. - - It goes without saying that deleting a file out from under another - instance results in Undefined Behavior. - */ - OPFS_UNLINK_BEFORE_OPEN: 0x02, - /** - If true, any async routine which implicitly acquires a sync - access handle (i.e. an OPFS lock) will release that lock at - the end of the call which acquires it. If false, such - "autolocks" are not released until the VFS is idle for some - brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). - */ - defaultUnlockAsap: false - }); /** Runs the given operation (by name) in the async worker @@ -827,6 +666,8 @@ const installOpfsVfs = function callee(options){ } }/*sanityCheck()*/; + //const initS11n = state.initS11n || toss("Missing state.initS11n()"); + //delete state.initS11n; W.onmessage = function({data}){ //log("Worker.onmessage:",data); switch(data.type){ diff --git a/manifest b/manifest index f334f15c5a..82c6c5bf4a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Consolidate\smuch\sof\sthe\sOPFS\sutility\scode\sinto\sa\snew\sfile\sfor\suse\sby\stwo\sof\sthe\sOPFS\sVFSes. -D 2026-03-04T11:37:39.256 +C Baby\ssteps\sin\sconsolidating\scommon\sOPFS\sVFS\scode. +D 2026-03-04T14:33:33.214 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -584,22 +584,22 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 7ba933e8f1290cc65459dd371c0c9a031d96bdf14 F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 -F ext/wasm/api/opfs-common-inline.c-pp.js b9c4e080698792cbc04ce9dd9dda7d8316c6db0262f74820706f98b352b949d5 w ext/wasm/api/opfs-common.c-pp.js -F ext/wasm/api/opfs-common-shared.c-pp.js 3f8f3f2ab4790fdd1e6d1d9224e232cef07f1c8753827c18bbba965dbe98795f +F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 +F ext/wasm/api/opfs-common-shared.c-pp.js fdebcb821f9f732eb263e3ee2dbd6af5709aaa5fec9da6eaa0e10fd93b72f547 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa -F ext/wasm/api/sqlite3-api-prologue.js ccd8ece4b4580d2a70996218f28e810d70a86f5e2795f4d4a75f0603af24aef6 +F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 25b856508ac94336419133c6ec10594f576b469f85cc69cde4c09cfa06a8e1c7 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f0a2aa8712211ff9db2ef548ae8b676be3e7c82f61586d03fd8317fbc95bbedd F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 54f71e563dda30af73ed84ff9de03441537b2e8fb8d2ae2a0b0c8187f51db67a -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 2e2a72a40e2ad6ea92f52eb7adbd925d4acd874ffeecaa00b85234ad49862655 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js c6e5a281756e8ed7bbabf086dae765021486e17b91b6c4eee3c08dc2485fa348 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b8db3ccfedb457634cc07df7a955bc44c6d1be8b40d35f47822c2168ab8b1968 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec -R 82203270087631af34878be94f4d1594 +P db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7 +R a1c4d958895cbdc094c8287aa9f23443 U stephan -Z 1548da853f7a79cab3a86616c1bf35cc +Z b1a56499cd2b6bf9891139561c1b1fe5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 65aadb1877..8f61158031 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7 +b0dd23299e97ff975f213cb3a8b051f4d7b785b29def82e01f53427fdf77ecb6 From 691494a93bcfd6753c8594e9874077fcec44a920 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 16:30:51 +0000 Subject: [PATCH 09/32] Consolidate the OPFS VFS's metrics-tracking code. FossilOrigin-Name: b71c79ef9672c77a72a976ffcd7cbebfaf0ff314dff97b274f7d092de6a7773f --- ext/wasm/api/opfs-common-shared.c-pp.js | 47 +++++++++++++++++++++--- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 44 ++-------------------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 45 ++--------------------- ext/wasm/mkwasmbuilds.c | 6 +++ manifest | 18 ++++----- manifest.uuid | 2 +- 6 files changed, 66 insertions(+), 96 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 077f4c5a6a..376780217d 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -453,6 +453,40 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const vfsName = wasm.cstrToJs(opfsVfs.$zName); const isWebLocker = 'opfs-wl'===vfsName; const state = util.nu(); + opfsVfs.metrics = util.nu({ + counters: util.nu(), + dump: function(){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; + } + sqlite3.config.log(globalThis.location.href, + "metrics for",globalThis.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); + sqlite3.config.log("Serialization metrics:",opfsVfs.metrics.counters.s11n); + //W.postMessage({type:'opfs-async-metrics'}); + }, + reset: function(){ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + const m = opfsVfs.metrics.counters; + for(k in state.opIds){ + r(m[k] = Object.create(null)); + } + let s = m.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = m.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + } + })/*opfsVfs.metrics*/; + /** asyncIdleWaitTime is how long (ms) to wait, in the async proxy, for each Atomics.wait() when waiting on inbound VFS API calls. @@ -638,12 +672,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ defaultUnlockAsap: false }); -//#if nope -/* does not yet work this way */ -//#define opfs-has-metrics + opfsVfs.metrics.reset(); +//#if not defined nope +//#// does not yet work this way +//#define vfs.metrics.enable + const metrics = opfsVfs.metrics.counters; //#include api/opfs-common-inline.c-pp.js -//#undef opfs-has-metrics - state.initS11n = initS11n; +//#// import initS11n() +//#undef vfs.metrics.enable + opfsVfs.initS11n = initS11n; //#endif return state; }/*createVfsStateObject()*/; diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index dee1f6d190..288f01e650 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -134,43 +134,8 @@ const installOpfsWlVfs = function callee(options){ ); const state = opfsUtil.createVfsStateObject(opfsVfs); state.verbose = options.verbose; - const metrics = Object.create(null); -//#define opfs-has-metrics -//#include api/opfs-common-inline.c-pp.js -//#undef opfs-has-metrics + const metrics = opfsVfs.metrics.counters; - const vfsMetrics = { - dump: function(){ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; - } - sqlite3.config.log(globalThis.location.href, - "metrics for",globalThis.location.href,":",metrics, - "\nTotal of",n,"op(s) for",t, - "ms (incl. "+w+" ms of waiting on the async side)"); - sqlite3.config.log("Serialization metrics:",metrics.s11n); - W.postMessage({type:'opfs-async-metrics'}); - }, - reset: function(){ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; - } - }/*vfsMetrics*/; - vfsMetrics.reset(); let promiseWasRejected = undefined; const promiseReject = (err)=>{ promiseWasRejected = true; @@ -643,8 +608,6 @@ const installOpfsWlVfs = function callee(options){ } }/*sanityCheck()*/; - //const initS11n = state.initS11n || toss("Missing state.initS11n()"); - //delete state.initS11n; W.onmessage = function({data}){ //log("Worker.onmessage:",data); switch(data.type){ @@ -674,7 +637,8 @@ const installOpfsWlVfs = function callee(options){ state.sabOPView = new Int32Array(state.sabOP); state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - initS11n(); + opfsVfs.initS11n(); + delete opfsVfs.initS11n; if(options.sanityChecks){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); @@ -720,7 +684,7 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ //sqlite3.config.warn("installOpfsWlVfs.defaultProxyUri =",installOpfsWlVfs.defaultProxyUri); } return installOpfsWlVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS-WL sqlite3_vfs:",e.message); + sqlite3.config.warn("Ignoring inability to install OPFS-WL sqlite3_vfs:",e); }); }catch(e){ sqlite3.config.error("installOpfsWlVfs() exception:",e); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index b294163236..ba5fe9918e 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -181,43 +181,7 @@ const installOpfsVfs = function callee(options){ */ const state = opfsUtil.createVfsStateObject(opfsVfs); state.verbose = options.verbose; - const metrics = Object.create(null); -//#define opfs-has-metrics -//#include api/opfs-common-inline.c-pp.js -//#undef opfs-has-metrics - - const vfsMetrics = { - dump: function(){ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; - } - sqlite3.config.log(globalThis.location.href, - "metrics for",globalThis.location.href,":",metrics, - "\nTotal of",n,"op(s) for",t, - "ms (incl. "+w+" ms of waiting on the async side)"); - sqlite3.config.log("Serialization metrics:",metrics.s11n); - W.postMessage({type:'opfs-async-metrics'}); - }, - reset: function(){ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; - } - }/*metrics*/; - vfsMetrics.reset(); + const metrics = opfsVfs.metrics.counters; let promiseWasRejected = undefined; const promiseReject = (err)=>{ promiseWasRejected = true; @@ -666,8 +630,6 @@ const installOpfsVfs = function callee(options){ } }/*sanityCheck()*/; - //const initS11n = state.initS11n || toss("Missing state.initS11n()"); - //delete state.initS11n; W.onmessage = function({data}){ //log("Worker.onmessage:",data); switch(data.type){ @@ -697,7 +659,8 @@ const installOpfsVfs = function callee(options){ state.sabOPView = new Int32Array(state.sabOP); state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - initS11n(); + opfsVfs.initS11n(); + delete opfsVfs.initS11n; if(options.sanityChecks){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); @@ -743,7 +706,7 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); } return installOpfsVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); + sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e); }); }catch(e){ sqlite3.config.error("installOpfsVfs() exception:",e); diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 748594f4ca..022a926397 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -747,6 +747,12 @@ static void emit_api_js(char const *zBuildName){ zBuildName, zBuildName, zBuildName); pf("$(out.%s.js): $(sqlite3-api.%s.js)\n", zBuildName, zBuildName); + pf("$(sqlite3-api.%s.js):" + /* Extra deps needed by the OPFS pieces... */ + " $(dir.api)/opfs-common-shared.c-pp.js" + " $(dir.api)/opfs-common-inline.c-pp.js" + "\n", + zBuildName); } /* diff --git a/manifest b/manifest index 82c6c5bf4a..ea250bde8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Baby\ssteps\sin\sconsolidating\scommon\sOPFS\sVFS\scode. -D 2026-03-04T14:33:33.214 +C Consolidate\sthe\sOPFS\sVFS's\smetrics-tracking\scode. +D 2026-03-04T16:30:51.574 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js fdebcb821f9f732eb263e3ee2dbd6af5709aaa5fec9da6eaa0e10fd93b72f547 +F ext/wasm/api/opfs-common-shared.c-pp.js e00a2f18b6ed6d560b19503d9adce49f2151d1b488cbca24545a631bbed72d25 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -598,8 +598,8 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f0a2aa8712211ff9db2ef548ae8b676b F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js c6e5a281756e8ed7bbabf086dae765021486e17b91b6c4eee3c08dc2485fa348 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b8db3ccfedb457634cc07df7a955bc44c6d1be8b40d35f47822c2168ab8b1968 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 007430b82089b2aa8720a5866a9a9b0e9a7ddf3a1e7bafd1b652bec6c6a18196 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 9061455eb3a51aa898d02737aae1210c439d9f412b97d342ae9123746f72fcad F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -628,7 +628,7 @@ F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x -F ext/wasm/mkwasmbuilds.c b1ed20dfba178e6ccae91b4594ac9439992fd0cb783f202c639ca761f1b1d2a3 +F ext/wasm/mkwasmbuilds.c 5961177b9d6ad2b49efbdbb3076527e6886fecef7adf99d71cce58bfca89be08 F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7 -R a1c4d958895cbdc094c8287aa9f23443 +P b0dd23299e97ff975f213cb3a8b051f4d7b785b29def82e01f53427fdf77ecb6 +R 950709f8ceba3a494ed70c19c6d2296a U stephan -Z b1a56499cd2b6bf9891139561c1b1fe5 +Z a91e3522d6c27cde9f5c40c8510d95f1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8f61158031..3dfa0db211 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b0dd23299e97ff975f213cb3a8b051f4d7b785b29def82e01f53427fdf77ecb6 +b71c79ef9672c77a72a976ffcd7cbebfaf0ff314dff97b274f7d092de6a7773f From 080da5f56da5c9a1c13e2002d670f22e9b2da599 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 17:54:02 +0000 Subject: [PATCH 10/32] Factor out about 300 lines of common OPFS VFS bootstrapping code. FossilOrigin-Name: 57adecbab71795b62b1c2e4570ff504f35681e81dd8c94f78ad8e05ef39d36fd --- ext/wasm/api/opfs-common-shared.c-pp.js | 399 ++++++++++++++++-- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 506 +++++------------------ ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 484 ++++------------------ manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 551 insertions(+), 856 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 376780217d..feefca5eb4 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -421,38 +421,80 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; + opfsUtil.initOptions = function(options, callee){ + options = util.nu(options); + const urlParams = new URL(globalThis.location.href).searchParams; + if(urlParams.has('opfs-disable')){ + //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); + options.disableOpfs = true; + return options; + } + if(undefined===options.verbose){ + options.verbose = urlParams.has('opfs-verbose') + ? (+urlParams.get('opfs-verbose') || 2) : 1; + } + if(undefined===options.sanityChecks){ + options.sanityChecks = urlParams.has('opfs-sanity-check'); + } + if(undefined===options.proxyUri){ + options.proxyUri = callee.defaultProxyUri; + } + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } + return options; + }; + /** - Populates the main state object used by "opfs" and "opfs-wl", and + Creates and populates the main state object used by "opfs" and "opfs-wl", and transfered from those to their async counterpart. - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. + Returns an object containing state which we send to the async-api + Worker or share with it. - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. + Because the returned object must be serializable to be posted to + the async proxy, after this returns, the caller must: - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. + - Make a local-scope reference of state.vfs then (delete + state.vfs). That's the capi.sqlite3_vfs instance for the VFS. + + This object must, when it's passed to the async part, contain + only cloneable or sharable objects. After the worker's "inited" + message arrives, other types of data may be added to it. */ - opfsUtil.createVfsStateObject = function(opfsVfs){ - if( !(opfsVfs instanceof capi.sqlite3_vfs) ){ - toss("Expecting a sqlite3_vfs instance"); - } - const vfsName = wasm.cstrToJs(opfsVfs.$zName); - const isWebLocker = 'opfs-wl'===vfsName; + opfsUtil.createVfsState = function(vfsName, options){ const state = util.nu(); + state.verbose = options.verbose; + + const opfsVfs = state.vfs = new capi.sqlite3_vfs(); + const opfsIoMethods = opfsVfs.ioMethods = new capi.sqlite3_io_methods(); + + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit + is undocumented/unspecified. */; + opfsVfs.$zName = wasm.allocCString(vfsName); + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, opfsIoMethods + /** + Pedantic sidebar: the entries in this array are items to + clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown + order of the wasm engine and the JS one are undefined so + there is no guaranty that the opfsVfs instance would be + available in one environment or the other when + sqlite3_os_end() is called (_if_ it gets called at all in a + wasm build, which is undefined). i.e. addOnDispose() here is + a matter of "correctness", not necessity. It just wouldn't do + to leave the impression that we're blindly leaking memory. + */ + ); + + const isWebLocker = 'opfs-wl'===vfsName; opfsVfs.metrics = util.nu({ counters: util.nu(), dump: function(){ @@ -470,7 +512,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ "\nTotal of",n,"op(s) for",t, "ms (incl. "+w+" ms of waiting on the async side)"); sqlite3.config.log("Serialization metrics:",opfsVfs.metrics.counters.s11n); - //W.postMessage({type:'opfs-async-metrics'}); + opfsVfs.worker?.postMessage?.({type:'opfs-async-metrics'}); }, reset: function(){ let k; @@ -533,6 +575,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ state.fileBufferSize/* file i/o block */ + state.sabS11nSize/* argument/result serialization block */ ); + + /** + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. state.opIds holds the SAB slot + IDs of each of those. + */ state.opIds = Object.create(null); { /* @@ -582,9 +633,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ for that, doing so might lead to undesired side effects. */ state.opIds.retry = i++; - /* Slots for submitting the lock type and receiving its acknowledgement. - Only used by "opfs-wl". */ state.lock = util.nu({ + /* Slots for submitting the lock type and receiving its + acknowledgement. Only used by "opfs-wl". */ type: i++ /* SQLITE_LOCK_xyz value */, atomicsHandshake: i++ /* 0=pending, 1=release, 2=granted */ }); @@ -673,17 +724,297 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }); opfsVfs.metrics.reset(); -//#if not defined nope -//#// does not yet work this way -//#define vfs.metrics.enable const metrics = opfsVfs.metrics.counters; -//#include api/opfs-common-inline.c-pp.js + + /** + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The + 2nd and subsequent arguments must be the arguments for the + async op. + */ + const opRun = opfsVfs.opRun = (op,...args)=>{ + const opNdx = state.opIds[op] || toss("Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, state.opIds.rc, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + /* async thread will take over here */; + const t = performance.now(); + while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ + /* + The reason for this loop is buried in the details of a long + discussion at: + + https://github.com/sqlite/sqlite-wasm/issues/12 + + Summary: in at least one browser flavor, under high loads, + the wait()/notify() pairings can get out of sync. Calling + wait() here until it returns 'not-equal' gets them back in + sync. + */ + } + /* When the above wait() call returns 'not-equal', the async + half will have completed the operation and reported its results + in the state.opIds.rc slot of the SAB. */ + const rc = Atomics.load(state.sabOPView, state.opIds.rc); + metrics[op].wait += performance.now() - t; + if(rc && state.asyncS11nExceptions){ + const err = state.s11n.deserialize(); + if(err) error(op+"() async error:",...err); + } + return rc; + }; + + const opTimer = Object.create(null); + opTimer.op = undefined; + opTimer.start = undefined; + const mTimeStart = opfsVfs.mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + ++metrics[op].count; + }; + const mTimeEnd = opfsVfs.mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start + ); + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = opfsVfs.__openFiles = Object.create(null); + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = opfsVfs.ioSyncWrappers = util.nu({ + xCheckReservedLock: function(pFile,pOut){ + /** + After consultation with a topic expert: "opfs-wl" will + continue to use the same no-op impl which "opfs" does + because: + + - xCheckReservedLock() is just a hint. If SQLite needs to + lock, it's still going to try to lock. + + - We cannot do this check synchronously in "opfs-wl", + so would need to pass it to the async proxy. That would + make it inordinately expensive considering that it's + just a hint. + */ + wasm.poke(pOut, 0, 'i32'); + return 0; + }, + xClose: function(pFile){ + mTimeStart('xClose'); + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + mTimeEnd(); + return rc; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + /*mTimeStart('xFileControl'); + mTimeEnd();*/ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + mTimeStart('xFileSize'); + let rc = opRun('xFileSize', pFile); + if(0==rc){ + try { + const sz = state.s11n.deserialize()[0]; + wasm.poke(pSz64, sz, 'i64'); + }catch(e){ + error("Unexpected error reading xFileSize() result:",e); + rc = state.sq3Codes.SQLITE_IOERR; + } + } + mTimeEnd(); + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + mTimeStart('xRead'); + const f = __openFiles[pFile]; + let rc; + try { + rc = opRun('xRead',pFile, n, Number(offset64)); + if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ + /** + Results get written to the SharedArrayBuffer f.sabView. + Because the heap is _not_ a SharedArrayBuffer, we have + to copy the results. TypedArray.set() seems to be the + fastest way to copy this. */ + wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); + } + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + mTimeEnd(); + return rc; + }, + xSync: function(pFile,flags){ + mTimeStart('xSync'); + ++metrics.xSync.count; + const rc = opRun('xSync', pFile, flags); + mTimeEnd(); + return rc; + }, + xTruncate: function(pFile,sz64){ + mTimeStart('xTruncate'); + const rc = opRun('xTruncate', pFile, Number(sz64)); + mTimeEnd(); + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + mTimeStart('xWrite'); + const f = __openFiles[pFile]; + let rc; + try { + f.sabView.set(wasm.heap8u().subarray( + Number(pSrc), Number(pSrc) + n + )); + rc = opRun('xWrite', pFile, n, Number(offset64)); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_WRITE; + } + mTimeEnd(); + return rc; + } + })/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = opfsVfs.vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + mTimeStart('xAccess'); + const rc = opRun('xAccess', wasm.cstrToJs(zName)); + wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); + mTimeEnd(); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + mTimeStart('xDelete'); + const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); + mTimeEnd(); + return rc; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i{ if(options.verbose>level) loggers[level]("OPFS syncer:",...args); }; - const log = (...args)=>logImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); - const toss = sqlite3.util.toss; - const capi = sqlite3.capi; - const wasm = sqlite3.wasm; - const sqlite3_vfs = capi.sqlite3_vfs; - const sqlite3_file = capi.sqlite3_file; - const sqlite3_io_methods = capi.sqlite3_io_methods; - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - const opfsIoMethods = new sqlite3_io_methods(); - const opfsVfs = new sqlite3_vfs() - .addOnDispose( ()=>opfsIoMethods.dispose()); - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit - is undocumented/unspecified. */; - opfsVfs.$zName = wasm.allocCString('opfs-wl'); - opfsVfs.addOnDispose( - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) - ); - const state = opfsUtil.createVfsStateObject(opfsVfs); - state.verbose = options.verbose; - const metrics = opfsVfs.metrics.counters; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; + const state = opfsUtil.createVfsState('opfs-wl', options), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + __openFiles = opfsVfs.__openFiles; + delete state.vfs; + + /* At this point, createVfsState() has populated state and + opfsVfs with any code common to both the "opfs" and "opfs-wl" + VFSes. Now comes the VFS-dependent work... */ + + opfsVfs.ioSyncWrappers.xLock = function(pFile, lockType){ + mTimeStart('xLock'); + ++metrics.xLock.count; + const f = __openFiles[pFile]; + let rc = 0; + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( f.lockType ) { + f.lockType = lockType; + }else{ + try{ + const view = state.sabOPView; + /* We need to pass pFile's name to the async proxy so that + it can create the WebLock name. */ + state.s11n.serialize(f.filename) + Atomics.store(view, state.lock.atomicsHandshake, 0); + Atomics.store(view, state.lock.type, lockType); + Atomics.store(view, state.opIds.whichOp, state.opIds.lockControl); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + while('not-equal'!==Atomics.wait(view, state.lock.atomicsHandshake, 0)){ + /* Loop is a workaround for environment-specific quirks. See + notes in similar loops. */ + } + f.lockType = lockType; + }catch(e){ + error("xLock(",arguments,") failed", e, f); + rc = capi.SQLITE_IOERR_LOCK; + } + } + mTimeEnd(); + return rc; + }; + + opfsVfs.ioSyncWrappers.xUnlock =function(pFile,lockType){ + mTimeStart('xUnlock'); + ++metrics.xUnlock.count; + const f = __openFiles[pFile]; + let rc = 0; + if( lockType < f.lockType ){ + try{ + const view = state.sabOPView; + Atomics.store(view, state.lock.atomicsHandshake, 1); + Atomics.notify(view, state.lock.atomicsHandshake); + Atomics.wait(view, state.lock.atomicsHandshake, 1); + }catch(e){ + error("xUnlock(",pFile,lockType,") failed",e, f); + rc = capi.SQLITE_IOERR_LOCK; + } + } + if( 0===rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + }; + + let promiseWasRejected = undefined; const promiseReject = (err)=>{ @@ -147,7 +177,7 @@ const installOpfsWlVfs = function callee(options){ return promiseResolve_(sqlite3); }; options.proxyUri += '?vfs=opfs-wl'; - const W = + const W = opfsVfs.worker = //#if target:es6-bundler-friendly new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); //#elif target:es6-module @@ -175,47 +205,7 @@ const installOpfsWlVfs = function callee(options){ promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); }; - /** - Runs the given operation (by name) in the async worker - counterpart, waits for its response, and returns the result - which the async worker writes to SAB[state.opIds.rc]. The - 2nd and subsequent arguments must be the arguments for the - async op. - */ - const opRun = (op,...args)=>{ - const opNdx = state.opIds[op] || toss("Invalid op ID:",op); - state.s11n.serialize(...args); - Atomics.store(state.sabOPView, state.opIds.rc, -1); - Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); - Atomics.notify(state.sabOPView, state.opIds.whichOp) - /* async thread will take over here */; - const t = performance.now(); - while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ - /* - The reason for this loop is buried in the details of a long - discussion at: - - https://github.com/sqlite/sqlite-wasm/issues/12 - - Summary: in at least one browser flavor, under high loads, - the wait()/notify() pairings can get out of sync. Calling - wait() here until it returns 'not-equal' gets them back in - sync. - */ - } - /* When the above wait() call returns 'not-equal', the async - half will have completed the operation and reported its results - in the state.opIds.rc slot of the SAB. */ - const rc = Atomics.load(state.sabOPView, state.opIds.rc); - metrics[op].wait += performance.now() - t; - if(rc && state.asyncS11nExceptions){ - const err = state.s11n.deserialize(); - if(err) error(op+"() async error:",...err); - } - return rc; - }; - -//#if nope + const opRun = opfsVfs.opRun; /** Not part of the public API. Only for test/development use. */ @@ -229,296 +219,6 @@ const installOpfsWlVfs = function callee(options){ W.postMessage({type: 'opfs-async-restart'}); } }; -//#endif - - /** - Map of sqlite3_file pointers to objects constructed by xOpen(). - */ - const __openFiles = Object.create(null); - - const opTimer = Object.create(null); - opTimer.op = undefined; - opTimer.start = undefined; - const mTimeStart = (op)=>{ - opTimer.start = performance.now(); - opTimer.op = op; - ++metrics[op].count; - }; - const mTimeEnd = ()=>( - metrics[opTimer.op].time += performance.now() - opTimer.start - ); - - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioSyncWrappers = { - xCheckReservedLock: function(pFile,pOut){ - /** - After consultation with a topic expert: "opfs-wl" will - continue to use the same no-op impl which "opfs" does - because: - - - xCheckReservedLock() is just a hint. If SQLite needs to - lock, it's still going to try to lock. - - - We cannot do this check synchronously in "opfs-wl", - so would need to pass it to the async proxy. That would - make it inordinately expensive considering that it's - just a hint. - */ - wasm.poke(pOut, 0, 'i32'); - return 0; - }, - xClose: function(pFile){ - mTimeStart('xClose'); - let rc = 0; - const f = __openFiles[pFile]; - if(f){ - delete __openFiles[pFile]; - rc = opRun('xClose', pFile); - if(f.sq3File) f.sq3File.dispose(); - } - mTimeEnd(); - return rc; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - /*mTimeStart('xFileControl'); - mTimeEnd();*/ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - mTimeStart('xFileSize'); - let rc = opRun('xFileSize', pFile); - if(0==rc){ - try { - const sz = state.s11n.deserialize()[0]; - wasm.poke(pSz64, sz, 'i64'); - }catch(e){ - error("Unexpected error reading xFileSize() result:",e); - rc = state.sq3Codes.SQLITE_IOERR; - } - } - mTimeEnd(); - return rc; - }, - xLock: function(pFile, lockType){ - mTimeStart('xLock'); - const f = __openFiles[pFile]; - let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ - if( !f.lockType ) { - try{ - const view = state.sabOPView; - /* We need to pass pFile's name through so that the other - side can create the WebLock name. */ - state.s11n.serialize(f.filename) - Atomics.store(view, state.lock.atomicsHandshake, 0); - Atomics.store(view, state.lock.type, lockType); - Atomics.store(view, state.opIds.whichOp, state.opIds.lockControl); - Atomics.notify(state.sabOPView, state.opIds.whichOp) - while('not-equal'!==Atomics.wait(view, state.lock.atomicsHandshake, 0)){ - /* Loop is a workaround for environment-specific quirks. See - notes in similar loops. */ - } - f.lockType = lockType; - }catch(e){ - error("xLock(",arguments,") failed", e, f); - rc = capi.SQLITE_IOERR_LOCK; - } - }else{ - f.lockType = lockType; - } - mTimeEnd(); - return rc; - }, - xRead: function(pFile,pDest,n,offset64){ - mTimeStart('xRead'); - const f = __openFiles[pFile]; - let rc; - try { - rc = opRun('xRead',pFile, n, Number(offset64)); - if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ - /** - Results get written to the SharedArrayBuffer f.sabView. - Because the heap is _not_ a SharedArrayBuffer, we have - to copy the results. TypedArray.set() seems to be the - fastest way to copy this. */ - wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); - } - }catch(e){ - error("xRead(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_READ; - } - mTimeEnd(); - return rc; - }, - xSync: function(pFile,flags){ - mTimeStart('xSync'); - ++metrics.xSync.count; - const rc = opRun('xSync', pFile, flags); - mTimeEnd(); - return rc; - }, - xTruncate: function(pFile,sz64){ - mTimeStart('xTruncate'); - const rc = opRun('xTruncate', pFile, Number(sz64)); - mTimeEnd(); - return rc; - }, - xUnlock: function(pFile,lockType){ - mTimeStart('xUnlock'); - const f = __openFiles[pFile]; - let rc = 0; - if( lockType < f.lockType ){ - try{ - const view = state.sabOPView; - Atomics.store(view, state.lock.atomicsHandshake, 1); - Atomics.notify(view, state.lock.atomicsHandshake); - Atomics.wait(view, state.lock.atomicsHandshake, 1); - }catch(e){ - error("xUnlock(",pFile,lockType,") failed",e, f); - rc = capi.SQLITE_IOERR_LOCK; - } - } - if( 0===rc ) f.lockType = lockType; - mTimeEnd(); - return rc; - }, - xWrite: function(pFile,pSrc,n,offset64){ - mTimeStart('xWrite'); - const f = __openFiles[pFile]; - let rc; - try { - f.sabView.set(wasm.heap8u().subarray( - Number(pSrc), Number(pSrc) + n - )); - rc = opRun('xWrite', pFile, n, Number(offset64)); - }catch(e){ - error("xWrite(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_WRITE; - } - mTimeEnd(); - return rc; - } - }/*ioSyncWrappers*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsSyncWrappers = { - xAccess: function(pVfs,zName,flags,pOut){ - mTimeStart('xAccess'); - const rc = opRun('xAccess', wasm.cstrToJs(zName)); - wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); - mTimeEnd(); - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - /* If it turns out that we need to adjust for timezone, see: - https://stackoverflow.com/a/11760121/1458521 */ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - mTimeStart('xDelete'); - const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); - mTimeEnd(); - return rc; - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return ilogImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); - const toss = sqlite3.util.toss; - const capi = sqlite3.capi; - const util = sqlite3.util; - const wasm = sqlite3.wasm; - const sqlite3_vfs = capi.sqlite3_vfs; - const sqlite3_file = capi.sqlite3_file; - const sqlite3_io_methods = capi.sqlite3_io_methods; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; - const opfsIoMethods = new sqlite3_io_methods(); - const opfsVfs = new sqlite3_vfs() - .addOnDispose( ()=>opfsIoMethods.dispose()); - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; + const state = opfsUtil.createVfsState('opfs', options), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + __openFiles = opfsVfs.__openFiles; + delete state.vfs; - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit - is undocumented/unspecified. */; - opfsVfs.$zName = wasm.allocCString("opfs"); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.addOnDispose( - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) - /** - Pedantic sidebar: the entries in this array are items to - clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown - order of the wasm engine and the JS one are undefined so - there is no guaranty that the opfsVfs instance would be - available in one environment or the other when - sqlite3_os_end() is called (_if_ it gets called at all in a - wasm build, which is undefined). - */ - ); + /* At this point, createVfsState() has populated state and + opfsVfs with any code common to both the "opfs" and "opfs-wl" + VFSes. Now comes the VFS-dependent work... */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = opfsUtil.createVfsStateObject(opfsVfs); - state.verbose = options.verbose; - const metrics = opfsVfs.metrics.counters; let promiseWasRejected = undefined; const promiseReject = (err)=>{ promiseWasRejected = true; @@ -192,14 +129,14 @@ const installOpfsVfs = function callee(options){ promiseWasRejected = false; return promiseResolve_(sqlite3); }; - const workerArgs = '?vfs=opfs'; - const W = + options.proxyUri += '?vfs=opfs'; + const W = opfsVfs.worker = //#if target:es6-bundler-friendly new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); //#elif target:es6-module - new Worker(new URL(options.proxyUri+workerArgs, import.meta.url)); + new Worker(new URL(options.proxyUri, import.meta.url)); //#else - new Worker(options.proxyUri+workerArgs); + new Worker(options.proxyUri); //#endif setTimeout(()=>{ /* At attempt to work around a browser-specific quirk in which @@ -221,46 +158,7 @@ const installOpfsVfs = function callee(options){ promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); }; - /** - Runs the given operation (by name) in the async worker - counterpart, waits for its response, and returns the result - which the async worker writes to SAB[state.opIds.rc]. The - 2nd and subsequent arguments must be the arguments for the - async op. - */ - const opRun = (op,...args)=>{ - const opNdx = state.opIds[op] || toss("Invalid op ID:",op); - state.s11n.serialize(...args); - Atomics.store(state.sabOPView, state.opIds.rc, -1); - Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); - Atomics.notify(state.sabOPView, state.opIds.whichOp) - /* async thread will take over here */; - const t = performance.now(); - while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ - /* - The reason for this loop is buried in the details of a long - discussion at: - - https://github.com/sqlite/sqlite-wasm/issues/12 - - Summary: in at least one browser flavor, under high loads, - the wait()/notify() pairings can get out of sync. Calling - wait() here until it returns 'not-equal' gets them back in - sync. - */ - } - /* When the above wait() call returns 'not-equal', the async - half will have completed the operation and reported its results - in the state.opIds.rc slot of the SAB. */ - const rc = Atomics.load(state.sabOPView, state.opIds.rc); - metrics[op].wait += performance.now() - t; - if(rc && state.asyncS11nExceptions){ - const err = state.s11n.deserialize(); - if(err) error(op+"() async error:",...err); - } - return rc; - }; - + const opRun = opfsVfs.opRun; //#if nope /** Not part of the public API. Only for test/development use. @@ -277,272 +175,38 @@ const installOpfsVfs = function callee(options){ }; //#endif - /** - Map of sqlite3_file pointers to objects constructed by xOpen(). - */ - const __openFiles = Object.create(null); - - const opTimer = Object.create(null); - opTimer.op = undefined; - opTimer.start = undefined; - const mTimeStart = (op)=>{ - opTimer.start = performance.now(); - opTimer.op = op; - ++metrics[op].count; - }; - const mTimeEnd = ()=>( - metrics[opTimer.op].time += performance.now() - opTimer.start - ); - - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioSyncWrappers = { - xCheckReservedLock: function(pFile,pOut){ - /** - As of late 2022, only a single lock can be held on an OPFS - file. We have no way of checking whether any _other_ db - connection has a lock except by trying to obtain and (on - success) release a sync-handle for it, but doing so would - involve an inherent race condition. For the time being, - pending a better solution, we simply report whether the - given pFile is open. - - Update 2024-06-12: based on forum discussions, this - function now always sets pOut to 0 (false): - - https://sqlite.org/forum/forumpost/a2f573b00cda1372 - */ - wasm.poke(pOut, 0, 'i32'); - return 0; - }, - xClose: function(pFile){ - mTimeStart('xClose'); - let rc = 0; - const f = __openFiles[pFile]; - if(f){ - delete __openFiles[pFile]; - rc = opRun('xClose', pFile); - if(f.sq3File) f.sq3File.dispose(); - } - mTimeEnd(); - return rc; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - /*mTimeStart('xFileControl'); - mTimeEnd();*/ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - mTimeStart('xFileSize'); - let rc = opRun('xFileSize', pFile); - if(0==rc){ - try { - const sz = state.s11n.deserialize()[0]; - wasm.poke(pSz64, sz, 'i64'); - }catch(e){ - error("Unexpected error reading xFileSize() result:",e); - rc = state.sq3Codes.SQLITE_IOERR; - } - } - mTimeEnd(); - return rc; - }, - xLock: function(pFile,lockType){ - mTimeStart('xLock'); - const f = __openFiles[pFile]; - let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ - if( !f.lockType ) { - rc = opRun('xLock', pFile, lockType); - if( 0===rc ) f.lockType = lockType; - }else{ - f.lockType = lockType; - } - mTimeEnd(); - return rc; - }, - xRead: function(pFile,pDest,n,offset64){ - mTimeStart('xRead'); - const f = __openFiles[pFile]; - let rc; - try { - rc = opRun('xRead',pFile, n, Number(offset64)); - if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ - /** - Results get written to the SharedArrayBuffer f.sabView. - Because the heap is _not_ a SharedArrayBuffer, we have - to copy the results. TypedArray.set() seems to be the - fastest way to copy this. */ - wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); - } - }catch(e){ - error("xRead(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_READ; - } - mTimeEnd(); - return rc; - }, - xSync: function(pFile,flags){ - mTimeStart('xSync'); - ++metrics.xSync.count; - const rc = opRun('xSync', pFile, flags); - mTimeEnd(); - return rc; - }, - xTruncate: function(pFile,sz64){ - mTimeStart('xTruncate'); - const rc = opRun('xTruncate', pFile, Number(sz64)); - mTimeEnd(); - return rc; - }, - xUnlock: function(pFile,lockType){ - mTimeStart('xUnlock'); - const f = __openFiles[pFile]; - let rc = 0; - if( capi.SQLITE_LOCK_NONE === lockType - && f.lockType ){ - rc = opRun('xUnlock', pFile, lockType); - } + opfsVfs.ioSyncWrappers.xLock = function(pFile,lockType){ + mTimeStart('xLock'); + ++metrics.xLock.count; + const f = __openFiles[pFile]; + let rc = 0; + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( f.lockType ) { + f.lockType = lockType; + }else{ + rc = opRun('xLock', pFile, lockType); if( 0===rc ) f.lockType = lockType; - mTimeEnd(); - return rc; - }, - xWrite: function(pFile,pSrc,n,offset64){ - mTimeStart('xWrite'); - const f = __openFiles[pFile]; - let rc; - try { - f.sabView.set(wasm.heap8u().subarray( - Number(pSrc), Number(pSrc) + n - )); - rc = opRun('xWrite', pFile, n, Number(offset64)); - }catch(e){ - error("xWrite(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_WRITE; - } - mTimeEnd(); - return rc; } - }/*ioSyncWrappers*/; + mTimeEnd(); + return rc; + }; - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsSyncWrappers = { - xAccess: function(pVfs,zName,flags,pOut){ - mTimeStart('xAccess'); - const rc = opRun('xAccess', wasm.cstrToJs(zName)); - wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); - mTimeEnd(); - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - /* If it turns out that we need to adjust for timezone, see: - https://stackoverflow.com/a/11760121/1458521 */ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - mTimeStart('xDelete'); - const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); - mTimeEnd(); - return rc; - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return i Date: Wed, 4 Mar 2026 19:21:09 +0000 Subject: [PATCH 11/32] Consolidate the last 200 lines of common OPFS VFS code. "opfs" still works, "opfs-wl" registers fine but is still otherwise untested. FossilOrigin-Name: 5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88 --- ext/wasm/api/opfs-common-shared.c-pp.js | 287 ++++++++++++++++++++-- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 296 +++-------------------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 267 +++----------------- manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 334 insertions(+), 534 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index feefca5eb4..5150fb8e61 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -18,19 +18,18 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; - const toss = sqlite3.util.toss; - const capi = sqlite3.capi; - const util = sqlite3.util; - const wasm = sqlite3.wasm; + const toss = sqlite3.util.toss, + capi = sqlite3.capi, + util = sqlite3.util, + wasm = sqlite3.wasm; /** Generic utilities for working with OPFS. This will get filled out by the Promise setup and, on success, installed as sqlite3.opfs. - This is an internal/private namespace intended for use solely - by the OPFS VFSes and test code for them. The library bootstrapping + This is an internal/private namespace intended for use solely by + the OPFS VFSes and test code for them. The library bootstrapping process removes this object in non-testing contexts. - */ const opfsUtil = sqlite3.opfs = Object.create(null); @@ -421,14 +420,44 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; + /** + Must be called by the VFS's main installation routine and passed + the options object that function receives and a reference to that + function itself (which is assumed to have a defaultProxyUri + property set on it. See sqlite3-vfs-opfs{,-wl}.c-pp.js for + examples. + + It throws if OPFS is not available. + + If it returns falsy, it detected that OPFS should be disabled, in + which case the callee should immediately return/resolve to the + sqlite3 object. + + Else it returns a new copy of the options object, fleshed out + with any missing defaults. The caller must: + + - Set up any local state they need. + + - Call opfsUtil.createVfsState(vfsName,opt), where opt is the + object returned by this function. + + - Set up any references they may need to state returned + by the previous step. + + - Call opfvs.doTheThing() + */ opfsUtil.initOptions = function(options, callee){ - options = util.nu(options); const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('opfs-disable')){ //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); - options.disableOpfs = true; - return options; + return; } + try{ + opfsUtil.vfsInstallationFeatureCheck(); + }catch(e){ + return; + } + options = util.nu(options); if(undefined===options.verbose){ options.verbose = urlParams.has('opfs-verbose') ? (+urlParams.get('opfs-verbose') || 2) : 1; @@ -449,14 +478,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Creates and populates the main state object used by "opfs" and "opfs-wl", and transfered from those to their async counterpart. - Returns an object containing state which we send to the async-api - Worker or share with it. + Returns an object containing state which we send to the async + proxy Worker. - Because the returned object must be serializable to be posted to - the async proxy, after this returns, the caller must: + The returned object's vfs property holds the fully-populated + capi.sqlite3_vfs instance. - - Make a local-scope reference of state.vfs then (delete - state.vfs). That's the capi.sqlite3_vfs instance for the VFS. + After setting up any local state needed, the caller must + call theVfs.doTheThing(X,Y), where X is an object containing + the sqlite3_io_methods to override and Y is a callback which + gets triggered if init succeeds, before the final Promise + decides whether or not to reject. The result of doTheThing() + must be returned from their main installation function. This object must, when it's passed to the async part, contain only cloneable or sharable objects. After the worker's "inited" @@ -1013,6 +1046,228 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //#include api/opfs-common-inline.c-pp.js //#undef vfs.metrics.enable opfsVfs.initS11n = initS11n; + + /** + To be called by the VFS's main installation routine after it has + wired up enough state to provide its overridden io-method impls + (which must be properties of the ioMethods argument). Returns a + Promise which the installation routine must return. callback must + be a function which performs any post-bootstrap touchups, namely + plugging in a sqlite3.oo1 wrapper. It is passed (sqlite3, opfsVfs), + where opfsVfs is the sqlite3_vfs object which was set up by + opfsUtil.createVfsState(). + */ + opfsVfs.doTheThing = function(ioMethods, callback){ + Object.assign(opfsVfs.ioSyncWrappers, ioMethods); + const thePromise = new Promise(function(promiseResolve_, promiseReject_){ + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; + + let promiseWasRejected = undefined; + const promiseReject = (err)=>{ + promiseWasRejected = true; + opfsVfs.dispose(); + return promiseReject_(err); + }; + const promiseResolve = ()=>{ + try{ + callback(sqlite3, opfsVfs); + }catch(e){ + return promiseReject(e); + } + promiseWasRejected = false; + return promiseResolve_(sqlite3); + }; + options.proxyUri += '?vfs='+vfsName; + const W = opfsVfs.worker = +//#if target:es6-bundler-friendly + new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); +//#elif target:es6-module + new Worker(new URL(options.proxyUri, import.meta.url)); +//#else + new Worker(options.proxyUri); +//#endif + setTimeout(()=>{ + /* At attempt to work around a browser-specific quirk in which + the Worker load is failing in such a way that we neither + resolve nor reject it. This workaround gives that resolve/reject + a time limit and rejects if that timer expires. Discussion: + https://sqlite.org/forum/forumpost/a708c98dcb3ef */ + if(undefined===promiseWasRejected){ + promiseReject( + new Error("Timeout while waiting for OPFS async proxy worker.") + ); + } + }, 4000); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + + const opRun = opfsVfs.opRun; +//#if nope + /** + Not part of the public API. Only for test/development use. + */ + opfsVfs.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; +//#endif + + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new capi.sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.poke(pOut,0,'i64'); + rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.peek(pOut, 'i64')); + rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.poke(readBuf+6,0); + let jRead = wasm.cstrToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(opfsVfs.vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = opfsVfs.ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //log("Worker.onmessage:",data); + switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; + case 'opfs-async-loaded': + /* Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async + worker. */ + delete state.vfs; + W.postMessage({type: 'opfs-async-init', args: util.nu(state)}); + break; + case 'opfs-async-inited': { + /* Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin... */ + if(true===promiseWasRejected){ + break /* promise was already rejected via timer */; + } + try { + sqlite3.vfs.installVfs({ + io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} + }); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + opfsVfs.initS11n(); + delete opfsVfs.initS11n; + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(); + }).catch(promiseReject); + }else{ + promiseResolve(); + } + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + default: { + const errMsg = ( + "Unexpected message from the OPFS async worker: " + + JSON.stringify(data) + ); + error(errMsg); + promiseReject(new Error(errMsg)); + break; + } + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; + }/*doTheThing()*/; + return state; }/*createVfsState()*/; diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 134b62b107..e62f23ed14 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -36,9 +36,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const util = sqlite3.util, toss = sqlite3.util.toss; - const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") - /* These get removed from sqlite3 during bootstrap, so we need an - early reference to it. */; + const opfsUtil = sqlite3.opfs || toss("Missing sqlite3.opfs"); /** installOpfsWlVfs() returns a Promise which, on success, installs an @@ -46,7 +44,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ which accept a VFS. It is intended to be called via sqlite3ApiBootstrap.initializers or an equivalent mechanism. - This VFS is essentially a copy of the "opfs" VFS but uses + This VFS is essentially identical to the "opfs" VFS but uses WebLocks for its xLock() and xUnlock() implementations. Quirks specific to this VFS: @@ -54,69 +52,30 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Because WebLocks effectively block until they return, they will effectively hang on locks rather than returning SQLITE_BUSY. - - The argument may optionally be a plain object with the following - configuration options: - - - proxyUri: name of the async proxy JS file. - - - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables - logging of errors. 2 enables logging of warnings and errors. 3 - additionally enables debugging info. Logging is performed - via the sqlite3.config.{log|warn|error}() functions. - - - On success, the Promise resolves to the top-most sqlite3 namespace - object. Success does not necessarily mean that it installs the VFS, - as there are legitimate non-error reasons for OPFS not to be - available. + Aside from locking differences in the VFSes, this function + otherwise behaves the same as + sqlite3-vfs-opfs.c-pp.js:installOpfsVfs(). */ -const installOpfsWlVfs = function callee(options){ - try{ - opfsUtil.vfsInstallationFeatureCheck(); - }catch(e){ - return Promise.reject(e); - } +const installOpfsWlVfs = async function callee(options){ options = opfsUtil.initOptions(options, callee); - if( options.disableOpfs ){ - return Promise.resolve(sqlite3); - } - - const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args), - warn = (...args)=>logImpl(1, ...args), - error = (...args)=>logImpl(0, ...args), - capi = sqlite3.capi, - wasm = sqlite3.wasm; - const state = opfsUtil.createVfsState('opfs-wl', options), - opfsVfs = state.vfs, - metrics = opfsVfs.metrics.counters, - mTimeStart = opfsVfs.mTimeStart, - mTimeEnd = opfsVfs.mTimeEnd, - __openFiles = opfsVfs.__openFiles; - delete state.vfs; - - /* At this point, createVfsState() has populated state and - opfsVfs with any code common to both the "opfs" and "opfs-wl" - VFSes. Now comes the VFS-dependent work... */ - - opfsVfs.ioSyncWrappers.xLock = function(pFile, lockType){ + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState('opfs-wl', options), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + __openFiles = opfsVfs.__openFiles; + /* At this point, createVfsState() has populated state and opfsVfs + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... */ + return opfsVfs.doTheThing(util.nu({ + xLock: function(pFile, lockType){ mTimeStart('xLock'); ++metrics.xLock.count; const f = __openFiles[pFile]; let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ + /* See notes in sqlite3-vfs-opfs.c-pp.js. */ if( f.lockType ) { f.lockType = lockType; }else{ @@ -141,9 +100,8 @@ const installOpfsWlVfs = function callee(options){ } mTimeEnd(); return rc; - }; - - opfsVfs.ioSyncWrappers.xUnlock =function(pFile,lockType){ + }, + xUnlock: function(pFile,lockType){ mTimeStart('xUnlock'); ++metrics.xUnlock.count; const f = __openFiles[pFile]; @@ -162,64 +120,8 @@ const installOpfsWlVfs = function callee(options){ if( 0===rc ) f.lockType = lockType; mTimeEnd(); return rc; - }; - - - - let promiseWasRejected = undefined; - const promiseReject = (err)=>{ - promiseWasRejected = true; - opfsVfs.dispose(); - return promiseReject_(err); - }; - const promiseResolve = ()=>{ - promiseWasRejected = false; - return promiseResolve_(sqlite3); - }; - options.proxyUri += '?vfs=opfs-wl'; - const W = opfsVfs.worker = -//#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); -//#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); -//#else - new Worker(options.proxyUri); -//#endif - setTimeout(()=>{ - /* At attempt to work around a browser-specific quirk in which - the Worker load is failing in such a way that we neither - resolve nor reject it. This workaround gives that resolve/reject - a time limit and rejects if that timer expires. Discussion: - https://sqlite.org/forum/forumpost/a708c98dcb3ef */ - if(undefined===promiseWasRejected){ - promiseReject( - new Error("Timeout while waiting for OPFS async proxy worker.") - ); - } - }, 4000); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - error("Error initializing OPFS asyncer:",err); - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - - const opRun = opfsVfs.opRun; - /** - Not part of the public API. Only for test/development use. - */ - opfsUtil.debug = { - asyncShutdown: ()=>{ - warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); - opRun('opfs-async-shutdown'); - }, - asyncRestart: ()=>{ - warn("Attempting to restart OPFS VFS async listener. Might work, might not."); - W.postMessage({type: 'opfs-async-restart'}); - } - }; - + } + }), function(sqlite3, vfs){ if(sqlite3.oo1){ const OpfsWlDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -229,156 +131,14 @@ const installOpfsWlVfs = function callee(options){ OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsWlDb = OpfsWlDb; OpfsWlDb.importDb = opfsUtil.importDb; -//#if nope - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( - opfsVfs.pointer, - function(oo1Db, sqlite3){ - /* Set a relatively high default busy-timeout handler to - help OPFS dbs deal with multi-tab/multi-worker - contention. */ - sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); - } - ); -//#endif }/*extend sqlite3.oo1*/ - - const sanityCheck = function(){ - const scope = wasm.scopedAllocPush(); - const sq3File = new capi.sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"+randomFilename(8); - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - state.s11n.serialize("This is ä string."); - rc = state.s11n.deserialize(); - log("deserialize() says:",rc); - if("This is ä string."!==rc[0]) toss("String d13n error."); - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.sabOPView[xOpen] =", - state.sabOPView[state.opIds.xOpen]); - if(0!==rc){ - error("open failed with code",rc); - return; - } - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.poke(pOut,0,'i64'); - rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.peek(pOut, 'i64')); - rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.poke(readBuf+6,0); - let jRead = wasm.cstrToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - if(opfsVfs.vfsSyncWrappers.xSleep){ - log("xSleep()ing before close()ing..."); - opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); - log("waking up from xSleep()"); - } - rc = opfsVfs.ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"sabOPView =",state.sabOPView); - log("Deleting file:",dbFile); - opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - warn("End of OPFS sanity checks."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); - } - }/*sanityCheck()*/; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'opfs-unavailable': - /* Async proxy has determined that OPFS is unavailable. There's - nothing more for us to do here. */ - promiseReject(new Error(data.payload.join(' '))); - break; - case 'opfs-async-loaded': - /* Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async - worker. */ - W.postMessage({type: 'opfs-async-init', args: util.nu(state)}); - break; - case 'opfs-async-inited': { - /* Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin... */ - if(true===promiseWasRejected){ - break /* promise was already rejected via timer */; - } - try { - sqlite3.vfs.installVfs({ - io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} - }); - state.sabOPView = new Int32Array(state.sabOP); - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - opfsVfs.initS11n(); - delete opfsVfs.initS11n; - if(options.sanityChecks){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - if(opfsUtil.thisThreadHasOPFS()){ - opfsUtil.getRootDir().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - log("End of OPFS-WL sqlite3_vfs setup.", opfsVfs); - promiseResolve(); - }).catch(promiseReject); - }else{ - promiseResolve(); - } - }catch(e){ - error(e); - promiseReject(e); - } - break; - } - default: { - const errMsg = ( - "Unexpected message from the OPFS async worker: " + - JSON.stringify(data) - ); - error(errMsg); - promiseReject(new Error(errMsg)); - break; - } - }/*switch(data.type)*/ - }/*W.onmessage()*/; - })/*thePromise*/; - return thePromise; + })/*doTheThing()*/; }/*installOpfsWlVfs()*/; -installOpfsWlVfs.defaultProxyUri = - "sqlite3-opfs-async-proxy.js"; +installOpfsWlVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ let proxyJs = installOpfsWlVfs.defaultProxyUri; - if( sqlite3?.scriptInfo?.sqlite3Dir ){ + if( sqlite3.scriptInfo?.sqlite3Dir ){ installOpfsWlVfs.defaultProxyUri = sqlite3.scriptInfo.sqlite3Dir + proxyJs; //sqlite3.config.warn("installOpfsWlVfs.defaultProxyUri =",installOpfsWlVfs.defaultProxyUri); @@ -392,6 +152,4 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ } }); }/*sqlite3ApiBootstrap.initializers.push()*/); -//#else -/* The OPFS VFS parts are elided from builds targeting node.js. */ //#endif target:node diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 7b72199e65..6fe317ec8d 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -21,10 +21,7 @@ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const util = sqlite3.util, - toss = sqlite3.util.toss; - const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") - /* These get removed from sqlite3 during bootstrap, so we need an - early reference to it. */; + opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs"); /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs @@ -75,107 +72,31 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Promise resolves. This is only intended for testing and development of the VFS, not client-side use. + Additionaly, the (officially undocumented) 'opfs-disable' URL + argument will disable OPFS, making this function a no-op. + On success, the Promise resolves to the top-most sqlite3 namespace object. Success does not necessarily mean that it installs the VFS, as there are legitimate non-error reasons for OPFS not to be available. */ -const installOpfsVfs = function callee(options){ - try{ - opfsUtil.vfsInstallationFeatureCheck(); - }catch(e){ - return Promise.reject(e); - } +const installOpfsVfs = async function callee(options){ options = opfsUtil.initOptions(options, callee); - if( options.disableOpfs ){ - return Promise.resolve(sqlite3); - } + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState('opfs', options), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + __openFiles = opfsVfs.__openFiles; - //sqlite3.config.warn("OPFS options =",options,globalThis.location); - const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args), - warn = (...args)=>logImpl(1, ...args), - error = (...args)=>logImpl(0, ...args), - capi = sqlite3.capi, - wasm = sqlite3.wasm; - - const state = opfsUtil.createVfsState('opfs', options), - opfsVfs = state.vfs, - metrics = opfsVfs.metrics.counters, - mTimeStart = opfsVfs.mTimeStart, - mTimeEnd = opfsVfs.mTimeEnd, - __openFiles = opfsVfs.__openFiles; - delete state.vfs; - - /* At this point, createVfsState() has populated state and - opfsVfs with any code common to both the "opfs" and "opfs-wl" - VFSes. Now comes the VFS-dependent work... */ - - let promiseWasRejected = undefined; - const promiseReject = (err)=>{ - promiseWasRejected = true; - opfsVfs.dispose(); - return promiseReject_(err); - }; - const promiseResolve = ()=>{ - promiseWasRejected = false; - return promiseResolve_(sqlite3); - }; - options.proxyUri += '?vfs=opfs'; - const W = opfsVfs.worker = -//#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); -//#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); -//#else - new Worker(options.proxyUri); -//#endif - setTimeout(()=>{ - /* At attempt to work around a browser-specific quirk in which - the Worker load is failing in such a way that we neither - resolve nor reject it. This workaround gives that resolve/reject - a time limit and rejects if that timer expires. Discussion: - https://sqlite.org/forum/forumpost/a708c98dcb3ef */ - if(undefined===promiseWasRejected){ - promiseReject( - new Error("Timeout while waiting for OPFS async proxy worker.") - ); - } - }, 4000); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - error("Error initializing OPFS asyncer:",err); - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - - const opRun = opfsVfs.opRun; -//#if nope - /** - Not part of the public API. Only for test/development use. - */ - opfsUtil.debug = { - asyncShutdown: ()=>{ - warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); - opRun('opfs-async-shutdown'); - }, - asyncRestart: ()=>{ - warn("Attempting to restart OPFS VFS async listener. Might work, might not."); - W.postMessage({type: 'opfs-async-restart'}); - } - }; -//#endif - - opfsVfs.ioSyncWrappers.xLock = function(pFile,lockType){ + /* At this point, createVfsState() has populated state and + opfsVfs with any code common to both the "opfs" and "opfs-wl" + VFSes. Now comes the VFS-dependent work... */ + return opfsVfs.doTheThing(util.nu({ + xLock: function(pFile,lockType){ mTimeStart('xLock'); ++metrics.xLock.count; const f = __openFiles[pFile]; @@ -192,9 +113,8 @@ const installOpfsVfs = function callee(options){ } mTimeEnd(); return rc; - }; - - opfsVfs.ioSyncWrappers.xUnlock = function(pFile,lockType){ + }, + xUnlock: function(pFile,lockType){ mTimeStart('xUnlock'); ++metrics.xUnlock.count; const f = __openFiles[pFile]; @@ -206,12 +126,12 @@ const installOpfsVfs = function callee(options){ if( 0===rc ) f.lockType = lockType; mTimeEnd(); return rc; - }; - + } + }), function(sqlite3, vfs){ if(sqlite3.oo1){ const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; + opt.vfs = vfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); @@ -227,144 +147,13 @@ const installOpfsVfs = function callee(options){ } ); }/*extend sqlite3.oo1*/ - - const sanityCheck = function(){ - const scope = wasm.scopedAllocPush(); - const sq3File = new capi.sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"+randomFilename(8); - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - state.s11n.serialize("This is ä string."); - rc = state.s11n.deserialize(); - log("deserialize() says:",rc); - if("This is ä string."!==rc[0]) toss("String d13n error."); - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.sabOPView[xOpen] =", - state.sabOPView[state.opIds.xOpen]); - if(0!==rc){ - error("open failed with code",rc); - return; - } - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.poke(pOut,0,'i64'); - rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.peek(pOut, 'i64')); - rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.poke(readBuf+6,0); - let jRead = wasm.cstrToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - if(opfsVfs.vfsSyncWrappers.xSleep){ - log("xSleep()ing before close()ing..."); - opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); - log("waking up from xSleep()"); - } - rc = opfsVfs.ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"sabOPView =",state.sabOPView); - log("Deleting file:",dbFile); - opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - warn("End of OPFS sanity checks."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); - } - }/*sanityCheck()*/; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'opfs-unavailable': - /* Async proxy has determined that OPFS is unavailable. There's - nothing more for us to do here. */ - promiseReject(new Error(data.payload.join(' '))); - break; - case 'opfs-async-loaded': - /* Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async - worker. */ - W.postMessage({type: 'opfs-async-init',args: state}); - break; - case 'opfs-async-inited': { - /* Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin... */ - if(true===promiseWasRejected){ - break /* promise was already rejected via timer */; - } - try { - sqlite3.vfs.installVfs({ - io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} - }); - state.sabOPView = new Int32Array(state.sabOP); - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - opfsVfs.initS11n(); - delete opfsVfs.initS11n; - if(options.sanityChecks){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - if(opfsUtil.thisThreadHasOPFS()){ - opfsUtil.getRootDir().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); - promiseResolve(); - }).catch(promiseReject); - }else{ - promiseResolve(); - } - }catch(e){ - error(e); - promiseReject(e); - } - break; - } - default: { - const errMsg = ( - "Unexpected message from the OPFS async worker: " + - JSON.stringify(data) - ); - error(errMsg); - promiseReject(new Error(errMsg)); - break; - } - }/*switch(data.type)*/ - }/*W.onmessage()*/; - })/*thePromise*/; - return thePromise; + })/*doTheThing()*/; }/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = - "sqlite3-opfs-async-proxy.js"; +installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ let proxyJs = installOpfsVfs.defaultProxyUri; - if( sqlite3?.scriptInfo?.sqlite3Dir ){ + if( sqlite3.scriptInfo?.sqlite3Dir ){ installOpfsVfs.defaultProxyUri = sqlite3.scriptInfo.sqlite3Dir + proxyJs; //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); @@ -378,6 +167,4 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ } }); }/*sqlite3ApiBootstrap.initializers.push()*/); -//#else -/* The OPFS VFS parts are elided from builds targeting node.js. */ //#endif target:node diff --git a/manifest b/manifest index 34b1032854..003a670b2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sout\sabout\s300\slines\sof\scommon\sOPFS\sVFS\sbootstrapping\scode. -D 2026-03-04T17:54:02.085 +C Consolidate\sthe\slast\s200\slines\sof\scommon\sOPFS\sVFS\scode.\s"opfs"\sstill\sworks,\s"opfs-wl"\sregisters\sfine\sbut\sis\sstill\sotherwise\suntested. +D 2026-03-04T19:21:09.278 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 91b1291447c689a77ffcc3297dc478dd29196311facb063737aeaaf70660a0f0 +F ext/wasm/api/opfs-common-shared.c-pp.js d8ecb1c7f6b29c2eb501ab8da6f9d9867c1ceb8a42c9c883dd53aed8ddfe106a F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -598,8 +598,8 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f0a2aa8712211ff9db2ef548ae8b676b F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 929bad4b98f176b2d0a8c1509ca833b42a11f5f0871d2b3bb2597b9e29c8ea24 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 9babe167f28ecd8fe67c97fe0734ec88beecbb61a0580d5218edcb8b3d8670ce +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js a755ea941f254f89fcd519789097a7401362d9e9dfba19a9bfc972861257c3c5 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 50a955ef393722d498177ad09c9e2d05bbe8dccae4c40c501482a860ca30017d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P b71c79ef9672c77a72a976ffcd7cbebfaf0ff314dff97b274f7d092de6a7773f -R c0d13d0a3e8b693179cc92fc3fedae39 +P 57adecbab71795b62b1c2e4570ff504f35681e81dd8c94f78ad8e05ef39d36fd +R 18d89b2ee7fb09440d0eac4e9c8dc540 U stephan -Z 3ba0938d21136544be85b7e2c587b666 +Z d66223f806eab9c2d766ad2e9d73c95d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a8ccc41dc5..5c99fbd65b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57adecbab71795b62b1c2e4570ff504f35681e81dd8c94f78ad8e05ef39d36fd +5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88 From 232446b532f6a88395bf1ca3583942a7a11ac59a Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 20:33:21 +0000 Subject: [PATCH 12/32] Get opfs-wl plugged in to the concurrency tester. Somewhat ironically, all competing workers fail with locking errors while worker 1 is busy running off the rails somewhere. Stashing for closer investigation later. FossilOrigin-Name: 3b27310aa29ea84f459974981a600301abac5c705029a289d3872ecacf231da3 --- ext/wasm/api/opfs-common-shared.c-pp.js | 67 +++++++++++++---------- ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js | 3 + ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 5 +- ext/wasm/tests/opfs/concurrency/test.js | 12 +++- ext/wasm/tests/opfs/concurrency/worker.js | 16 +++++- manifest | 20 +++---- manifest.uuid | 2 +- 7 files changed, 77 insertions(+), 48 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 5150fb8e61..72c366adab 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -499,6 +499,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const state = util.nu(); state.verbose = options.verbose; + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; + const opfsVfs = state.vfs = new capi.sqlite3_vfs(); const opfsIoMethods = opfsVfs.ioMethods = new capi.sqlite3_io_methods(); @@ -571,7 +585,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ of this value is also used for determining how long to wait on lock contention to free up. */ - state.asyncIdleWaitTime = isWebLocker ? 100 : 150; + state.asyncIdleWaitTime = isWebLocker ? 300 : 150; /** Whether the async counterpart should log exceptions to @@ -966,15 +980,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ convey error messages from xOpen() because there would be a race condition between sqlite3_open()'s call to xOpen() and this function. */ - warn("OPFS xGetLastError() has nothing sensible to return."); + sqlite3.config.warn("OPFS xGetLastError() has nothing sensible to return."); return 0; }, //xSleep is optionally defined below xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ mTimeStart('xOpen'); let opfsFlags = 0; - if(0===zName){ - zName = opfsUtil.randomFilename(); + let jzName, zToFree; + if( !zName ){ + jzName = opfsUtil.randomFilename(); + zName = zToFree = wasm.allocCString(jzName); }else if(wasm.isPtr(zName)){ if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ /* -----------------------^^^^^ MUST pass the untranslated @@ -984,18 +1000,24 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(capi.sqlite3_uri_boolean(zName, "delete-before-open", 0)){ opfsFlags |= state.opfsFlags.OPFS_UNLINK_BEFORE_OPEN; } - zName = wasm.cstrToJs(zName); - //warn("xOpen zName =",zName, "opfsFlags =",opfsFlags); + jzName = wasm.cstrToJs(zName); + //sqlite3.config.warn("xOpen zName =",zName, "opfsFlags =",opfsFlags); + }else{ + sqlite3.config.error("Impossible zName value in xOpen?", zName); + return capi.SQLITE_CANTOPEN; } - const fh = Object.create(null); - fh.fid = pFile; - fh.filename = wasm.cstrToJs(zName); - fh.sab = new SharedArrayBuffer(state.fileBufferSize); - fh.flags = flags; - fh.readOnly = !(capi.SQLITE_OPEN_CREATE & flags) - && !!(flags & capi.SQLITE_OPEN_READONLY); - const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); - if(!rc){ + const fh = util.nu({ + fid: pFile, + filename: jzName, + sab: new SharedArrayBuffer(state.fileBufferSize), + flags: flags, + readOnly: !(capi.SQLITE_OPEN_CREATE & flags) + && !!(flags & capi.SQLITE_OPEN_READONLY) + }); + const rc = opRun('xOpen', pFile, jzName, flags, opfsFlags); + if(rc){ + if( zToFree ) wasm.dealloc(zToFree); + }else{ /* Recall that sqlite3_vfs::xClose() will be called, even on error, unless pFile->pMethods is NULL. */ if(fh.readOnly){ @@ -1004,6 +1026,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ __openFiles[pFile] = fh; fh.sabView = state.sabFileBufView; fh.sq3File = new capi.sqlite3_file(pFile); + if( zToFree ) fh.sq3File.addOnDispose(zToFree); fh.sq3File.$pMethods = opfsIoMethods.pointer; fh.lockType = capi.SQLITE_LOCK_NONE; } @@ -1060,20 +1083,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ opfsVfs.doTheThing = function(ioMethods, callback){ Object.assign(opfsVfs.ioSyncWrappers, ioMethods); const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args), - warn = (...args)=>logImpl(1, ...args), - error = (...args)=>logImpl(0, ...args), - capi = sqlite3.capi, - wasm = sqlite3.wasm; - let promiseWasRejected = undefined; const promiseReject = (err)=>{ promiseWasRejected = true; diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index e3fb722877..0fdd7cb4cc 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -1137,9 +1137,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*.ioJrnl*/ }/*methodOverrides*/; +//#if nope debug("pVfs and friends", pVfs, pIoDb, pIoJrnl, kvvfsMethods, capi.sqlite3_file.structInfo, KVVfsFile.structInfo); +//#endif + try { util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough" /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index e62f23ed14..4b2094ee81 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -60,6 +60,7 @@ const installOpfsWlVfs = async function callee(options){ options = opfsUtil.initOptions(options, callee); if( !options ) return sqlite3; const capi = sqlite3.capi, + debug = sqlite3.config.debug, state = opfsUtil.createVfsState('opfs-wl', options), opfsVfs = state.vfs, metrics = opfsVfs.metrics.counters, @@ -94,7 +95,7 @@ const installOpfsWlVfs = async function callee(options){ } f.lockType = lockType; }catch(e){ - error("xLock(",arguments,") failed", e, f); + sqlite3.config.error("xLock(",arguments,") failed", e, f); rc = capi.SQLITE_IOERR_LOCK; } } @@ -113,7 +114,7 @@ const installOpfsWlVfs = async function callee(options){ Atomics.notify(view, state.lock.atomicsHandshake); Atomics.wait(view, state.lock.atomicsHandshake, 1); }catch(e){ - error("xUnlock(",pFile,lockType,") failed",e, f); + sqlite3.config.error("xUnlock(",pFile,lockType,") failed",e, f); rc = capi.SQLITE_IOERR_LOCK; } } diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 1848901afe..c2932290b7 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -55,8 +55,8 @@ const options = Object.create(null); options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); options.workerCount = ( - urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 - ) || 4; + urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 0 + ) || 3; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 ) || 1; @@ -69,6 +69,7 @@ options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; + options.vfs = urlArgsHtml.get('vfs') || 'opfs'; options.noUnlink = !!urlArgsHtml.has('no-unlink'); const workers = []; workers.post = (type,...args)=>{ @@ -118,12 +119,16 @@ const eTestLinks = document.querySelector('#testlinks'); const optArgs = function(obj){ const li = []; - for(const k of ['interval','iterations','workers','verbose','unlock-asap']){ + for(const k of [ + 'interval','iterations','unlock-asap', + 'verbose','vfs','workers', + ]){ if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]); } return li.join('&'); }; for(const opt of [ + {interval: 500, workers: 3, iterations: 30, vfs: 'opfs-wl'}, {interval: 1000, workers: 5, iterations: 30}, {interval: 500, workers: 5, iterations: 30}, {interval: 250, workers: 3, iterations: 30}, @@ -142,6 +147,7 @@ workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + + '&vfs='+options.vfs + '&interval='+options.interval + '&iterations='+options.iterations + '&opfs-verbose='+options.opfsVerbose diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index a28d80f202..f68bf3516b 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -6,7 +6,8 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ const urlArgs = new URL(globalThis.location.href).searchParams; const options = { workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), - unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/, + vfs: urlArgs.get('vfs') }; const wPost = (type,...payload)=>{ postMessage({type, worker: options.workerName, payload}); @@ -14,7 +15,7 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); if(!sqlite3.opfs){ - stderr("OPFS support not detected. Aborting."); + stderr("This code requires the (private) sqlite3.opfs object. Aborting."); return; } @@ -47,7 +48,16 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.oo1.OpfsDb({ + const Ctors = Object.assign(Object.create(null),{ + opfs: sqlite3.oo1.OpfsDb, + 'opfs-wl': sqlite3.oo1.OpfsWlDb + }); + const ctor = Ctors[options.vfs]; + if( !ctor ){ + stderr("Invalid VFS name:",vfs); + return; + } + db = new ctor({ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c' }); diff --git a/manifest b/manifest index 003a670b2e..fa51c7fc6c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Consolidate\sthe\slast\s200\slines\sof\scommon\sOPFS\sVFS\scode.\s"opfs"\sstill\sworks,\s"opfs-wl"\sregisters\sfine\sbut\sis\sstill\sotherwise\suntested. -D 2026-03-04T19:21:09.278 +C Get\sopfs-wl\splugged\sin\sto\sthe\sconcurrency\stester.\sSomewhat\sironically,\sall\scompeting\sworkers\sfail\swith\slocking\serrors\swhile\sworker\s1\sis\sbusy\srunning\soff\sthe\srails\ssomewhere.\sStashing\sfor\scloser\sinvestigation\slater. +D 2026-03-04T20:33:21.840 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js d8ecb1c7f6b29c2eb501ab8da6f9d9867c1ceb8a42c9c883dd53aed8ddfe106a +F ext/wasm/api/opfs-common-shared.c-pp.js 1218fb9e72f3a208e62a4caafcf1fb0a41b66ca46df27601e5bfb06220822f24 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -596,9 +596,9 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f0a2aa8712211ff9db2ef548ae8b676be3e7c82f61586d03fd8317fbc95bbedd F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js a755ea941f254f89fcd519789097a7401362d9e9dfba19a9bfc972861257c3c5 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 62d41024ad20c388c022f032d3f25838809cb00915ebeb97b598b9f370fba2c3 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 50a955ef393722d498177ad09c9e2d05bbe8dccae4c40c501482a860ca30017d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -646,8 +646,8 @@ F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c9 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e -F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 -F ext/wasm/tests/opfs/concurrency/worker.js 3e29c07bdd11f4ff9ee50e641e39c254d39243bbb83fb20a255755ee92739d12 +F ext/wasm/tests/opfs/concurrency/test.js 2150850176a1ffbe7f6b19f5cf5db9c77290764d6d54430b2796b6512b676a4a +F ext/wasm/tests/opfs/concurrency/worker.js 537e11f86fdc5d5bc64032aaa48cd027a2f8cf4b020aca6c3c012fec85da2eb1 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 57adecbab71795b62b1c2e4570ff504f35681e81dd8c94f78ad8e05ef39d36fd -R 18d89b2ee7fb09440d0eac4e9c8dc540 +P 5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88 +R 0cf8357d9b35fb1b75ea83762f01c38b U stephan -Z d66223f806eab9c2d766ad2e9d73c95d +Z eba11c8254e99ee26ae9feba85ef81c4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c99fbd65b..15d2b8cd82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88 +3b27310aa29ea84f459974981a600301abac5c705029a289d3872ecacf231da3 From 671a6f83f455991801efa25a030d375e23f7023f Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 21:16:36 +0000 Subject: [PATCH 13/32] Add some debugging output to opfs and track down the breakage to the initial WebLock request, which is never reaching its callback. Still broken, but this is progress. FossilOrigin-Name: 62fc8b35aeec75f5648b3daa24162c638999d447aa874bdfcbac1431c5c97b6f --- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 7 ++- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 8 +++- ext/wasm/tests/opfs/concurrency/test.js | 8 ++-- ext/wasm/tests/opfs/concurrency/worker.js | 46 ++++++++----------- manifest | 18 ++++---- manifest.uuid | 2 +- 6 files changed, 45 insertions(+), 44 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index c9bd620fa5..f682c4ce42 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -620,11 +620,12 @@ const installAsyncProxy = function(){ const view = state.sabOPView; const slock = state.lock; const lockType = Atomics.load(view, slock.type); - - await navigator.locks.request('sqlite3-vfs-opfs:'+args[0], { + warn("handleLockRequest()", args, lockType); + navigator.locks.request('sqlite3-vfs-opfs:'+args[0], { mode: (lockType===state.sq3Codes.SQLITE_LOCK_EXCLUSIVE) ? 'exclusive' : 'shared' }, async (wl)=>{ + warn("handleLockRequest() starting lock", args, lockType); /** A. Tell the C-side we have the browser lock. We use the same handshake slot, but a specific 'Granted' value. @@ -642,6 +643,7 @@ const installAsyncProxy = function(){ Atomics.store(view, slock.atomicsHandshake, 0); Atomics.notify(view, lock.atomicsHandshake); }); + warn("handleLockRequest() ending", args, lockType); }; const waitLoop = async function f(){ @@ -679,6 +681,7 @@ const installAsyncProxy = function(){ continue; } const opId = Atomics.exchange(state.sabOPView, opIds.whichOp, 0); + warn("opId =",opId, opIds); if( opId===opIds.lockControl ){ handleLockRequest(); continue; diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 4b2094ee81..6b18a38691 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -60,13 +60,14 @@ const installOpfsWlVfs = async function callee(options){ options = opfsUtil.initOptions(options, callee); if( !options ) return sqlite3; const capi = sqlite3.capi, - debug = sqlite3.config.debug, + debug = (...args)=>sqlite3.config.warn("opfs-wl:",...args), state = opfsUtil.createVfsState('opfs-wl', options), opfsVfs = state.vfs, metrics = opfsVfs.metrics.counters, mTimeStart = opfsVfs.mTimeStart, mTimeEnd = opfsVfs.mTimeEnd, __openFiles = opfsVfs.__openFiles; + debug("state",JSON.stringify(state,false,' ')); /* At this point, createVfsState() has populated state and opfsVfs with any code common to both the "opfs" and "opfs-wl" VFSes. Now comes the VFS-dependent work... */ @@ -75,6 +76,7 @@ const installOpfsWlVfs = async function callee(options){ mTimeStart('xLock'); ++metrics.xLock.count; const f = __openFiles[pFile]; + debug("xLock()",f,lockType); let rc = 0; /* See notes in sqlite3-vfs-opfs.c-pp.js. */ if( f.lockType ) { @@ -89,10 +91,13 @@ const installOpfsWlVfs = async function callee(options){ Atomics.store(view, state.lock.type, lockType); Atomics.store(view, state.opIds.whichOp, state.opIds.lockControl); Atomics.notify(state.sabOPView, state.opIds.whichOp) + debug("xLock waiting..."); while('not-equal'!==Atomics.wait(view, state.lock.atomicsHandshake, 0)){ /* Loop is a workaround for environment-specific quirks. See notes in similar loops. */ + debug("xLock still waiting..."); } + debug("xLock done waiting"); f.lockType = lockType; }catch(e){ sqlite3.config.error("xLock(",arguments,") failed", e, f); @@ -103,6 +108,7 @@ const installOpfsWlVfs = async function callee(options){ return rc; }, xUnlock: function(pFile,lockType){ + debug("xUnlock()",f,lockType); mTimeStart('xUnlock'); ++metrics.xUnlock.count; const f = __openFiles[pFile]; diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index c2932290b7..9208ff47fc 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -120,15 +120,15 @@ const optArgs = function(obj){ const li = []; for(const k of [ - 'interval','iterations','unlock-asap', - 'verbose','vfs','workers', + 'interval', 'iterations', 'unlock-asap', + 'verbose', 'vfs', 'workers' ]){ if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]); } return li.join('&'); }; for(const opt of [ - {interval: 500, workers: 3, iterations: 30, vfs: 'opfs-wl'}, + {interval: 500, workers: 2, iterations: 30, vfs: 'opfs-wl', verbose: 2}, {interval: 1000, workers: 5, iterations: 30}, {interval: 500, workers: 5, iterations: 30}, {interval: 250, workers: 3, iterations: 30}, @@ -154,7 +154,7 @@ + '&opfs-unlock-asap='+options.unlockAsap ); for(let i = 0; i < options.workerCount; ++i){ - stdout("Launching worker..."); + stdout("Launching worker...", i, ); workers.push(new Worker( workers.uri+'&workerId='+(i+1)+( (i || options.noUnlink) ? '' : '&unlink-db' diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index f68bf3516b..b4cb5a30a4 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -36,8 +36,7 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ count: 0 }); const finish = ()=>{ - if(db){ - if(!db.pointer) return; + if(db?.pointer){ db.close(); } if(interval.error){ @@ -71,7 +70,7 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ const maxIterations = urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; - stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); + stdout("Starting",maxIterations,"interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); ++interval.count; @@ -86,39 +85,32 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ }catch(e){ interval.error = e; } + stdout("doWork()",prefix,"error ",interval.error); }; - if(1){/*use setInterval()*/ - setTimeout(async function timer(){ - await doWork(); - if(interval.error || maxIterations === interval.count){ - finish(); - }else{ - setTimeout(timer, interval.delay); - } - }, interval.delay); - }else{ - /*This approach provides no concurrency whatsoever: each worker - is run to completion before any others can work.*/ - let i; - for(i = 0; i < maxIterations; ++i){ - await doWork(); - if(interval.error) break; - await wait(interval.ms); + setTimeout(async function timer(){ + await doWork(); + if(interval.error || maxIterations === interval.count){ + finish(); + }else{ + setTimeout(timer, interval.delay); } - finish(); - } + }, interval.delay); }/*run()*/; globalThis.onmessage = function({data}){ switch(data.type){ - case 'run': run().catch((e)=>{ + case 'run': + run().catch((e)=>{ if(!interval.error) interval.error = e; + }).catch(e=>{ + /* Don't do this in finally() - it fires too soon. */ finish(); + throw e; }); - break; - default: - stderr("Unhandled message type '"+data.type+"'."); - break; + break; + default: + stderr("Unhandled message type '"+data.type+"'."); + break; } }; }); diff --git a/manifest b/manifest index fa51c7fc6c..68ce7cc015 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sopfs-wl\splugged\sin\sto\sthe\sconcurrency\stester.\sSomewhat\sironically,\sall\scompeting\sworkers\sfail\swith\slocking\serrors\swhile\sworker\s1\sis\sbusy\srunning\soff\sthe\srails\ssomewhere.\sStashing\sfor\scloser\sinvestigation\slater. -D 2026-03-04T20:33:21.840 +C Add\ssome\sdebugging\soutput\sto\sopfs\sand\strack\sdown\sthe\sbreakage\sto\sthe\sinitial\sWebLock\srequest,\swhich\sis\snever\sreaching\sits\scallback.\sStill\sbroken,\sbut\sthis\sis\sprogress. +D 2026-03-04T21:16:36.068 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -594,11 +594,11 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f0a2aa8712211ff9db2ef548ae8b676be3e7c82f61586d03fd8317fbc95bbedd +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 792a18708a6207d9bf274b04dcddd06fd805e93d1b7f716230df3c745f2800ee F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 62d41024ad20c388c022f032d3f25838809cb00915ebeb97b598b9f370fba2c3 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 1e6e087a89e9cd202d457c84dae29fa4808612ff9f37210c383eb6bbab3dfd34 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 50a955ef393722d498177ad09c9e2d05bbe8dccae4c40c501482a860ca30017d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -646,8 +646,8 @@ F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c9 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e -F ext/wasm/tests/opfs/concurrency/test.js 2150850176a1ffbe7f6b19f5cf5db9c77290764d6d54430b2796b6512b676a4a -F ext/wasm/tests/opfs/concurrency/worker.js 537e11f86fdc5d5bc64032aaa48cd027a2f8cf4b020aca6c3c012fec85da2eb1 +F ext/wasm/tests/opfs/concurrency/test.js b80e17488a8e4d5500deaf68f73a7298d7d351f74bb952691a4e7b6c02a5643b +F ext/wasm/tests/opfs/concurrency/worker.js de0be3be41da20e6639cdea10467fb5d9cdf540e711b403728a08d99bbb82ee6 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88 -R 0cf8357d9b35fb1b75ea83762f01c38b +P 3b27310aa29ea84f459974981a600301abac5c705029a289d3872ecacf231da3 +R 5c13d459ac5b107e6cc5943524da9a40 U stephan -Z eba11c8254e99ee26ae9feba85ef81c4 +Z 9fafbc035e4ee3a2d442f9b59c8a5807 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 15d2b8cd82..6c88a8044b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b27310aa29ea84f459974981a600301abac5c705029a289d3872ecacf231da3 +62fc8b35aeec75f5648b3daa24162c638999d447aa874bdfcbac1431c5c97b6f From cef3ff74cd89cecc1ee88068f1f8cc8fbce9f51c Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 5 Mar 2026 06:30:57 +0000 Subject: [PATCH 14/32] The opfs-wl lock hang has been traced to starvation between the WebLock and the tight wait-on-VFS-calls Atomics.wait() loop. This can reportedly be resolved with another level of indirection in which the WebLock takes over the wait-on-VFS-calls part until it's unlocked, returning to the global loop when it's done. That exceeds this morning's ambitions but is next to try out. FossilOrigin-Name: 113bd910e12fea17f9f4a0a3baf706f15627c08cfa6b47a960a83eee761ef4dd --- ext/wasm/api/opfs-common-shared.c-pp.js | 5 +- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 69 ++++++++++++++----- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 6 ++ manifest | 16 ++--- manifest.uuid | 2 +- 5 files changed, 72 insertions(+), 26 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 72c366adab..43a1b05cdc 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -585,7 +585,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ of this value is also used for determining how long to wait on lock contention to free up. */ - state.asyncIdleWaitTime = isWebLocker ? 300 : 150; + state.asyncIdleWaitTime = isWebLocker ? 100 : 150; /** Whether the async counterpart should log exceptions to @@ -1262,6 +1262,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } break; } + case 'debug': + warn("debug message from worker:",data); + break; default: { const errMsg = ( "Unexpected message from the OPFS async worker: " + diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index f682c4ce42..8338b96c05 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -615,15 +615,18 @@ const installAsyncProxy = function(){ Starts a new WebLock request. */ const handleLockRequest = async function(){ + const view = state.sabOPView; +//#if not nope const args = state.s11n.deserialize(true) || toss("Expecting a filename argument from the proxy xLock()"); - const view = state.sabOPView; const slock = state.lock; const lockType = Atomics.load(view, slock.type); - warn("handleLockRequest()", args, lockType); - navigator.locks.request('sqlite3-vfs-opfs:'+args[0], { - mode: (lockType===state.sq3Codes.SQLITE_LOCK_EXCLUSIVE) - ? 'exclusive' : 'shared' + warn("handleLockRequest()", args, lockType, JSON.stringify(slock)); + //hangs warn(JSON.stringify((await navigator.locks.query()).held)); + //warn("Navigator locks:", !!navigator.locks); + await navigator.locks.request('sqlite3-vfs-opfs:'+args[0], { + mode: 'exclusive' /*(lockType===state.sq3Codes.SQLITE_LOCK_EXCLUSIVE) + ? 'exclusive' : 'shared'*/ }, async (wl)=>{ warn("handleLockRequest() starting lock", args, lockType); /** @@ -637,24 +640,57 @@ const installAsyncProxy = function(){ Receptionist receives 'unlockControl'. */ while( 1!==Atomics.load(view, slock.atomicsHandshake) ){ - Atomics.wait(view. slock.atomicsHandshake, 2); + Atomics.wait(view, slock.atomicsHandshake, 2); } /* C. Reset the handshake slot. */ Atomics.store(view, slock.atomicsHandshake, 0); Atomics.notify(view, lock.atomicsHandshake); }); warn("handleLockRequest() ending", args, lockType); + //setTimeout(waitLoop, 0); +//#else + warn("handleLockRequest()", ...arguments); + const [filename] = state.s11n.deserialize(true); + const lockType = Atomics.load(view, state.lock.type); + warn("handleLockRequest()", filename, lockType); + // Use 'exclusive' to ensure we aren't getting a weak shared lock + navigator.locks.request(filename, { mode: 'exclusive' }, (lock) => { + warn("handleLockRequest() inside the lock"); + // VIOLENT DEBUGGING: Use globalThis.postMessage to bypass Atomics + globalThis.postMessage({type: 'debug', msg: 'CALLBACK ENTERED'}); + + // 1. Signal C-side: "I have it" + Atomics.store(view, state.lock.atomicsHandshake, 2); + Atomics.notify(view, state.lock.atomicsHandshake); + + // 2. The Guard: This loop must not be async. + // It must stay synchronous to keep the callback alive. + while(Atomics.load(view, state.lock.atomicsHandshake) !== 1){ + Atomics.wait(view, state.lock.atomicsHandshake, 2, 100); + } + + // 3. Departure + Atomics.store(view, state.lock.atomicsHandshake, 0); + globalThis.postMessage({type: 'debug', msg: 'CALLBACK EXITING'}); + return new Promise(()=>{/* never resolve to keep lock held until Departure */}); + }).catch(e => { + globalThis.postMessage({type: 'debug', msg: 'LOCK ERROR: ' + e.message}); + }); +//#endif }; const waitLoop = async function f(){ - const opHandlers = Object.create(null); - for(let k of Object.keys(state.opIds)){ - const vi = vfsAsyncImpls[k]; - if(!vi) continue; - const o = Object.create(null); - opHandlers[state.opIds[k]] = o; - o.key = k; - o.f = vi; + if( !f.inited ){ + f.inited = true; + f.opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + f.opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } } const opIds = state.opIds; while(!flagAsyncShutdown){ @@ -684,9 +720,10 @@ const installAsyncProxy = function(){ warn("opId =",opId, opIds); if( opId===opIds.lockControl ){ handleLockRequest(); - continue; + setTimeout(f, 50); + return; } - const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + const hnd = f.opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with an exception string written by the upcoming diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 6b18a38691..b2d25b85e4 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -92,11 +92,17 @@ const installOpfsWlVfs = async function callee(options){ Atomics.store(view, state.opIds.whichOp, state.opIds.lockControl); Atomics.notify(state.sabOPView, state.opIds.whichOp) debug("xLock waiting..."); +//#if not nope + while( 2 !== Atomics.load(view, state.lock.atomicsHandshake) ){ + Atomics.wait(view, state.lock.atomicsHandshake, 0); + } +//#else while('not-equal'!==Atomics.wait(view, state.lock.atomicsHandshake, 0)){ /* Loop is a workaround for environment-specific quirks. See notes in similar loops. */ debug("xLock still waiting..."); } +//#endif debug("xLock done waiting"); f.lockType = lockType; }catch(e){ diff --git a/manifest b/manifest index 68ce7cc015..a92a7a8886 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\sdebugging\soutput\sto\sopfs\sand\strack\sdown\sthe\sbreakage\sto\sthe\sinitial\sWebLock\srequest,\swhich\sis\snever\sreaching\sits\scallback.\sStill\sbroken,\sbut\sthis\sis\sprogress. -D 2026-03-04T21:16:36.068 +C The\sopfs-wl\slock\shang\shas\sbeen\straced\sto\sstarvation\sbetween\sthe\sWebLock\sand\sthe\stight\swait-on-VFS-calls\sAtomics.wait()\sloop.\sThis\scan\sreportedly\sbe\sresolved\swith\sanother\slevel\sof\sindirection\sin\swhich\sthe\sWebLock\stakes\sover\sthe\swait-on-VFS-calls\spart\suntil\sit's\sunlocked,\sreturning\sto\sthe\sglobal\sloop\swhen\sit's\sdone.\sThat\sexceeds\sthis\smorning's\sambitions\sbut\sis\snext\sto\stry\sout. +D 2026-03-05T06:30:57.097 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 1218fb9e72f3a208e62a4caafcf1fb0a41b66ca46df27601e5bfb06220822f24 +F ext/wasm/api/opfs-common-shared.c-pp.js 404aa8fd676791f0ce71e5c48f5f19e13e7b8e967a9af6a76927d524bbb46158 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -594,11 +594,11 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 792a18708a6207d9bf274b04dcddd06fd805e93d1b7f716230df3c745f2800ee +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 82339eb043af53638b127a4649685da62b6fa33426d2013520eb6aeb114ede46 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 1e6e087a89e9cd202d457c84dae29fa4808612ff9f37210c383eb6bbab3dfd34 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 88add6362468df4ecbbd00878c7a83cd81e555c677103ead51c34bd2be1a3963 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 50a955ef393722d498177ad09c9e2d05bbe8dccae4c40c501482a860ca30017d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 3b27310aa29ea84f459974981a600301abac5c705029a289d3872ecacf231da3 -R 5c13d459ac5b107e6cc5943524da9a40 +P 62fc8b35aeec75f5648b3daa24162c638999d447aa874bdfcbac1431c5c97b6f +R 4e4429a4d6f21ed4af3f33058413d5c8 U stephan -Z 9fafbc035e4ee3a2d442f9b59c8a5807 +Z 5eeca3e1560bb544ee40d011df9232c0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6c88a8044b..c36b9ce9ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62fc8b35aeec75f5648b3daa24162c638999d447aa874bdfcbac1431c5c97b6f +113bd910e12fea17f9f4a0a3baf706f15627c08cfa6b47a960a83eee761ef4dd From 14b5eadefe0e8285f097fa948bfb08ae0dfa9f00 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 5 Mar 2026 11:51:27 +0000 Subject: [PATCH 15/32] Strip the opfs-wl "back to formula", removing the current false starts so that this can be tried again without tripping over any cruft. The current impl is subject, with no obvious way out of it, to starvation in the async proxy. FossilOrigin-Name: d022f1f5e74dedae044801330eb099022498f359f408e69b3574885641b312f5 --- ext/wasm/api/opfs-common-shared.c-pp.js | 36 +++++---- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 79 +------------------ ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 61 ++++---------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 4 +- ext/wasm/tests/opfs/concurrency/test.js | 5 +- ext/wasm/tests/opfs/concurrency/worker.js | 3 +- manifest | 22 +++--- manifest.uuid | 2 +- 8 files changed, 57 insertions(+), 155 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 43a1b05cdc..c709e0d15b 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -444,7 +444,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Set up any references they may need to state returned by the previous step. - - Call opfvs.doTheThing() + - Call opfvs.bindVfs() */ opfsUtil.initOptions = function(options, callee){ const urlParams = new URL(globalThis.location.href).searchParams; @@ -485,10 +485,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.sqlite3_vfs instance. After setting up any local state needed, the caller must - call theVfs.doTheThing(X,Y), where X is an object containing + call theVfs.bindVfs(X,Y), where X is an object containing the sqlite3_io_methods to override and Y is a callback which gets triggered if init succeeds, before the final Promise - decides whether or not to reject. The result of doTheThing() + decides whether or not to reject. The result of bindVfs() must be returned from their main installation function. This object must, when it's passed to the async part, contain @@ -505,7 +505,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.config.log ]; const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + if(state.verbose>level) loggers[level]("OPFS syncer:",...args); }; const log = (...args)=>logImpl(2, ...args), warn = (...args)=>logImpl(1, ...args), @@ -670,7 +670,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ state.opIds.xUnlock = i++; state.opIds.xWrite = i++; state.opIds.mkdir = i++; - state.opIds.lockControl = i++ /* opfs-wl signals the intent to lock here */; /** Internal signals which are used only during development and testing via the dev console. */ state.opIds['opfs-async-metrics'] = i++; @@ -679,13 +678,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ semantics. Though we could hypothetically use the xSleep slot for that, doing so might lead to undesired side effects. */ state.opIds.retry = i++; - - state.lock = util.nu({ - /* Slots for submitting the lock type and receiving its - acknowledgement. Only used by "opfs-wl". */ - type: i++ /* SQLITE_LOCK_xyz value */, - atomicsHandshake: i++ /* 0=pending, 1=release, 2=granted */ - }); state.sabOP = new SharedArrayBuffer( i * 4/* ==sizeof int32, noting that Atomics.wait() and friends can only function on Int32Array views of an SAB. */); @@ -1080,7 +1072,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ where opfsVfs is the sqlite3_vfs object which was set up by opfsUtil.createVfsState(). */ - opfsVfs.doTheThing = function(ioMethods, callback){ + opfsVfs.bindVfs = function(ioMethods, callback){ Object.assign(opfsVfs.ioSyncWrappers, ioMethods); const thePromise = new Promise(function(promiseResolve_, promiseReject_){ let promiseWasRejected = undefined; @@ -1101,13 +1093,21 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ options.proxyUri += '?vfs='+vfsName; const W = opfsVfs.worker = //#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); + (()=>{ + /* _Sigh_... */ + switch(vfsName){ + case 'opfs': + return new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); + case 'opfs-wl': + return new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); + } + })(); //#elif target:es6-module new Worker(new URL(options.proxyUri, import.meta.url)); //#else new Worker(options.proxyUri); //#endif - setTimeout(()=>{ + let zombieTimer = setTimeout(()=>{ /* At attempt to work around a browser-specific quirk in which the Worker load is failing in such a way that we neither resolve nor reject it. This workaround gives that resolve/reject @@ -1211,7 +1211,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*sanityCheck()*/; W.onmessage = function({data}){ - //log("Worker.onmessage:",data); + //sqlite3.config.warn(vfsName,"Worker.onmessage:",data); switch(data.type){ case 'opfs-unavailable': /* Async proxy has determined that OPFS is unavailable. There's @@ -1232,6 +1232,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(true===promiseWasRejected){ break /* promise was already rejected via timer */; } + clearTimeout(zombieTimer); + zombieTimer = null; try { sqlite3.vfs.installVfs({ io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, @@ -1278,7 +1280,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*W.onmessage()*/; })/*thePromise*/; return thePromise; - }/*doTheThing()*/; + }/*bindVfs()*/; return state; }/*createVfsState()*/; diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 8338b96c05..0db9a99141 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -326,7 +326,7 @@ const installAsyncProxy = function(){ } log("Got",opName+"() sync handle for",fh.filenameAbs, 'in',performance.now() - t,'ms'); - if(!isWebLocker && !fh.xLock){ + if(!fh.xLock){ __implicitLocks.add(fh.fid); log("Acquired implicit lock for",opName+"()",fh.fid,fh.filenameAbs); } @@ -611,74 +611,6 @@ const installAsyncProxy = function(){ } }/*vfsAsyncImpls*/; - /** - Starts a new WebLock request. - */ - const handleLockRequest = async function(){ - const view = state.sabOPView; -//#if not nope - const args = state.s11n.deserialize(true) - || toss("Expecting a filename argument from the proxy xLock()"); - const slock = state.lock; - const lockType = Atomics.load(view, slock.type); - warn("handleLockRequest()", args, lockType, JSON.stringify(slock)); - //hangs warn(JSON.stringify((await navigator.locks.query()).held)); - //warn("Navigator locks:", !!navigator.locks); - await navigator.locks.request('sqlite3-vfs-opfs:'+args[0], { - mode: 'exclusive' /*(lockType===state.sq3Codes.SQLITE_LOCK_EXCLUSIVE) - ? 'exclusive' : 'shared'*/ - }, async (wl)=>{ - warn("handleLockRequest() starting lock", args, lockType); - /** - A. Tell the C-side we have the browser lock. We use the same - handshake slot, but a specific 'Granted' value. - */ - Atomics.store(view, slock.atomicsHandshake, 2); - Atomics.notify(view, slock.atomicsHandshake); - /** - B. Sit here and keep the lock room occupied until the - Receptionist receives 'unlockControl'. - */ - while( 1!==Atomics.load(view, slock.atomicsHandshake) ){ - Atomics.wait(view, slock.atomicsHandshake, 2); - } - /* C. Reset the handshake slot. */ - Atomics.store(view, slock.atomicsHandshake, 0); - Atomics.notify(view, lock.atomicsHandshake); - }); - warn("handleLockRequest() ending", args, lockType); - //setTimeout(waitLoop, 0); -//#else - warn("handleLockRequest()", ...arguments); - const [filename] = state.s11n.deserialize(true); - const lockType = Atomics.load(view, state.lock.type); - warn("handleLockRequest()", filename, lockType); - // Use 'exclusive' to ensure we aren't getting a weak shared lock - navigator.locks.request(filename, { mode: 'exclusive' }, (lock) => { - warn("handleLockRequest() inside the lock"); - // VIOLENT DEBUGGING: Use globalThis.postMessage to bypass Atomics - globalThis.postMessage({type: 'debug', msg: 'CALLBACK ENTERED'}); - - // 1. Signal C-side: "I have it" - Atomics.store(view, state.lock.atomicsHandshake, 2); - Atomics.notify(view, state.lock.atomicsHandshake); - - // 2. The Guard: This loop must not be async. - // It must stay synchronous to keep the callback alive. - while(Atomics.load(view, state.lock.atomicsHandshake) !== 1){ - Atomics.wait(view, state.lock.atomicsHandshake, 2, 100); - } - - // 3. Departure - Atomics.store(view, state.lock.atomicsHandshake, 0); - globalThis.postMessage({type: 'debug', msg: 'CALLBACK EXITING'}); - return new Promise(()=>{/* never resolve to keep lock held until Departure */}); - }).catch(e => { - globalThis.postMessage({type: 'debug', msg: 'LOCK ERROR: ' + e.message}); - }); -//#endif - }; - const waitLoop = async function f(){ if( !f.inited ){ f.inited = true; @@ -717,12 +649,6 @@ const installAsyncProxy = function(){ continue; } const opId = Atomics.exchange(state.sabOPView, opIds.whichOp, 0); - warn("opId =",opId, opIds); - if( opId===opIds.lockControl ){ - handleLockRequest(); - setTimeout(f, 50); - return; - } const hnd = f.opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with @@ -741,6 +667,7 @@ const installAsyncProxy = function(){ navigator.storage.getDirectory().then(function(d){ state.rootDir = d; globalThis.onmessage = function({data}){ + warn(globalThis.location.href,"onmessage()",data); switch(data.type){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ @@ -767,7 +694,7 @@ const installAsyncProxy = function(){ flagAsyncShutdown = false; waitLoop(); } - break; + break; } }; wPost('opfs-async-loaded'); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index b2d25b85e4..8119b32469 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -59,6 +59,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const installOpfsWlVfs = async function callee(options){ options = opfsUtil.initOptions(options, callee); if( !options ) return sqlite3; + options.verbose = 2; const capi = sqlite3.capi, debug = (...args)=>sqlite3.config.warn("opfs-wl:",...args), state = opfsUtil.createVfsState('opfs-wl', options), @@ -67,85 +68,55 @@ const installOpfsWlVfs = async function callee(options){ mTimeStart = opfsVfs.mTimeStart, mTimeEnd = opfsVfs.mTimeEnd, __openFiles = opfsVfs.__openFiles; - debug("state",JSON.stringify(state,false,' ')); + //debug("state",JSON.stringify(state,false,' ')); /* At this point, createVfsState() has populated state and opfsVfs with any code common to both the "opfs" and "opfs-wl" VFSes. Now comes the VFS-dependent work... */ - return opfsVfs.doTheThing(util.nu({ - xLock: function(pFile, lockType){ + return opfsVfs.bindVfs(util.nu({ + xLock: function(pFile,lockType){ mTimeStart('xLock'); ++metrics.xLock.count; const f = __openFiles[pFile]; - debug("xLock()",f,lockType); let rc = 0; - /* See notes in sqlite3-vfs-opfs.c-pp.js. */ + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ if( f.lockType ) { f.lockType = lockType; }else{ - try{ - const view = state.sabOPView; - /* We need to pass pFile's name to the async proxy so that - it can create the WebLock name. */ - state.s11n.serialize(f.filename) - Atomics.store(view, state.lock.atomicsHandshake, 0); - Atomics.store(view, state.lock.type, lockType); - Atomics.store(view, state.opIds.whichOp, state.opIds.lockControl); - Atomics.notify(state.sabOPView, state.opIds.whichOp) - debug("xLock waiting..."); -//#if not nope - while( 2 !== Atomics.load(view, state.lock.atomicsHandshake) ){ - Atomics.wait(view, state.lock.atomicsHandshake, 0); - } -//#else - while('not-equal'!==Atomics.wait(view, state.lock.atomicsHandshake, 0)){ - /* Loop is a workaround for environment-specific quirks. See - notes in similar loops. */ - debug("xLock still waiting..."); - } -//#endif - debug("xLock done waiting"); - f.lockType = lockType; - }catch(e){ - sqlite3.config.error("xLock(",arguments,") failed", e, f); - rc = capi.SQLITE_IOERR_LOCK; - } + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; } mTimeEnd(); return rc; }, xUnlock: function(pFile,lockType){ - debug("xUnlock()",f,lockType); mTimeStart('xUnlock'); ++metrics.xUnlock.count; const f = __openFiles[pFile]; let rc = 0; - if( lockType < f.lockType ){ - try{ - const view = state.sabOPView; - Atomics.store(view, state.lock.atomicsHandshake, 1); - Atomics.notify(view, state.lock.atomicsHandshake); - Atomics.wait(view, state.lock.atomicsHandshake, 1); - }catch(e){ - sqlite3.config.error("xUnlock(",pFile,lockType,") failed",e, f); - rc = capi.SQLITE_IOERR_LOCK; - } + if( capi.SQLITE_LOCK_NONE === lockType + && f.lockType ){ + rc = opRun('xUnlock', pFile, lockType); } if( 0===rc ) f.lockType = lockType; mTimeEnd(); return rc; } }), function(sqlite3, vfs){ + //debug("registered VFS"); if(sqlite3.oo1){ const OpfsWlDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; + opt.vfs = vfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsWlDb = OpfsWlDb; OpfsWlDb.importDb = opfsUtil.importDb; }/*extend sqlite3.oo1*/ - })/*doTheThing()*/; + })/*bindVfs()*/; }/*installOpfsWlVfs()*/; installOpfsWlVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 6fe317ec8d..89b52c836d 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -95,7 +95,7 @@ const installOpfsVfs = async function callee(options){ /* At this point, createVfsState() has populated state and opfsVfs with any code common to both the "opfs" and "opfs-wl" VFSes. Now comes the VFS-dependent work... */ - return opfsVfs.doTheThing(util.nu({ + return opfsVfs.bindVfs(util.nu({ xLock: function(pFile,lockType){ mTimeStart('xLock'); ++metrics.xLock.count; @@ -147,7 +147,7 @@ const installOpfsVfs = async function callee(options){ } ); }/*extend sqlite3.oo1*/ - })/*doTheThing()*/; + })/*bindVfs()*/; }/*installOpfsVfs()*/; installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 9208ff47fc..18c01a5b7e 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -121,14 +121,15 @@ const li = []; for(const k of [ 'interval', 'iterations', 'unlock-asap', - 'verbose', 'vfs', 'workers' + 'opfsVerbose', 'vfs', 'workers' ]){ if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]); } return li.join('&'); }; for(const opt of [ - {interval: 500, workers: 2, iterations: 30, vfs: 'opfs-wl', verbose: 2}, + {interval: 500, workers: 1, iterations: 5, vfs: 'opfs-wl', opfsVerbose: 2}, + {interval: 500, workers: 2, iterations: 5, vfs: 'opfs-wl', opfsVerbose: 2}, {interval: 1000, workers: 5, iterations: 30}, {interval: 500, workers: 5, iterations: 30}, {interval: 250, workers: 3, iterations: 30}, diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index b4cb5a30a4..77e320e9af 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -1,6 +1,7 @@ importScripts( (new URL(globalThis.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' ); +//const sqlite3InitModule = (await import("../../../jswasm/sqlite3.mjs", )).default; globalThis.sqlite3InitModule.__isUnderTest = true; globalThis.sqlite3InitModule().then(async function(sqlite3){ const urlArgs = new URL(globalThis.location.href).searchParams; @@ -85,7 +86,7 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ }catch(e){ interval.error = e; } - stdout("doWork()",prefix,"error ",interval.error); + //stdout("doWork()",prefix,"error ",interval.error); }; setTimeout(async function timer(){ await doWork(); diff --git a/manifest b/manifest index a92a7a8886..c3f7604220 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sopfs-wl\slock\shang\shas\sbeen\straced\sto\sstarvation\sbetween\sthe\sWebLock\sand\sthe\stight\swait-on-VFS-calls\sAtomics.wait()\sloop.\sThis\scan\sreportedly\sbe\sresolved\swith\sanother\slevel\sof\sindirection\sin\swhich\sthe\sWebLock\stakes\sover\sthe\swait-on-VFS-calls\spart\suntil\sit's\sunlocked,\sreturning\sto\sthe\sglobal\sloop\swhen\sit's\sdone.\sThat\sexceeds\sthis\smorning's\sambitions\sbut\sis\snext\sto\stry\sout. -D 2026-03-05T06:30:57.097 +C Strip\sthe\sopfs-wl\s"back\sto\sformula",\sremoving\sthe\scurrent\sfalse\sstarts\sso\sthat\sthis\scan\sbe\stried\sagain\swithout\stripping\sover\sany\scruft.\sThe\scurrent\simpl\sis\ssubject,\swith\sno\sobvious\sway\sout\sof\sit,\sto\sstarvation\sin\sthe\sasync\sproxy. +D 2026-03-05T11:51:27.592 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 404aa8fd676791f0ce71e5c48f5f19e13e7b8e967a9af6a76927d524bbb46158 +F ext/wasm/api/opfs-common-shared.c-pp.js d54054b9b39ceb088692153f400d2d9a147e4a45d039c8a027575bdc7a02ac68 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -594,12 +594,12 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 82339eb043af53638b127a4649685da62b6fa33426d2013520eb6aeb114ede46 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 81d24de69b43d3c0940dbaa9bc82de193b720107e5a2ddfd31bc50640dd4903d F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 88add6362468df4ecbbd00878c7a83cd81e555c677103ead51c34bd2be1a3963 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 50a955ef393722d498177ad09c9e2d05bbe8dccae4c40c501482a860ca30017d +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 8233c5f9021b0213134e2adbaf6036b8f1dffd4747083a4087c1c19ae107f962 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js f3b7296480984bcc6050fe9724a8b215c405977dd69daea7145ece25751e4b33 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -646,8 +646,8 @@ F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c9 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e -F ext/wasm/tests/opfs/concurrency/test.js b80e17488a8e4d5500deaf68f73a7298d7d351f74bb952691a4e7b6c02a5643b -F ext/wasm/tests/opfs/concurrency/worker.js de0be3be41da20e6639cdea10467fb5d9cdf540e711b403728a08d99bbb82ee6 +F ext/wasm/tests/opfs/concurrency/test.js 74f4ef9a827d081e6bb0ffb1d124bb54015dab8f7ae47abd5b5f26d71633331a +F ext/wasm/tests/opfs/concurrency/worker.js 3425e6dad755a1c69a6efc63a47a3ade4e7f0a9a138994ba37f996571fb46288 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 62fc8b35aeec75f5648b3daa24162c638999d447aa874bdfcbac1431c5c97b6f -R 4e4429a4d6f21ed4af3f33058413d5c8 +P 113bd910e12fea17f9f4a0a3baf706f15627c08cfa6b47a960a83eee761ef4dd +R 4ea42d5818e8a1a0aa1e2fe935a1308b U stephan -Z 5eeca3e1560bb544ee40d011df9232c0 +Z 71a80b89b42b2027ae01cf15e67091dc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c36b9ce9ac..e12e126346 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -113bd910e12fea17f9f4a0a3baf706f15627c08cfa6b47a960a83eee761ef4dd +d022f1f5e74dedae044801330eb099022498f359f408e69b3574885641b312f5 From 831e2332027c0b70ba801bef4148b6b2ca79f8a1 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 09:06:24 +0000 Subject: [PATCH 16/32] Reimplement the OPFS async proxy's wait-forever loop to be more async-friendly for upcoming changes. FossilOrigin-Name: fd775772fbb50b04c8c37977b90a708784eefed403e3668196c9d95559d3a5e1 --- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 30 +++++++------------ manifest | 12 ++++---- manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 0db9a99141..16e2d2aa1c 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -625,31 +625,21 @@ const installAsyncProxy = function(){ } } const opIds = state.opIds; + const opView = state.sabOPView; + const slotWhichOp = opIds.whichOp; + const idleWaitTime = state.asyncIdleWaitTime; while(!flagAsyncShutdown){ try { - if('not-equal'!==Atomics.wait( - state.sabOPView, opIds.whichOp, 0, state.asyncIdleWaitTime - )){ - /* Maintenance note: we compare against 'not-equal' because - - https://github.com/tomayac/sqlite-wasm/issues/12 - - is reporting that this occasionally, under high loads, - returns 'ok', which leads to the whichOp being 0 (which - isn't a valid operation ID and leads to an exception, - along with a corresponding ugly console log - message). Unfortunately, the conditions for that cannot - be reliably reproduced. The only place in our code which - writes a 0 to the state.opIds.whichOp SharedArrayBuffer - index is a few lines down from here, and that instance - is required in order for clear communication between - the sync half of this proxy and this half. - */ + const opId = Atomics.load(opView, slotWhichOp); + if( 0===opId ){ + const rv = Atomics.waitAsync(opView, slotWhichOp, 0, + idleWaitTime); + if( rv.async ) await rv.value; await releaseImplicitLocks(); continue; } - const opId = Atomics.exchange(state.sabOPView, opIds.whichOp, 0); const hnd = f.opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + Atomics.store(opView, slotWhichOp, 0); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with an exception string written by the upcoming @@ -659,7 +649,7 @@ const installAsyncProxy = function(){ if(hnd.f) await hnd.f(...args); else error("Missing callback for opId",opId); }catch(e){ - error('in waitLoop():',e); + error('in waitLoop():', e); } } }; diff --git a/manifest b/manifest index c3f7604220..db41159ec8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Strip\sthe\sopfs-wl\s"back\sto\sformula",\sremoving\sthe\scurrent\sfalse\sstarts\sso\sthat\sthis\scan\sbe\stried\sagain\swithout\stripping\sover\sany\scruft.\sThe\scurrent\simpl\sis\ssubject,\swith\sno\sobvious\sway\sout\sof\sit,\sto\sstarvation\sin\sthe\sasync\sproxy. -D 2026-03-05T11:51:27.592 +C Reimplement\sthe\sOPFS\sasync\sproxy's\swait-forever\sloop\sto\sbe\smore\sasync-friendly\sfor\supcoming\schanges. +D 2026-03-06T09:06:24.644 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -594,7 +594,7 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 81d24de69b43d3c0940dbaa9bc82de193b720107e5a2ddfd31bc50640dd4903d +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js df2c3be6bcdbfab5c7426c14edf768d97de81af0c9d22282c618ab7c4bb3f48f F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 113bd910e12fea17f9f4a0a3baf706f15627c08cfa6b47a960a83eee761ef4dd -R 4ea42d5818e8a1a0aa1e2fe935a1308b +P d022f1f5e74dedae044801330eb099022498f359f408e69b3574885641b312f5 +R 29e425791ac08c7a9478759e76e59fc3 U stephan -Z 71a80b89b42b2027ae01cf15e67091dc +Z fa163f510f2e4230b70e49c7b3e367ed # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e12e126346..22849172c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d022f1f5e74dedae044801330eb099022498f359f408e69b3574885641b312f5 +fd775772fbb50b04c8c37977b90a708784eefed403e3668196c9d95559d3a5e1 From 891f5a176b66dc2428f44b1f4c9fd1ebd0949c47 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 09:11:47 +0000 Subject: [PATCH 17/32] A slight simplification/optimization in the OPFS proxy's waitLoop(). FossilOrigin-Name: 521bb140b7ed237c118ac9094732d06907229a6ff385502e850c679bd623fd58 --- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 5 ++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 16e2d2aa1c..0fc23540e1 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -638,16 +638,15 @@ const installAsyncProxy = function(){ await releaseImplicitLocks(); continue; } - const hnd = f.opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); Atomics.store(opView, slotWhichOp, 0); + const hnd = f.opHandlers[opId]?.f ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with an exception string written by the upcoming operation */ ) || []; //warn("waitLoop() whichOp =",opId, hnd, args); - if(hnd.f) await hnd.f(...args); - else error("Missing callback for opId",opId); + await hnd(...args); }catch(e){ error('in waitLoop():', e); } diff --git a/manifest b/manifest index db41159ec8..654ff7de0b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reimplement\sthe\sOPFS\sasync\sproxy's\swait-forever\sloop\sto\sbe\smore\sasync-friendly\sfor\supcoming\schanges. -D 2026-03-06T09:06:24.644 +C A\sslight\ssimplification/optimization\sin\sthe\sOPFS\sproxy's\swaitLoop(). +D 2026-03-06T09:11:47.789 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -594,7 +594,7 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js df2c3be6bcdbfab5c7426c14edf768d97de81af0c9d22282c618ab7c4bb3f48f +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 5f95c210183d203c7bc5be4a8139cc7fa2e50810c306bb2651648c32ab419fcf F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P d022f1f5e74dedae044801330eb099022498f359f408e69b3574885641b312f5 -R 29e425791ac08c7a9478759e76e59fc3 +P fd775772fbb50b04c8c37977b90a708784eefed403e3668196c9d95559d3a5e1 +R f7984b1956caa0bacce9f6eadf906e85 U stephan -Z fa163f510f2e4230b70e49c7b3e367ed +Z c9e43a6b144c73d106bca120c0ac01f1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 22849172c8..82686c7137 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd775772fbb50b04c8c37977b90a708784eefed403e3668196c9d95559d3a5e1 +521bb140b7ed237c118ac9094732d06907229a6ff385502e850c679bd623fd58 From 544ef18bbdaa9ac6dcae07182640dd7893b52e1c Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 11:49:36 +0000 Subject: [PATCH 18/32] Remove an extraneous OPFS metrics increment. FossilOrigin-Name: bf3548a37712e848c7a9cadfdc1669a2be572ea0a0c28d84c157ab30f8c30c44 --- ext/wasm/api/opfs-common-shared.c-pp.js | 1 - manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index c709e0d15b..6540939b44 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -902,7 +902,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }, xSync: function(pFile,flags){ mTimeStart('xSync'); - ++metrics.xSync.count; const rc = opRun('xSync', pFile, flags); mTimeEnd(); return rc; diff --git a/manifest b/manifest index 654ff7de0b..94e2e47dac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sslight\ssimplification/optimization\sin\sthe\sOPFS\sproxy's\swaitLoop(). -D 2026-03-06T09:11:47.789 +C Remove\san\sextraneous\sOPFS\smetrics\sincrement. +D 2026-03-06T11:49:36.430 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js d54054b9b39ceb088692153f400d2d9a147e4a45d039c8a027575bdc7a02ac68 +F ext/wasm/api/opfs-common-shared.c-pp.js ecc3c69dc8a3b676f6999ef1b2ac4be825b7952ce5acbe86941f4f2f38607cb7 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P fd775772fbb50b04c8c37977b90a708784eefed403e3668196c9d95559d3a5e1 -R f7984b1956caa0bacce9f6eadf906e85 +P 521bb140b7ed237c118ac9094732d06907229a6ff385502e850c679bd623fd58 +R 334abfd624b27a29d02bb7116f94ced4 U stephan -Z c9e43a6b144c73d106bca120c0ac01f1 +Z 944b558939ff5f7ba7e037ff2a332b5b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 82686c7137..48801140c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -521bb140b7ed237c118ac9094732d06907229a6ff385502e850c679bd623fd58 +bf3548a37712e848c7a9cadfdc1669a2be572ea0a0c28d84c157ab30f8c30c44 From 7b9cd03c0ad4b7bdfc3f07434345a5ad7496d47a Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 16:04:21 +0000 Subject: [PATCH 19/32] This Web Lock impl can reliably run a single OPFS connection but rather unreliably 'loses' workers with higher counts, presumably due to deadlock or deadly embrace (how _all_ of them can deadlock at once is unclear, but clearly a bug). FossilOrigin-Name: d4e8583e2e80665adfe4e814adb6c219936af1dcac4105795045cb1a7b1e4864 --- ext/wasm/api/opfs-common-shared.c-pp.js | 38 ++- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 259 +++++++++++++++--- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 50 ++-- ext/wasm/tester1.c-pp.js | 24 +- ext/wasm/tests/opfs/concurrency/worker.js | 26 +- manifest | 20 +- manifest.uuid | 2 +- 7 files changed, 299 insertions(+), 120 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 6540939b44..2020184173 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -446,7 +446,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Call opfvs.bindVfs() */ - opfsUtil.initOptions = function(options, callee){ + opfsUtil.initOptions = function callee(options, callee){ const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('opfs-disable')){ //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); @@ -458,19 +458,22 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return; } options = util.nu(options); - if(undefined===options.verbose){ - options.verbose = urlParams.has('opfs-verbose') - ? (+urlParams.get('opfs-verbose') || 2) : 1; - } - if(undefined===options.sanityChecks){ - options.sanityChecks = urlParams.has('opfs-sanity-check'); - } - if(undefined===options.proxyUri){ - options.proxyUri = callee.defaultProxyUri; - } + options.verbose ??= urlParams.has('opfs-verbose') + ? (+urlParams.get('opfs-verbose') || 2) : 1; + options.sanityChecks ??= urlParams.has('opfs-sanity-check'); + options.proxyUri ??= callee.defaultProxyUri; if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } + if( false ){ + /* This ends up with the same values for all Worker instances. */ + callee.counter ??= 0; + ++callee.counter; + options.workerId ??= urlParams.get('opfs-async-proxy-id') ?? callee.counter; + }else{ + options.workerId ??= (Math.random() * 10000000) | 0; + } + //sqlite3.config.warn("opfsUtil options =",JSON.stringify(options), 'urlParams =', urlParams); return options; }; @@ -585,7 +588,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ of this value is also used for determining how long to wait on lock contention to free up. */ - state.asyncIdleWaitTime = isWebLocker ? 100 : 150; + state.asyncIdleWaitTime = isWebLocker ? 150 : 150; /** Whether the async counterpart should log exceptions to @@ -1089,7 +1092,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ promiseWasRejected = false; return promiseResolve_(sqlite3); }; - options.proxyUri += '?vfs='+vfsName; + let proxyUri = options.proxyUri +( + (options.proxyUri.indexOf('?')<0) ? '?' : '&' + )+'vfs='+vfsName; + if( options.workerId ){ + proxyUri += '&opfs-async-proxy-id='+encodeURIComponent(options.workerId); + } const W = opfsVfs.worker = //#if target:es6-bundler-friendly (()=>{ @@ -1102,9 +1110,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } })(); //#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); + new Worker(new URL(proxyUri, import.meta.url)); //#else - new Worker(options.proxyUri); + new Worker(proxyUri); //#endif let zombieTimer = setTimeout(()=>{ /* At attempt to work around a browser-specific quirk in which diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 0fc23540e1..5c6a11dfd2 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -54,7 +54,9 @@ const urlParams = new URL(globalThis.location.href).searchParams; if( !urlParams.has('vfs') ){ throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); } -const isWebLocker = 'opfs-wl'===urlParams.get('vfs'); +const workerId = urlParams.get('opfs-async-proxy-id') + ?? 'unnamed'; +const isWebLocker = true; //'opfs-wl'===urlParams.get('vfs'); const wPost = (type,...args)=>postMessage({type, payload:args}); const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; @@ -70,6 +72,10 @@ const installAsyncProxy = function(){ this API. */ const state = Object.create(null); + + /* initS11n() is preprocessor-injected so that we have identical + copies in the synchronous and async halves. This side does not + load the SQLite library, so does not have access to that copy. */ //#define opfs-async-proxy //#include api/opfs-common-inline.c-pp.js //#undef opfs-async-proxy @@ -82,7 +88,7 @@ const installAsyncProxy = function(){ 2 = warnings and errors 3 = debug, warnings, and errors */ - state.verbose = 1; + state.verbose = 2; const loggers = { 0:console.error.bind(console), @@ -90,7 +96,7 @@ const installAsyncProxy = function(){ 2:console.log.bind(console) }; const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); + if(state.verbose>level) loggers[level]('opfs-async-proxy',workerId+":",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); @@ -297,13 +303,26 @@ const installAsyncProxy = function(){ there's another race condition there). That's easy to say but creating a viable test for that condition has proven challenging so far. + + 2026-03-06: + + - baseWaitTime is the number of milliseconds to wait for the + first retry, doubling for each retry. It defaults to + (state.asyncIdleWaitTime*2). + + - maxTries is the number of attempt to make, each one spaced out + by one additional factor of the baseWaitTime (e.g. 300, then 600, + then 900, the 1200...). This MUST be an integer >0 and defaults + to 6. + + Only the Web Locks impl should use the 3rd and 4th parameters. */ - const getSyncHandle = async (fh,opName)=>{ + const getSyncHandle = async (fh,opName, baseWaitTime, maxTries)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, - msBase = state.asyncIdleWaitTime * 2; + const msBase = baseWaitTime ?? (state.asyncIdleWaitTime * 2); + maxTries ??= 6; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -337,6 +356,9 @@ const installAsyncProxy = function(){ /** Stores the given value at state.sabOPView[state.opIds.rc] and then Atomics.notify()'s it. + + The opName is only used for logging and debugging - all result + codes are expected on the same state.sabOPView slot. */ const storeAndNotify = (opName, value)=>{ log(opName+"() => notify(",value,")"); @@ -466,24 +488,12 @@ const installAsyncProxy = function(){ await releaseImplicitLock(fh); storeAndNotify('xFileSize', rc); }, - xLock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - const fh = __openFiles[fid]; - let rc = 0; - const oldLockType = fh.xLock; - fh.xLock = lockType; - if( !fh.syncHandle ){ - try { - await getSyncHandle(fh,'xLock'); - __implicitLocks.delete(fid); - }catch(e){ - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); - fh.xLock = oldLockType; - } - } - storeAndNotify('xLock',rc); - }, + /** + The first argument is semantically invalid here - it's an + address in the synchronous side's heap. We can do nothing with + it here except use it as a unique-per-file identifier. + i.e. a lookup key. + */ xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags/*SQLITE_OPEN_...*/, opfsFlags/*OPFS_...*/){ @@ -575,22 +585,6 @@ const installAsyncProxy = function(){ await releaseImplicitLock(fh); storeAndNotify('xTruncate',rc); }, - xUnlock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - let rc = 0; - const fh = __openFiles[fid]; - if( fh.syncHandle - && state.sq3Codes.SQLITE_LOCK_NONE===lockType - /* Note that we do not differentiate between lock types in - this VFS. We're either locked or unlocked. */ ){ - try { await closeSyncHandle(fh) } - catch(e){ - state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; - } - } - storeAndNotify('xUnlock',rc); - }, xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ let rc; const fh = __openFiles[fid]; @@ -611,6 +605,187 @@ const installAsyncProxy = function(){ } }/*vfsAsyncImpls*/; + if( isWebLocker ){ + /* We require separate xLock() and xUnlock() implementations for the + original and Web Lock implementations. The ones in this block + are for the WebLock impl. */ + + /** Registry of active Web Locks: fid -> { mode, resolveRelease } */ + const __activeWebLocks = Object.create(null); + + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/, + isFromUnlock/*only if called from this.xUnlock()*/){ + const whichOp = isFromUnlock ? 'xUnlock' : 'xLock'; + const fh = __openFiles[fid]; + const lockName = "sqlite3-vfs-opfs:" + fh.filenameAbs; + //error("xLock()",fid, lockType, isFromUnlock, fh); + const requestedMode = (lockType >= state.sq3Codes.SQLITE_LOCK_RESERVED) + ? 'exclusive' : 'shared'; + const existing = __activeWebLocks[fid]; + if( existing ){ + if( existing.mode === requestedMode + || (existing.mode === 'exclusive' + && requestedMode === 'shared') ) { + storeAndNotify(whichOp, 0); + existing.mode = requestedMode/* ??? */; + fh.lockType = lockType; + return 0 /* Already held at required or higher level */; + } + /* + Upgrade path: we must release shared and acquire exclusive. + This transition is NOT atomic in Web Locks API. + */ + if( 0 ){ + /* Except that it _effectively_ is atomic if we don't call + closeSyncHandle(fh), as no other worker can lock that + until we let it go. */ + await closeSyncHandle(fh); + } + existing.resolveRelease(); + delete __activeWebLocks[fid]; + } + + const oldLockType = fh.xLock; + let didNotify = false; + return new Promise((resolveWaitLoop) => { + //error("xLock() initial promise entered..."); + navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => { + //error("xLock() Web Lock entered.", fh); + fh.xLock = lockType; + __implicitLocks.delete(fid); + if( 1 ){ + /* Make ONE attempt to get the handle, but with a + higher-than-default retry-wait time. */ + await getSyncHandle(fh, 'xLock', 1000, 5); + }else{ + /* Try to get a lock until either we get one or trying to + results in a "not found" error (see getSyncHandle() docs). */ + while( !fh.syncHandle ){ + try{ + await getSyncHandle(fh, 'xLock', 1000, 3); + }catch(e){ + const rc = GetSyncHandleError.convertRc(e, 0); + if( rc === state.sq3Codes.SQLITE_CANTOPEN ){ + /* File was deleted - see getSyncHandle() */ + throw e; + } + error("xLock() still waiting to unlock SyncAccessHandle",fh); + } + } + } + error("xLock() SAH acquired.", fh); + const releasePromise = new Promise((resolveRelease) => { + __activeWebLocks[fid] = { mode: requestedMode, resolveRelease }; + }); + didNotify = true; + storeAndNotify(whichOp, 0) /* Unblock the C side */; + resolveWaitLoop(0) /* Unblock waitLoop() */; + await releasePromise; // Hold the lock until xUnlock + }).catch(e=>{ + /** + We have(?) a potential deadlock situation: if the above + throws, we can't just blindly storeAndNotify() here to + unlock the C side, as it might interfere with an + unrelated operation. The `didNotify` check here assumes + that any exception which can be thrown will happen before + the above `didNotify=true`. e.g. getSyncHandle() can + throw. Apropos: we probably need to be able to configure + the async side with busy timeout values, and try until + that limit is reached, or tell it to wait indefinitely. + + Because waitLoop() is `await`ing on this Promise, we can + be sure that the following storeAndNotify() is not + crossing wires with a different operation. + */ + fh.xLock = oldLockType; + error("Exception acquiring Web Lock", e); + if( !didNotify ){ + state.s11n.storeException(1, e); + const rc = GetSyncHandleError.convertRc(e, state.sq3Codes.SQLITE_IOERR_LOCK); + storeAndNotify(whichOp, rc); + } + throw e /* what else can we do? */; + }) + }); + }; + + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + const fh = __openFiles[fid]; + const existing = __activeWebLocks[fid]; + if( !existing ){ + await closeSyncHandle(fh); + storeAndNotify('xUnlock', 0); + return 0; + } + error("xUnlock()",fid, lockType, fh); + let rc = 0; + if( lockType === state.sq3Codes.SQLITE_LOCK_NONE ){ + /* SQLite usually unlocks all the way to NONE */ + existing.resolveRelease(); + delete __activeWebLocks[fid]; + try {await closeSyncHandle(fh)} + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + } + }else if( lockType === state.sq3Codes.SQLITE_LOCK_SHARED + && existing.mode === 'exclusive' ){ + /* downgrade Exclusive -> Shared */ + existing.resolveRelease(); + delete __activeWebLocks[fid]; + return vfsAsyncImpls.xLock(fid, lockType, true); + }else{ + /* ??? */ + error("xUnlock() unhandled condition", fh); + } + storeAndNotify('xUnlock', rc); + if( 0===rc ) fh.lockType = lockType; + return 0; + } + + }else{ + + /* Original/"legacy" xLock() and xUnlock() */ + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + const fh = __openFiles[fid]; + let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; + if( !fh.syncHandle ){ + try { + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); + }catch(e){ + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); + fh.xLock = oldLockType; + } + } + storeAndNotify('xLock',rc); + }; + + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + let rc = 0; + const fh = __openFiles[fid]; + if( fh.syncHandle + && state.sq3Codes.SQLITE_LOCK_NONE===lockType + /* Note that we do not differentiate between lock types in + this VFS. We're either locked or unlocked. */ ){ + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + } + } + storeAndNotify('xUnlock',rc); + } + + }/*xLock() and xUnlock() impls*/ + const waitLoop = async function f(){ if( !f.inited ){ f.inited = true; @@ -645,7 +820,7 @@ const installAsyncProxy = function(){ an exception string written by the upcoming operation */ ) || []; - //warn("waitLoop() whichOp =",opId, hnd, args); + //error("waitLoop() whichOp =",opId, f.opHandlers[opId].key, args); await hnd(...args); }catch(e){ error('in waitLoop():', e); @@ -656,7 +831,7 @@ const installAsyncProxy = function(){ navigator.storage.getDirectory().then(function(d){ state.rootDir = d; globalThis.onmessage = function({data}){ - warn(globalThis.location.href,"onmessage()",data); + //log(globalThis.location.href,"onmessage()",data); switch(data.type){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 89b52c836d..6d0988d6d7 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -86,43 +86,31 @@ const installOpfsVfs = async function callee(options){ const capi = sqlite3.capi, state = opfsUtil.createVfsState('opfs', options), opfsVfs = state.vfs, - metrics = opfsVfs.metrics.counters, mTimeStart = opfsVfs.mTimeStart, mTimeEnd = opfsVfs.mTimeEnd, opRun = opfsVfs.opRun, + debug = (...args)=>sqlite3.config.debug("opfs:",...args), __openFiles = opfsVfs.__openFiles; - /* At this point, createVfsState() has populated state and - opfsVfs with any code common to both the "opfs" and "opfs-wl" + //debug("options:",JSON.stringify(options)); + /* At this point, createVfsState() has populated `state` and + `opfsVfs` with any code common to both the "opfs" and "opfs-wl" VFSes. Now comes the VFS-dependent work... */ return opfsVfs.bindVfs(util.nu({ xLock: function(pFile,lockType){ mTimeStart('xLock'); - ++metrics.xLock.count; + debug("xLock()..."); const f = __openFiles[pFile]; - let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ - if( f.lockType ) { - f.lockType = lockType; - }else{ - rc = opRun('xLock', pFile, lockType); - if( 0===rc ) f.lockType = lockType; - } + const rc = opRun('xLock', pFile, lockType); + debug("xLock() rc ",rc); + if( 0===rc ) f.lockType = lockType; mTimeEnd(); return rc; }, xUnlock: function(pFile,lockType){ mTimeStart('xUnlock'); - ++metrics.xUnlock.count; const f = __openFiles[pFile]; - let rc = 0; - if( capi.SQLITE_LOCK_NONE === lockType - && f.lockType ){ - rc = opRun('xUnlock', pFile, lockType); - } + const rc = opRun('xUnlock', pFile, lockType); if( 0===rc ) f.lockType = lockType; mTimeEnd(); return rc; @@ -137,15 +125,17 @@ const installOpfsVfs = async function callee(options){ OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsDb = OpfsDb; OpfsDb.importDb = opfsUtil.importDb; - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( - opfsVfs.pointer, - function(oo1Db, sqlite3){ - /* Set a relatively high default busy-timeout handler to - help OPFS dbs deal with multi-tab/multi-worker - contention. */ - sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); - } - ); + if( false ){ + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + /* Set a relatively high default busy-timeout handler to + help OPFS dbs deal with multi-tab/multi-worker + contention. */ + sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); + } + ); + } }/*extend sqlite3.oo1*/ })/*bindVfs()*/; }/*installOpfsVfs()*/; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 083b5eca44..8c0ba19165 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -129,17 +129,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }; } } - const reportFinalTestStatus = function(pass){ - if(isUIThread()){ - let e = document.querySelector('#color-target'); - e.classList.add(pass ? 'tests-pass' : 'tests-fail'); - e = document.querySelector('title'); - e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText; - }else{ - postMessage({type:'test-result', payload:{pass}}); - } - TestUtil.checkHeapSize(true); - }; const log = (...args)=>{ //console.log(...args); logClass('',...args); @@ -157,6 +146,19 @@ globalThis.sqlite3InitModule = sqlite3InitModule; console.debug('tester1',...args); }; + const reportFinalTestStatus = function(pass){ + debug("Final test status:",pass); + if(isUIThread()){ + let e = document.querySelector('#color-target'); + e.classList.add(pass ? 'tests-pass' : 'tests-fail'); + e = document.querySelector('title'); + e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText; + }else{ + postMessage({type:'test-result', payload:{pass}}); + } + TestUtil.checkHeapSize(true); + }; + const toss = (...args)=>{ error(...args); throw new Error(args.join(' ')); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 77e320e9af..365900c2d5 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -1,15 +1,17 @@ -importScripts( - (new URL(globalThis.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' -); -//const sqlite3InitModule = (await import("../../../jswasm/sqlite3.mjs", )).default; +'use strict'; +const urlArgs = new URL(globalThis.location.href).searchParams; +const options = { + workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0, + vfs: urlArgs.get('vfs') +}; +const jsSqlite = urlArgs.get('sqlite3.dir') + +'/sqlite3.js?opfs-async-proxy-id=' + +options.workerName; +importScripts(jsSqlite)/*Sigh - URL args are not propagated this way*/; +//const sqlite3InitModule = (await import(jsSqlite)).default; globalThis.sqlite3InitModule.__isUnderTest = true; globalThis.sqlite3InitModule().then(async function(sqlite3){ - const urlArgs = new URL(globalThis.location.href).searchParams; - const options = { - workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), - unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/, - vfs: urlArgs.get('vfs') - }; const wPost = (type,...payload)=>{ postMessage({type, worker: options.workerName, payload}); }; @@ -41,7 +43,9 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ db.close(); } if(interval.error){ - wPost('failed',"Ending work after interval #"+interval.count, + stderr("Ending work at interval #"+interval.count, + "due to error:", interval.error); + wPost('failed', "at interval #"+interval.count, "due to error:",interval.error); }else{ wPost('finished',"Ending work after",interval.count,"intervals."); diff --git a/manifest b/manifest index 94e2e47dac..78515cb419 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sextraneous\sOPFS\smetrics\sincrement. -D 2026-03-06T11:49:36.430 +C This\sWeb\sLock\simpl\scan\sreliably\srun\sa\ssingle\sOPFS\sconnection\sbut\srather\sunreliably\s'loses'\sworkers\swith\shigher\scounts,\spresumably\sdue\sto\sdeadlock\sor\sdeadly\sembrace\s(how\s_all_\sof\sthem\scan\sdeadlock\sat\sonce\sis\sunclear,\sbut\sclearly\sa\sbug). +D 2026-03-06T16:04:21.050 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js ecc3c69dc8a3b676f6999ef1b2ac4be825b7952ce5acbe86941f4f2f38607cb7 +F ext/wasm/api/opfs-common-shared.c-pp.js 7bfbf3a5ce1b558ec3d0b3e14f375e9f6003b6bb49c58480e2fbf2726cf59b2c F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -594,12 +594,12 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 5f95c210183d203c7bc5be4a8139cc7fa2e50810c306bb2651648c32ab419fcf +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f471178310dd8c27f46aaa3bb2bdfb8e0b695b37a54b9a9c66b820e413b55f68 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 8233c5f9021b0213134e2adbaf6036b8f1dffd4747083a4087c1c19ae107f962 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js f3b7296480984bcc6050fe9724a8b215c405977dd69daea7145ece25751e4b33 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 3fde62ac67c963ee04030cf279357bb19b98a420973f55245c44396828b582d6 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -644,10 +644,10 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 +F ext/wasm/tester1.c-pp.js a4e79fbf63bb3255d2b8ffc1cd538c115d2f6b599bc324904c80f6644379a284 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js 74f4ef9a827d081e6bb0ffb1d124bb54015dab8f7ae47abd5b5f26d71633331a -F ext/wasm/tests/opfs/concurrency/worker.js 3425e6dad755a1c69a6efc63a47a3ade4e7f0a9a138994ba37f996571fb46288 +F ext/wasm/tests/opfs/concurrency/worker.js d0303b1403867e97455f7563285af3eb4471961b19bc22e45d021d896d48e27c F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 521bb140b7ed237c118ac9094732d06907229a6ff385502e850c679bd623fd58 -R 334abfd624b27a29d02bb7116f94ced4 +P bf3548a37712e848c7a9cadfdc1669a2be572ea0a0c28d84c157ab30f8c30c44 +R aeb14d0bd734de11301263e4e120c1e3 U stephan -Z 944b558939ff5f7ba7e037ff2a332b5b +Z c3b8772f8a9f1db66a44dbdb2829ac29 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 48801140c5..17f5b7a2ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf3548a37712e848c7a9cadfdc1669a2be572ea0a0c28d84c157ab30f8c30c44 +d4e8583e2e80665adfe4e814adb6c219936af1dcac4105795045cb1a7b1e4864 From c008b8fe3ee8bd06ca8f17cf45052566fa669ca2 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 16:15:32 +0000 Subject: [PATCH 20/32] Correct an internal doc falsehood. FossilOrigin-Name: 53aa080e357d7a2ffeab68a3584fda43d51ecef3dc8a1d46dd32392ae4f9740c --- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 5c6a11dfd2..89b1339a9e 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -307,8 +307,8 @@ const installAsyncProxy = function(){ 2026-03-06: - baseWaitTime is the number of milliseconds to wait for the - first retry, doubling for each retry. It defaults to - (state.asyncIdleWaitTime*2). + first retry, increasing by one factor for each retry. It defaults + to (state.asyncIdleWaitTime*2). - maxTries is the number of attempt to make, each one spaced out by one additional factor of the baseWaitTime (e.g. 300, then 600, diff --git a/manifest b/manifest index 78515cb419..b868981442 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sWeb\sLock\simpl\scan\sreliably\srun\sa\ssingle\sOPFS\sconnection\sbut\srather\sunreliably\s'loses'\sworkers\swith\shigher\scounts,\spresumably\sdue\sto\sdeadlock\sor\sdeadly\sembrace\s(how\s_all_\sof\sthem\scan\sdeadlock\sat\sonce\sis\sunclear,\sbut\sclearly\sa\sbug). -D 2026-03-06T16:04:21.050 +C Correct\san\sinternal\sdoc\sfalsehood. +D 2026-03-06T16:15:32.423 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -594,7 +594,7 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f471178310dd8c27f46aaa3bb2bdfb8e0b695b37a54b9a9c66b820e413b55f68 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js fd41c554fa0f25770841f4973eeaeda78db5fd9c24b2cbe7c62ce43bfae89840 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P bf3548a37712e848c7a9cadfdc1669a2be572ea0a0c28d84c157ab30f8c30c44 -R aeb14d0bd734de11301263e4e120c1e3 +P d4e8583e2e80665adfe4e814adb6c219936af1dcac4105795045cb1a7b1e4864 +R 88b88687281dda4abda2f12f0e145b46 U stephan -Z c3b8772f8a9f1db66a44dbdb2829ac29 +Z de5c6fc88fd27ce91eec8f28640e7eb1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 17f5b7a2ef..0a3bf6f088 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4e8583e2e80665adfe4e814adb6c219936af1dcac4105795045cb1a7b1e4864 +53aa080e357d7a2ffeab68a3584fda43d51ecef3dc8a1d46dd32392ae4f9740c From a98f278f581c1cf7517fed71370e102f97c019a6 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 17:10:28 +0000 Subject: [PATCH 21/32] This one reliably runs 5 workers. Checking in before subsequent cleanups and debug output removal break it. FossilOrigin-Name: ba81d95febc5fd0f9bbb2685fef5b1b10f9991751f2bdfafba80c15877af1cef --- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 109 +++++++----------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 50 insertions(+), 73 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 89b1339a9e..e2b48bff57 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -629,17 +629,18 @@ const installAsyncProxy = function(){ && requestedMode === 'shared') ) { storeAndNotify(whichOp, 0); existing.mode = requestedMode/* ??? */; - fh.lockType = lockType; + fh.xLock = lockType; return 0 /* Already held at required or higher level */; } /* Upgrade path: we must release shared and acquire exclusive. This transition is NOT atomic in Web Locks API. */ - if( 0 ){ + if( 1 ){ /* Except that it _effectively_ is atomic if we don't call closeSyncHandle(fh), as no other worker can lock that - until we let it go. */ + until we let it go. But we can't do that without leading + to a deadly embrace, so... */ await closeSyncHandle(fh); } existing.resolveRelease(); @@ -647,101 +648,77 @@ const installAsyncProxy = function(){ } const oldLockType = fh.xLock; - let didNotify = false; return new Promise((resolveWaitLoop) => { //error("xLock() initial promise entered..."); navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => { //error("xLock() Web Lock entered.", fh); - fh.xLock = lockType; + fh.xLock = lockType/*must be set before getSyncHandle() is called*/; __implicitLocks.delete(fid); - if( 1 ){ + let rc = 0; + try{ /* Make ONE attempt to get the handle, but with a - higher-than-default retry-wait time. */ - await getSyncHandle(fh, 'xLock', 1000, 5); - }else{ - /* Try to get a lock until either we get one or trying to - results in a "not found" error (see getSyncHandle() docs). */ - while( !fh.syncHandle ){ - try{ - await getSyncHandle(fh, 'xLock', 1000, 3); - }catch(e){ - const rc = GetSyncHandleError.convertRc(e, 0); - if( rc === state.sq3Codes.SQLITE_CANTOPEN ){ - /* File was deleted - see getSyncHandle() */ - throw e; - } - error("xLock() still waiting to unlock SyncAccessHandle",fh); - } - } + higher-than-default retry-wait. */ + await getSyncHandle(fh, 'xLock', 317, 5); + }catch(e){ + fh.xLock = oldLockType; + state.s11n.storeException(1, e); + rc = GetSyncHandleError.convertRc(e, state.sq3Codes.SQLITE_BUSY); } - error("xLock() SAH acquired.", fh); - const releasePromise = new Promise((resolveRelease) => { - __activeWebLocks[fid] = { mode: requestedMode, resolveRelease }; - }); - didNotify = true; - storeAndNotify(whichOp, 0) /* Unblock the C side */; + const releasePromise = rc + ? undefined + : new Promise((resolveRelease) => { + __activeWebLocks[fid] = { mode: requestedMode, resolveRelease }; + }); + storeAndNotify(whichOp, rc) /* Unblock the C side */; resolveWaitLoop(0) /* Unblock waitLoop() */; await releasePromise; // Hold the lock until xUnlock - }).catch(e=>{ - /** - We have(?) a potential deadlock situation: if the above - throws, we can't just blindly storeAndNotify() here to - unlock the C side, as it might interfere with an - unrelated operation. The `didNotify` check here assumes - that any exception which can be thrown will happen before - the above `didNotify=true`. e.g. getSyncHandle() can - throw. Apropos: we probably need to be able to configure - the async side with busy timeout values, and try until - that limit is reached, or tell it to wait indefinitely. - - Because waitLoop() is `await`ing on this Promise, we can - be sure that the following storeAndNotify() is not - crossing wires with a different operation. - */ - fh.xLock = oldLockType; - error("Exception acquiring Web Lock", e); - if( !didNotify ){ - state.s11n.storeException(1, e); - const rc = GetSyncHandleError.convertRc(e, state.sq3Codes.SQLITE_IOERR_LOCK); - storeAndNotify(whichOp, rc); - } - throw e /* what else can we do? */; - }) + }); }); }; + const wlCloseHandle = async(fh)=>{ + let rc = 0; + try{ + await closeSyncHandle(fh); + }catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + } + return rc; + }; + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ const fh = __openFiles[fid]; const existing = __activeWebLocks[fid]; if( !existing ){ - await closeSyncHandle(fh); - storeAndNotify('xUnlock', 0); - return 0; + const rc = await wlCloseHandle(fh); + storeAndNotify('xUnlock', rc); + return rc; } error("xUnlock()",fid, lockType, fh); let rc = 0; if( lockType === state.sq3Codes.SQLITE_LOCK_NONE ){ /* SQLite usually unlocks all the way to NONE */ + rc = await wlCloseHandle(fh); existing.resolveRelease(); delete __activeWebLocks[fid]; - try {await closeSyncHandle(fh)} - catch(e){ - state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; - } + fh.xLock = lockType; }else if( lockType === state.sq3Codes.SQLITE_LOCK_SHARED && existing.mode === 'exclusive' ){ /* downgrade Exclusive -> Shared */ - existing.resolveRelease(); - delete __activeWebLocks[fid]; - return vfsAsyncImpls.xLock(fid, lockType, true); + rc = await wlCloseHandle(fh); + if( 0===rc ){ + fh.xLock = lockType; + existing.resolveRelease(); + delete __activeWebLocks[fid]; + return vfsAsyncImpls.xLock(fid, lockType, true); + } }else{ /* ??? */ error("xUnlock() unhandled condition", fh); } storeAndNotify('xUnlock', rc); - if( 0===rc ) fh.lockType = lockType; return 0; } diff --git a/manifest b/manifest index b868981442..d85a70c591 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\san\sinternal\sdoc\sfalsehood. -D 2026-03-06T16:15:32.423 +C This\sone\sreliably\sruns\s5\sworkers.\sChecking\sin\sbefore\ssubsequent\scleanups\sand\sdebug\soutput\sremoval\sbreak\sit. +D 2026-03-06T17:10:28.455 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -594,7 +594,7 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js fd41c554fa0f25770841f4973eeaeda78db5fd9c24b2cbe7c62ce43bfae89840 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b3235922c15ee9b92a5424e34580bf16cb971adf23559c9e7119d563b8da2fe9 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P d4e8583e2e80665adfe4e814adb6c219936af1dcac4105795045cb1a7b1e4864 -R 88b88687281dda4abda2f12f0e145b46 +P 53aa080e357d7a2ffeab68a3584fda43d51ecef3dc8a1d46dd32392ae4f9740c +R fcfb0bfb06d58bd7f0d57908763a15cf U stephan -Z de5c6fc88fd27ce91eec8f28640e7eb1 +Z 62318d6c194878a798365e388df66b6b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0a3bf6f088..fc702daf45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53aa080e357d7a2ffeab68a3584fda43d51ecef3dc8a1d46dd32392ae4f9740c +ba81d95febc5fd0f9bbb2685fef5b1b10f9991751f2bdfafba80c15877af1cef From a85d09f1a47854b5279868bc8cfa79ec90241315 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 19:33:25 +0000 Subject: [PATCH 22/32] Minor cleanups and docs. Teach the OPFS concurrency tester to deal with SQLITE_BUSY instead of failing. FossilOrigin-Name: 247ffed141f66a6a5a396a3e002995a9f00c70333271199200530066e77956c4 --- ext/wasm/api/opfs-common-shared.c-pp.js | 96 +++++++++---------- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 50 ++++++---- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 10 +- ext/wasm/tests/opfs/concurrency/worker.js | 62 ++++++++---- manifest | 18 ++-- manifest.uuid | 2 +- 6 files changed, 137 insertions(+), 101 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 2020184173..734b211c09 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -588,7 +588,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ of this value is also used for determining how long to wait on lock contention to free up. */ - state.asyncIdleWaitTime = isWebLocker ? 150 : 150; + state.asyncIdleWaitTime = isWebLocker ? 250 : 150; /** Whether the async counterpart should log exceptions to @@ -599,7 +599,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 0 = no exception logging. 1 = only log exceptions for "significant" ops like xOpen(), - xRead(), and xWrite(). + xRead(), and xWrite(). Exceptions related to, e.g., wait/retry + loops in acquiring SyncAccessHandles are not logged. 2 = log all exceptions. */ @@ -636,29 +637,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ state.opIds = Object.create(null); { - /* - Maintenance reminder: - - Some of these fields are only for use by the "opfs-wl" VFS, - but they must also be set up for the "ofps" VFS so that the - sizes and offsets calculated here are consistent in the async - proxy. Hypothetically they could differ and it would cope - but... why invite disaster over eliding a few superfluous (for - "opfs') properties? - */ /* Indexes for use in our SharedArrayBuffer... */ let i = 0; /* SAB slot used to communicate which operation is desired between both workers. This worker writes to it and the other - listens for changes. */ + listens for changes and clears it. The values written to it + are state.opIds.x[A-Z][a-z]+, defined below.*/ state.opIds.whichOp = i++; - /* Slot for storing return values. This worker listens to that - slot and the other worker writes to it. */ + /* Slot for storing return values. This side listens to that + slot and the async proxy writes to it. */ state.opIds.rc = i++; - /* Each function gets an ID which this worker writes to - the whichOp slot. The async-api worker uses Atomic.wait() - on the whichOp slot to figure out which operation to run - next. */ + /* Each function gets an ID which this worker writes to the + state.opIds.whichOp slot. The async-api worker uses + Atomic.wait() on the whichOp slot to figure out which + operation to run next. */ state.opIds.xAccess = i++; state.opIds.xClose = i++; state.opIds.xDelete = i++; @@ -672,25 +664,28 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ state.opIds.xTruncate = i++; state.opIds.xUnlock = i++; state.opIds.xWrite = i++; - state.opIds.mkdir = i++; + state.opIds.mkdir = i++ /*currently unused*/; /** Internal signals which are used only during development and testing via the dev console. */ state.opIds['opfs-async-metrics'] = i++; state.opIds['opfs-async-shutdown'] = i++; /* The retry slot is used by the async part for wait-and-retry - semantics. Though we could hypothetically use the xSleep slot - for that, doing so might lead to undesired side effects. */ + semantics. It is never written to, only used as a convenient + place to wait-with-timeout for a value which will never be + written, i.e. sleep()ing, before retrying a failed attempt to + acquire a SharedAccessHandle. */ state.opIds.retry = i++; state.sabOP = new SharedArrayBuffer( - i * 4/* ==sizeof int32, noting that Atomics.wait() and friends - can only function on Int32Array views of an SAB. */); + i * 4/* 4==sizeof int32, noting that Atomics.wait() and + friends can only function on Int32Array views of an + SAB. */); } /** SQLITE_xxx constants to export to the async worker counterpart... */ state.sq3Codes = Object.create(null); - [ + for(const k of [ 'SQLITE_ACCESS_EXISTS', 'SQLITE_ACCESS_READWRITE', 'SQLITE_BUSY', @@ -724,17 +719,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'SQLITE_LOCK_RESERVED', 'SQLITE_LOCK_PENDING', 'SQLITE_LOCK_EXCLUSIVE' - ].forEach((k)=>{ - if(undefined === (state.sq3Codes[k] = capi[k])){ - toss("Maintenance required: not found:",k); - } - }); + ]){ + state.sq3Codes[k] = + capi[k] ?? toss("Maintenance required: not found:",k); + } state.opfsFlags = Object.assign(Object.create(null),{ /** Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" enables this. See defaultUnlockAsap, below. - */ + */ OPFS_UNLOCK_ASAP: 0x01, /** Flag for use with xOpen(). URI flag "delete-before-open=1" @@ -747,33 +741,34 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ downstream errors. An unlink can fail if, e.g., another tab has the handle open. - It goes without saying that deleting a file out from under another - instance results in Undefined Behavior. + It goes without saying that deleting a file out from under + another instance results in Undefined Behavior. */ OPFS_UNLINK_BEFORE_OPEN: 0x02, /** - If true, any async routine which implicitly acquires a sync - access handle (i.e. an OPFS lock) will release that lock at - the end of the call which acquires it. If false, such - "autolocks" are not released until the VFS is idle for some - brief amount of time. + If true, any async routine which must implicitly acquire a + sync access handle (i.e. an OPFS lock), without an active + xLock(), will release that lock at the end of the call which + acquires it. If false, such implicit locks are not released + until the VFS is idle for some brief amount of time, as + defined by state.asyncIdleWaitTime. - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). + The benefit of enabling this is higher concurrency. The + down-side is much-reduced performance (as much as a 4x + decrease in speedtest1). */ defaultUnlockAsap: false }); - opfsVfs.metrics.reset(); + opfsVfs.metrics.reset()/*must not be called until state.opIds is set up*/; const metrics = opfsVfs.metrics.counters; /** Runs the given operation (by name) in the async worker counterpart, waits for its response, and returns the result - which the async worker writes to SAB[state.opIds.rc]. The - 2nd and subsequent arguments must be the arguments for the - async op. + which the async worker writes to SAB[state.opIds.rc]. The 2nd + and subsequent arguments must be the arguments for the async op + (see sqlite3-opfs-async-proxy.c-pp.js). */ const opRun = opfsVfs.opRun = (op,...args)=>{ const opNdx = state.opIds[op] || toss("Invalid op ID:",op); @@ -791,14 +786,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ https://github.com/sqlite/sqlite-wasm/issues/12 Summary: in at least one browser flavor, under high loads, - the wait()/notify() pairings can get out of sync. Calling - wait() here until it returns 'not-equal' gets them back in - sync. + the wait()/notify() pairings can get out of sync and/or + spuriously wake up. Calling wait() here until it returns + 'not-equal' gets them back in sync. */ } /* When the above wait() call returns 'not-equal', the async - half will have completed the operation and reported its results - in the state.opIds.rc slot of the SAB. */ + half will have completed the operation and reported its + results in the state.opIds.rc slot of the SAB. It may have + also serialized an exception for us. */ const rc = Atomics.load(state.sabOPView, state.opIds.rc); metrics[op].wait += performance.now() - t; if(rc && state.asyncS11nExceptions){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index e2b48bff57..e83cdf416d 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -618,7 +618,6 @@ const installAsyncProxy = function(){ isFromUnlock/*only if called from this.xUnlock()*/){ const whichOp = isFromUnlock ? 'xUnlock' : 'xLock'; const fh = __openFiles[fid]; - const lockName = "sqlite3-vfs-opfs:" + fh.filenameAbs; //error("xLock()",fid, lockType, isFromUnlock, fh); const requestedMode = (lockType >= state.sq3Codes.SQLITE_LOCK_RESERVED) ? 'exclusive' : 'shared'; @@ -627,38 +626,48 @@ const installAsyncProxy = function(){ if( existing.mode === requestedMode || (existing.mode === 'exclusive' && requestedMode === 'shared') ) { - storeAndNotify(whichOp, 0); - existing.mode = requestedMode/* ??? */; fh.xLock = lockType; + storeAndNotify(whichOp, 0); + /* Don't do this: existing.mode = requestedMode; + + Paraphrased from advice given by a consultanting + developer: + + If you hold an exclusive lock and SQLite requests shared, + you should keep exiting.mode as exclusive in because the + underlying Web Lock is still exclusive. Changing it to + shared would trick xLock into thinking it needs to + perform a release/re-acquire dance if an exclusive is + later requested. + */ return 0 /* Already held at required or higher level */; } /* Upgrade path: we must release shared and acquire exclusive. This transition is NOT atomic in Web Locks API. + + Except that it _effectively_ is atomic if we don't call + closeSyncHandle(fh), as no other worker can lock that + until we let it go. But we can't do that without leading + to a deadly embrace, so... */ - if( 1 ){ - /* Except that it _effectively_ is atomic if we don't call - closeSyncHandle(fh), as no other worker can lock that - until we let it go. But we can't do that without leading - to a deadly embrace, so... */ - await closeSyncHandle(fh); - } + await closeSyncHandle(fh); existing.resolveRelease(); delete __activeWebLocks[fid]; } + const lockName = "sqlite3-vfs-opfs:" + fh.filenameAbs; const oldLockType = fh.xLock; return new Promise((resolveWaitLoop) => { //error("xLock() initial promise entered..."); navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => { //error("xLock() Web Lock entered.", fh); - fh.xLock = lockType/*must be set before getSyncHandle() is called*/; + fh.xLock = lockType/*must be set before getSyncHandle() is called!*/; __implicitLocks.delete(fid); let rc = 0; try{ - /* Make ONE attempt to get the handle, but with a - higher-than-default retry-wait. */ - await getSyncHandle(fh, 'xLock', 317, 5); + /* Make only one attempt to get the handle. */ + await getSyncHandle(fh, 'xLock'); }catch(e){ fh.xLock = oldLockType; state.s11n.storeException(1, e); @@ -669,16 +678,19 @@ const installAsyncProxy = function(){ : new Promise((resolveRelease) => { __activeWebLocks[fid] = { mode: requestedMode, resolveRelease }; }); - storeAndNotify(whichOp, rc) /* Unblock the C side */; - resolveWaitLoop(0) /* Unblock waitLoop() */; - await releasePromise; // Hold the lock until xUnlock + storeAndNotify(whichOp, rc) /* unblock the C side */; + resolveWaitLoop(0) /* unblock waitLoop() */; + await releasePromise /* hold the lock until xUnlock */; }); }); }; + /** Internal helper for the opfs-wl xUnlock() */ const wlCloseHandle = async(fh)=>{ let rc = 0; try{ + /* For the record, we've never once seen closeSyncHandle() + throw, nor should it because destructors do not throw. */ await closeSyncHandle(fh); }catch(e){ state.s11n.storeException(1,e); @@ -696,7 +708,7 @@ const installAsyncProxy = function(){ storeAndNotify('xUnlock', rc); return rc; } - error("xUnlock()",fid, lockType, fh); + //error("xUnlock()",fid, lockType, fh); let rc = 0; if( lockType === state.sq3Codes.SQLITE_LOCK_NONE ){ /* SQLite usually unlocks all the way to NONE */ @@ -723,8 +735,8 @@ const installAsyncProxy = function(){ } }else{ - /* Original/"legacy" xLock() and xUnlock() */ + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ const fh = __openFiles[fid]; diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 6d0988d6d7..ebe74e38b6 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -90,6 +90,7 @@ const installOpfsVfs = async function callee(options){ mTimeEnd = opfsVfs.mTimeEnd, opRun = opfsVfs.opRun, debug = (...args)=>sqlite3.config.debug("opfs:",...args), + warn = (...args)=>sqlite3.config.warn("opfs:",...args), __openFiles = opfsVfs.__openFiles; //debug("options:",JSON.stringify(options)); @@ -99,11 +100,14 @@ const installOpfsVfs = async function callee(options){ return opfsVfs.bindVfs(util.nu({ xLock: function(pFile,lockType){ mTimeStart('xLock'); - debug("xLock()..."); + //debug("xLock()..."); const f = __openFiles[pFile]; const rc = opRun('xLock', pFile, lockType); - debug("xLock() rc ",rc); - if( 0===rc ) f.lockType = lockType; + if( rc ){ + warn("xLock() rc ",rc); + }else{ + f.lockType = lockType; + } mTimeEnd(); return rc; }, diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 365900c2d5..0f9c9b68cb 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -61,17 +61,31 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ stderr("Invalid VFS name:",vfs); return; } - db = new ctor({ - filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c' - }); - sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); - db.transaction((db)=>{ - db.exec([ - "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", - "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" - ]); - }); + while(true){ + try{ + if( !db ){ + db = new ctor({ + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, + flags: 'c' + }); + sqlite3.capi.sqlite3_busy_timeout(db.pointer, 15000); + } + db.transaction((db)=>{ + db.exec([ + "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", + "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" + ]); + }); + break; + }catch(e){ + if(e instanceof sqlite3.SQLite3Error + && sqlite3.capi.SQLITE_BUSY===e.resultCode){ + stderr("Retrying for BUSY: ",e.message); + continue; + } + throw e; + } + } const maxIterations = urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; @@ -81,14 +95,24 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ ++interval.count; const prefix = "v(#"+interval.count+")"; stdout("Setting",prefix,"=",tm); - try{ - db.exec({ - sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", - bind: [options.workerName, new Date().getTime()] - }); - //stdout("Set",prefix); - }catch(e){ - interval.error = e; + while(true){ + try{ + db.exec({ + sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", + bind: [options.workerName, new Date().getTime()] + }); + //stdout("Set",prefix); + break; + }catch(e){ + if(e instanceof sqlite3.SQLite3Error + && sqlite3.capi.SQLITE_BUSY===e.resultCode){ + stderr("Retrying for BUSY: ",e.message); + continue; + } + stderr("Error: ",e.message); + interval.error = e; + throw e; + } } //stdout("doWork()",prefix,"error ",interval.error); }; diff --git a/manifest b/manifest index d85a70c591..0974a04f21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sone\sreliably\sruns\s5\sworkers.\sChecking\sin\sbefore\ssubsequent\scleanups\sand\sdebug\soutput\sremoval\sbreak\sit. -D 2026-03-06T17:10:28.455 +C Minor\scleanups\sand\sdocs.\sTeach\sthe\sOPFS\sconcurrency\stester\sto\sdeal\swith\sSQLITE_BUSY\sinstead\sof\sfailing. +D 2026-03-06T19:33:25.647 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 7bfbf3a5ce1b558ec3d0b3e14f375e9f6003b6bb49c58480e2fbf2726cf59b2c +F ext/wasm/api/opfs-common-shared.c-pp.js eccb37a2347b8b17a664401cd8ef0ee0a7e18cb81939ee4ef404905e8e9188bf F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -594,12 +594,12 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b3235922c15ee9b92a5424e34580bf16cb971adf23559c9e7119d563b8da2fe9 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js c19ca5986bceb60561973635bd68acbb93f5e1752b1d1b7f4cae20abaa8d5bd1 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 8233c5f9021b0213134e2adbaf6036b8f1dffd4747083a4087c1c19ae107f962 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 3fde62ac67c963ee04030cf279357bb19b98a420973f55245c44396828b582d6 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js f3bef4dbb8364a37471e4bc33e9b1e52795596456090007aaeae25acc35d2e85 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -647,7 +647,7 @@ F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3 F ext/wasm/tester1.c-pp.js a4e79fbf63bb3255d2b8ffc1cd538c115d2f6b599bc324904c80f6644379a284 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js 74f4ef9a827d081e6bb0ffb1d124bb54015dab8f7ae47abd5b5f26d71633331a -F ext/wasm/tests/opfs/concurrency/worker.js d0303b1403867e97455f7563285af3eb4471961b19bc22e45d021d896d48e27c +F ext/wasm/tests/opfs/concurrency/worker.js ce1d5d7545b17f62bac2dcce2505a89c3690e1d9209512cc51512cee6e3024f5 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 53aa080e357d7a2ffeab68a3584fda43d51ecef3dc8a1d46dd32392ae4f9740c -R fcfb0bfb06d58bd7f0d57908763a15cf +P ba81d95febc5fd0f9bbb2685fef5b1b10f9991751f2bdfafba80c15877af1cef +R 5bd89f58e70b2c6dcff7610e01717205 U stephan -Z 62318d6c194878a798365e388df66b6b +Z 52bcc9f2059e4acc631f2cd7a59885ba # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc702daf45..e3c556d7b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba81d95febc5fd0f9bbb2685fef5b1b10f9991751f2bdfafba80c15877af1cef +247ffed141f66a6a5a396a3e002995a9f00c70333271199200530066e77956c4 From 2251f0c4cc7823e338902c4c83675c4cfdc2ceb4 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 22:21:06 +0000 Subject: [PATCH 23/32] Get "opfs" and "opfs-wl" properly split. Speedtest1 and the concurrency tester are happy with opfs-wl but it is not yet plugged in to tester1. FossilOrigin-Name: 12dcd5425967b6306f3e3178babeac130962edb78e6298d545a429a1a20457cc --- ext/wasm/api/opfs-common-shared.c-pp.js | 39 ++++++--- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 27 +++--- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 86 +++++++++---------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 71 ++++++++------- manifest | 18 ++-- manifest.uuid | 2 +- 6 files changed, 135 insertions(+), 108 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 734b211c09..0bbaed930e 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -423,9 +423,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Must be called by the VFS's main installation routine and passed the options object that function receives and a reference to that - function itself (which is assumed to have a defaultProxyUri - property set on it. See sqlite3-vfs-opfs{,-wl}.c-pp.js for - examples. + function itself (we don't need this anymore). It throws if OPFS is not available. @@ -446,9 +444,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Call opfvs.bindVfs() */ - opfsUtil.initOptions = function callee(options, callee){ + opfsUtil.initOptions = function callee(vfsName, options){ const urlParams = new URL(globalThis.location.href).searchParams; - if(urlParams.has('opfs-disable')){ + if(urlParams.has(vfsName+'-disable')){ //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); return; } @@ -458,10 +456,23 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return; } options = util.nu(options); + options.vfsName = vfsName; options.verbose ??= urlParams.has('opfs-verbose') ? (+urlParams.get('opfs-verbose') || 2) : 1; options.sanityChecks ??= urlParams.has('opfs-sanity-check'); - options.proxyUri ??= callee.defaultProxyUri; + + if( true ){ + /* Doing this from one scope up does not work */ + opfsUtil.proxyUri = "sqlite3-opfs-async-proxy.js"; + if( sqlite3.scriptInfo?.sqlite3Dir ){ + opfsUtil.proxyUri = ( + sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri + ); + } + //sqlite3.config.error("proxyUri =",opfsUtil.proxyUri, sqlite3.scriptInfo); + } + + options.proxyUri ??= opfsUtil.proxyUri; if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } @@ -474,7 +485,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ options.workerId ??= (Math.random() * 10000000) | 0; } //sqlite3.config.warn("opfsUtil options =",JSON.stringify(options), 'urlParams =', urlParams); - return options; + return opfsUtil.options = options; }; /** @@ -498,8 +509,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ only cloneable or sharable objects. After the worker's "inited" message arrives, other types of data may be added to it. */ - opfsUtil.createVfsState = function(vfsName, options){ + opfsUtil.createVfsState = function(){ const state = util.nu(); + const options = opfsUtil.options; state.verbose = options.verbose; const loggers = [ @@ -507,8 +519,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.config.warn, sqlite3.config.log ]; + const vfsName = options.vfsName + || toss("Maintenance required: missing VFS name"); const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]("OPFS syncer:",...args); + if(state.verbose>level) loggers[level](vfsName+":",...args); }; const log = (...args)=>logImpl(2, ...args), warn = (...args)=>logImpl(1, ...args), @@ -544,7 +558,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ ); - const isWebLocker = 'opfs-wl'===vfsName; opfsVfs.metrics = util.nu({ counters: util.nu(), dump: function(){ @@ -588,7 +601,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ of this value is also used for determining how long to wait on lock contention to free up. */ - state.asyncIdleWaitTime = isWebLocker ? 250 : 150; + state.asyncIdleWaitTime = 150; /** Whether the async counterpart should log exceptions to @@ -771,7 +784,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ (see sqlite3-opfs-async-proxy.c-pp.js). */ const opRun = opfsVfs.opRun = (op,...args)=>{ - const opNdx = state.opIds[op] || toss("Invalid op ID:",op); + const opNdx = state.opIds[op] || toss(opfsVfs.vfsName+": Invalid op ID:",op); state.s11n.serialize(...args); Atomics.store(state.sabOPView, state.opIds.rc, -1); Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); @@ -1088,12 +1101,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ promiseWasRejected = false; return promiseResolve_(sqlite3); }; + const options = opfsUtil.options; let proxyUri = options.proxyUri +( (options.proxyUri.indexOf('?')<0) ? '?' : '&' )+'vfs='+vfsName; if( options.workerId ){ proxyUri += '&opfs-async-proxy-id='+encodeURIComponent(options.workerId); } + //sqlite3.config.error("proxyUri",options.proxyUri, (new Error())); const W = opfsVfs.worker = //#if target:es6-bundler-friendly (()=>{ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index e83cdf416d..797407e2c4 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -51,7 +51,8 @@ */ "use strict"; const urlParams = new URL(globalThis.location.href).searchParams; -if( !urlParams.has('vfs') ){ +const vfsName = urlParams.get('vfs'); +if( !vfsName ){ throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); } const workerId = urlParams.get('opfs-async-proxy-id') @@ -111,12 +112,13 @@ const installAsyncProxy = function(){ */ const __openFiles = Object.create(null); /** - __implicitLocks is a Set of sqlite3_file pointers (integers) which were - "auto-locked". i.e. those for which we obtained a sync access - handle without an explicit xLock() call. Such locks will be - released during db connection idle time, whereas a sync access - handle obtained via xLock(), or subsequently xLock()'d after - auto-acquisition, will not be released until xUnlock() is called. + __implicitLocks is a Set of sqlite3_file pointers (integers) + which were "auto-locked". i.e. those for which we necessarily + obtain a sync access handle without an explicit xLock() call + guarding access. Such locks will be released during + `waitLoop()`'s idle time, whereas a sync access handle obtained + via xLock(), or subsequently xLock()'d after auto-acquisition, + will not be released until xUnlock() is called. Maintenance reminder: if we relinquish auto-locks at the end of the operation which acquires them, we pay a massive performance @@ -853,16 +855,21 @@ const installAsyncProxy = function(){ wPost('opfs-async-loaded'); }).catch((e)=>error("error initializing OPFS asyncer:",e)); }/*installAsyncProxy()*/; -if(!globalThis.SharedArrayBuffer){ +if(globalThis.window === globalThis){ + wPost('opfs-unavailable', + "This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); +}else if(!globalThis.SharedArrayBuffer){ wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", "The server must emit the COOP/COEP response headers to enable that."); }else if(!globalThis.Atomics){ wPost('opfs-unavailable', "Missing Atomics API.", "The server must emit the COOP/COEP response headers to enable that."); +}else if('opfs-wl'===vfsName && !globalThis.Atomics.waitAsync){ + wPost('opfs-unavailable',"Missing required Atomics.waitSync() for "+vfsName); }else if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle || !navigator?.storage?.getDirectory){ wPost('opfs-unavailable',"Missing required OPFS APIs."); }else{ diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 8119b32469..c1d34f192d 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -49,63 +49,60 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Quirks specific to this VFS: - - Because WebLocks effectively block until they return, they will - effectively hang on locks rather than returning SQLITE_BUSY. + - The (officially undocumented) 'opfs-wl-disable' URL + argument will disable OPFS, making this function a no-op. Aside from locking differences in the VFSes, this function otherwise behaves the same as sqlite3-vfs-opfs.c-pp.js:installOpfsVfs(). */ -const installOpfsWlVfs = async function callee(options){ - options = opfsUtil.initOptions(options, callee); +const installOpfsWlVfs = async function(options){ + options = opfsUtil.initOptions('opfs-wl',options); if( !options ) return sqlite3; options.verbose = 2; const capi = sqlite3.capi, - debug = (...args)=>sqlite3.config.warn("opfs-wl:",...args), - state = opfsUtil.createVfsState('opfs-wl', options), + state = opfsUtil.createVfsState(), opfsVfs = state.vfs, metrics = opfsVfs.metrics.counters, mTimeStart = opfsVfs.mTimeStart, mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + debug = (...args)=>sqlite3.config.debug("opfs-wl:",...args), + warn = (...args)=>sqlite3.config.warn("opfs-wl:",...args), __openFiles = opfsVfs.__openFiles; - //debug("state",JSON.stringify(state,false,' ')); - /* At this point, createVfsState() has populated state and opfsVfs - with any code common to both the "opfs" and "opfs-wl" VFSes. Now - comes the VFS-dependent work... */ + + //debug("state",JSON.stringify(options)); + /* + At this point, createVfsState() has populated: + + - state: the configuration object we share with the async proxy. + + - opfsVfs: an sqlite3_vfs instance with lots of JS state attached + to it. + + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... + */ return opfsVfs.bindVfs(util.nu({ - xLock: function(pFile,lockType){ + xLock: function(pFile,lockType){ mTimeStart('xLock'); - ++metrics.xLock.count; + //debug("xLock()..."); const f = __openFiles[pFile]; - let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ - if( f.lockType ) { - f.lockType = lockType; - }else{ - rc = opRun('xLock', pFile, lockType); - if( 0===rc ) f.lockType = lockType; - } + const rc = opRun('xLock', pFile, lockType); + if( !rc ) f.lockType = lockType; mTimeEnd(); return rc; }, xUnlock: function(pFile,lockType){ mTimeStart('xUnlock'); - ++metrics.xUnlock.count; const f = __openFiles[pFile]; - let rc = 0; - if( capi.SQLITE_LOCK_NONE === lockType - && f.lockType ){ - rc = opRun('xUnlock', pFile, lockType); - } - if( 0===rc ) f.lockType = lockType; + const rc = opRun('xUnlock', pFile, lockType); + if( !rc ) f.lockType = lockType; mTimeEnd(); return rc; } }), function(sqlite3, vfs){ - //debug("registered VFS"); + /* Post-VFS-registration initialization... */ if(sqlite3.oo1){ const OpfsWlDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -115,25 +112,20 @@ const installOpfsWlVfs = async function callee(options){ OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsWlDb = OpfsWlDb; OpfsWlDb.importDb = opfsUtil.importDb; - }/*extend sqlite3.oo1*/ + /* The "opfs" VFS variant adds a + oo1.DB.dbCtorHelper.setVfsPostOpenCallback() callback to set + a high busy_timeout. That was a design mis-decision and is + inconsistent with sqlite3_open() and friends, but is retained + against the risk of introducing regressions if it's removed. + This variant does not repeat that mistake. + */ + } })/*bindVfs()*/; }/*installOpfsWlVfs()*/; -installOpfsWlVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - try{ - let proxyJs = installOpfsWlVfs.defaultProxyUri; - if( sqlite3.scriptInfo?.sqlite3Dir ){ - installOpfsWlVfs.defaultProxyUri = - sqlite3.scriptInfo.sqlite3Dir + proxyJs; - //sqlite3.config.warn("installOpfsWlVfs.defaultProxyUri =",installOpfsWlVfs.defaultProxyUri); - } - return installOpfsWlVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS-WL sqlite3_vfs:",e); - }); - }catch(e){ - sqlite3.config.error("installOpfsWlVfs() exception:",e); - return Promise.reject(e); - } + return installOpfsWlVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install the 'opfs-wl' sqlite3_vfs:",e); + }); }); }/*sqlite3ApiBootstrap.initializers.push()*/); //#endif target:node diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index ebe74e38b6..05e6a3f09f 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -60,7 +60,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The argument may optionally be a plain object with the following configuration options: - - proxyUri: name of the async proxy JS file. + - proxyUri: name of the async proxy JS file or a synchronous function + which, when called, returns such a name. - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables logging of errors. 2 enables logging of warnings and errors. 3 @@ -80,12 +81,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ as there are legitimate non-error reasons for OPFS not to be available. */ -const installOpfsVfs = async function callee(options){ - options = opfsUtil.initOptions(options, callee); +const installOpfsVfs = async function(options){ + options = opfsUtil.initOptions('opfs',options); if( !options ) return sqlite3; const capi = sqlite3.capi, - state = opfsUtil.createVfsState('opfs', options), + state = opfsUtil.createVfsState(), opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, mTimeStart = opfsVfs.mTimeStart, mTimeEnd = opfsVfs.mTimeEnd, opRun = opfsVfs.opRun, @@ -94,32 +96,51 @@ const installOpfsVfs = async function callee(options){ __openFiles = opfsVfs.__openFiles; //debug("options:",JSON.stringify(options)); - /* At this point, createVfsState() has populated `state` and - `opfsVfs` with any code common to both the "opfs" and "opfs-wl" - VFSes. Now comes the VFS-dependent work... */ + /* + At this point, createVfsState() has populated: + + - state: the configuration object we share with the async proxy. + + - opfsVfs: an sqlite3_vfs instance with lots of JS state attached + to it. + + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... + */ return opfsVfs.bindVfs(util.nu({ xLock: function(pFile,lockType){ mTimeStart('xLock'); - //debug("xLock()..."); + ++metrics.xLock.count; const f = __openFiles[pFile]; - const rc = opRun('xLock', pFile, lockType); - if( rc ){ - warn("xLock() rc ",rc); - }else{ + let rc = 0; + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( f.lockType ) { f.lockType = lockType; + }else{ + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; } mTimeEnd(); return rc; }, xUnlock: function(pFile,lockType){ mTimeStart('xUnlock'); + ++metrics.xUnlock.count; const f = __openFiles[pFile]; - const rc = opRun('xUnlock', pFile, lockType); + let rc = 0; + if( capi.SQLITE_LOCK_NONE === lockType + && f.lockType ){ + rc = opRun('xUnlock', pFile, lockType); + } if( 0===rc ) f.lockType = lockType; mTimeEnd(); return rc; } }), function(sqlite3, vfs){ + /* Post-VFS-registration initialization... */ if(sqlite3.oo1){ const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -129,7 +150,11 @@ const installOpfsVfs = async function callee(options){ OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsDb = OpfsDb; OpfsDb.importDb = opfsUtil.importDb; - if( false ){ + if( true ){ + /* 2026-03-06: this was a design mis-decision and is + inconsistent with sqlite3_open() and friends, but is + retained against the risk of introducing regressions if + it's removed. */ sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( opfsVfs.pointer, function(oo1Db, sqlite3){ @@ -143,22 +168,10 @@ const installOpfsVfs = async function callee(options){ }/*extend sqlite3.oo1*/ })/*bindVfs()*/; }/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - try{ - let proxyJs = installOpfsVfs.defaultProxyUri; - if( sqlite3.scriptInfo?.sqlite3Dir ){ - installOpfsVfs.defaultProxyUri = - sqlite3.scriptInfo.sqlite3Dir + proxyJs; - //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); - } - return installOpfsVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e); - }); - }catch(e){ - sqlite3.config.error("installOpfsVfs() exception:",e); - return Promise.reject(e); - } + return installOpfsVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install 'opfs' sqlite3_vfs:",e); + }) }); }/*sqlite3ApiBootstrap.initializers.push()*/); //#endif target:node diff --git a/manifest b/manifest index 0974a04f21..72316086e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanups\sand\sdocs.\sTeach\sthe\sOPFS\sconcurrency\stester\sto\sdeal\swith\sSQLITE_BUSY\sinstead\sof\sfailing. -D 2026-03-06T19:33:25.647 +C Get\s"opfs"\sand\s"opfs-wl"\sproperly\ssplit.\sSpeedtest1\sand\sthe\sconcurrency\stester\sare\shappy\swith\sopfs-wl\sbut\sit\sis\snot\syet\splugged\sin\sto\stester1. +D 2026-03-06T22:21:06.784 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js eccb37a2347b8b17a664401cd8ef0ee0a7e18cb81939ee4ef404905e8e9188bf +F ext/wasm/api/opfs-common-shared.c-pp.js b41b71ac06264e2d2db9a08e9cd7d5ee0e5730d224a4e81e0c42b0928028844e F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -594,12 +594,12 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js c19ca5986bceb60561973635bd68acbb93f5e1752b1d1b7f4cae20abaa8d5bd1 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 6025b22ebb92f9216e68d85c4e27c228e148f4d081e306f27d0a50342e56ec37 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 8233c5f9021b0213134e2adbaf6036b8f1dffd4747083a4087c1c19ae107f962 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js f3bef4dbb8364a37471e4bc33e9b1e52795596456090007aaeae25acc35d2e85 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js f65903e15f49a5475eaf8fd3f35971d4c4c4b43f0a00167f1a2c98389cf84900 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 872c29aa760b05c2703185368a39b4b2e1a8abf15c29ee7e23ee8ca7cdcbed9d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P ba81d95febc5fd0f9bbb2685fef5b1b10f9991751f2bdfafba80c15877af1cef -R 5bd89f58e70b2c6dcff7610e01717205 +P 247ffed141f66a6a5a396a3e002995a9f00c70333271199200530066e77956c4 +R d5ab78b2216c6ac5e35f6ab8d6aa4867 U stephan -Z 52bcc9f2059e4acc631f2cd7a59885ba +Z 836eb4f55d744876555cc67389157d1f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e3c556d7b3..31efc532c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -247ffed141f66a6a5a396a3e002995a9f00c70333271199200530066e77956c4 +12dcd5425967b6306f3e3178babeac130962edb78e6298d545a429a1a20457cc From ac63b524acdf789c9a44623023bda10d693a4e7e Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Mar 2026 23:50:44 +0000 Subject: [PATCH 24/32] Add some jitter and duration info to the opfs concurrency tester. Ensure that Atomics.waitAsync() is available for opfs-wl. Further testing has shown that Web opfs-wl is consistently fairer about doling out contested locks but it's dog slow compared to the original VFS under moderate concurrency. In single-connection use they're effectively on par. FossilOrigin-Name: a9aecc987512d60f2663973f43c769cf086fc14149edfbcb18c0aec9f5aa3dbf --- ext/wasm/api/opfs-common-shared.c-pp.js | 34 ++++++------ ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 2 +- ext/wasm/tests/opfs/concurrency/test.js | 55 +++++++++++-------- ext/wasm/tests/opfs/concurrency/worker.js | 7 ++- manifest | 18 +++--- manifest.uuid | 2 +- 6 files changed, 65 insertions(+), 53 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 0bbaed930e..17916426ef 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -402,21 +402,21 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ descriptive error message if they're not found. This is intended to be run as part of async VFS installation steps. */ - opfsUtil.vfsInstallationFeatureCheck = function(){ - if(!globalThis.SharedArrayBuffer - || !globalThis.Atomics){ - throw new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ - "The server must emit the COOP/COEP response headers to enable those. "+ - "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep"); - }else if('undefined'===typeof WorkerGlobalScope){ - throw new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ - "because it requires Atomics.wait()."); - }else if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - throw new newError("Missing required OPFS APIs."); + opfsUtil.vfsInstallationFeatureCheck = function(vfsName){ + if( !globalThis.SharedArrayBuffer || !globalThis.Atomics ){ + toss("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics.", + "The server must emit the COOP/COEP response headers to enable those.", + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep"); + }else if( 'undefined'===typeof WorkerGlobalScope ){ + toss("The OPFS sqlite3_vfs cannot run in the main thread", + "because it requires Atomics.wait()."); + }else if( !globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle || + !navigator?.storage?.getDirectory ){ + toss("Missing required OPFS APIs."); + }else if( 'opfs-wl'===vfsName && !globalThis.Atomics.waitAsync ){ + toss('The',vfsName,'VFS requires Atomics.waitAsync(), which is not available.'); } }; @@ -446,12 +446,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ opfsUtil.initOptions = function callee(vfsName, options){ const urlParams = new URL(globalThis.location.href).searchParams; - if(urlParams.has(vfsName+'-disable')){ + if( urlParams.has(vfsName+'-disable') ){ //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); return; } try{ - opfsUtil.vfsInstallationFeatureCheck(); + opfsUtil.vfsInstallationFeatureCheck(vfsName); }catch(e){ return; } diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 797407e2c4..f89f73c311 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -89,7 +89,7 @@ const installAsyncProxy = function(){ 2 = warnings and errors 3 = debug, warnings, and errors */ - state.verbose = 2; + state.verbose = 1; const loggers = { 0:console.error.bind(console), diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 18c01a5b7e..bdc15a3c4b 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -59,7 +59,7 @@ ) || 3; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 - ) || 1; + ) || 0; options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 1000 ) || 1000; @@ -76,41 +76,52 @@ for(const w of workers) w.postMessage({type, payload:args}); }; workers.counts = {loaded: 0, passed: 0, failed: 0}; + let timeStart; + const calcTime = (endDate)=>{ + return endDate.getTime() - timeStart?.getTime?.(); + }; const checkFinished = function(){ if(workers.counts.passed + workers.counts.failed !== workers.length){ return; } + stdout("Total Worker run time:",calcTime(new Date()),"ms"); if(workers.counts.failed>0){ logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); }else{ logCss('tests-pass',"All",workers.length,"workers finished."); } }; + workers.onmessage = function(msg){ msg = msg.data; const prefix = 'Worker #'+msg.worker+':'; switch(msg.type){ - case 'loaded': - stdout(prefix,"loaded"); - if(++workers.counts.loaded === workers.length){ - stdout("All",workers.length,"workers loaded. Telling them to run..."); - workers.post('run'); - } - break; - case 'stdout': stdout(prefix,...msg.payload); break; - case 'stderr': stderr(prefix,...msg.payload); break; - case 'error': stderr(prefix,"ERROR:",...msg.payload); break; - case 'finished': - ++workers.counts.passed; - logCss('tests-pass',prefix,...msg.payload); - checkFinished(); - break; - case 'failed': - ++workers.counts.failed; - logCss('tests-fail',prefix,"FAILED:",...msg.payload); - checkFinished(); - break; - default: logCss('error',"Unhandled message type:",msg); break; + case 'loaded': + stdout(prefix,"loaded"); + if(++workers.counts.loaded === workers.length){ + timeStart = new Date(); + stdout(timeStart,"All",workers.length,"workers loaded. Telling them to run..."); + workers.post('run'); + } + break; + case 'stdout': stdout(prefix,...msg.payload); break; + case 'stderr': stderr(prefix,...msg.payload); break; + case 'error': stderr(prefix,"ERROR:",...msg.payload); break; + case 'finished': { + ++workers.counts.passed; + const timeEnd = new Date(); + logCss('tests-pass',prefix,timeEnd,"("+calcTime(timeEnd)+"ms) ", ...msg.payload); + checkFinished(); + break; + } + case 'failed': { + ++workers.counts.failed; + const timeEnd = new Date(); + logCss('tests-fail',prefix,timeEnd,"("+calcTime(timeEnd)+"ms) FAILED:",...msg.payload); + checkFinished(); + break; + } + default: logCss('error',"Unhandled message type:",msg); break; } }; diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 0f9c9b68cb..75c2b8a64d 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -80,7 +80,7 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ }catch(e){ if(e instanceof sqlite3.SQLite3Error && sqlite3.capi.SQLITE_BUSY===e.resultCode){ - stderr("Retrying for BUSY: ",e.message); + stderr("Retrying",(db ? "db init" : "open()"),"for BUSY:",e.message); continue; } throw e; @@ -106,7 +106,7 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ }catch(e){ if(e instanceof sqlite3.SQLite3Error && sqlite3.capi.SQLITE_BUSY===e.resultCode){ - stderr("Retrying for BUSY: ",e.message); + stderr("Retrying interval #",interval.count,"for BUSY:",e.message); continue; } stderr("Error: ",e.message); @@ -121,7 +121,8 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ if(interval.error || maxIterations === interval.count){ finish(); }else{ - setTimeout(timer, interval.delay); + const jitter = (Math.random()*150 - Math.random()*150) | 0; + setTimeout(timer, interval.delay + jitter); } }, interval.delay); }/*run()*/; diff --git a/manifest b/manifest index 72316086e1..6c368d0569 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\s"opfs"\sand\s"opfs-wl"\sproperly\ssplit.\sSpeedtest1\sand\sthe\sconcurrency\stester\sare\shappy\swith\sopfs-wl\sbut\sit\sis\snot\syet\splugged\sin\sto\stester1. -D 2026-03-06T22:21:06.784 +C Add\ssome\sjitter\sand\sduration\sinfo\sto\sthe\sopfs\sconcurrency\stester.\sEnsure\sthat\sAtomics.waitAsync()\sis\savailable\sfor\sopfs-wl.\sFurther\stesting\shas\sshown\sthat\sWeb\sopfs-wl\sis\sconsistently\sfairer\sabout\sdoling\sout\scontested\slocks\sbut\sit's\sdog\sslow\scompared\sto\sthe\soriginal\sVFS\sunder\smoderate\sconcurrency.\sIn\ssingle-connection\suse\sthey're\seffectively\son\spar. +D 2026-03-06T23:50:44.079 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js b41b71ac06264e2d2db9a08e9cd7d5ee0e5730d224a4e81e0c42b0928028844e +F ext/wasm/api/opfs-common-shared.c-pp.js 612df64a2a159db30786c742c70a09d75b993d034f360367125747d2eca531a5 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -594,7 +594,7 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 6025b22ebb92f9216e68d85c4e27c228e148f4d081e306f27d0a50342e56ec37 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 341468dc0f322d98b032c6922c9865ef9028fafc8f713b950c5ff2c0305e2c87 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 @@ -646,8 +646,8 @@ F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c9 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb F ext/wasm/tester1.c-pp.js a4e79fbf63bb3255d2b8ffc1cd538c115d2f6b599bc324904c80f6644379a284 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e -F ext/wasm/tests/opfs/concurrency/test.js 74f4ef9a827d081e6bb0ffb1d124bb54015dab8f7ae47abd5b5f26d71633331a -F ext/wasm/tests/opfs/concurrency/worker.js ce1d5d7545b17f62bac2dcce2505a89c3690e1d9209512cc51512cee6e3024f5 +F ext/wasm/tests/opfs/concurrency/test.js b315a5480197e8da80c626d21b7939a485784bc6a13ec681115d4f5384c9b971 +F ext/wasm/tests/opfs/concurrency/worker.js b247316330d2c11817b6debc7d16e4cbc51c689f56a4cb5d6c4ec4a2bca45ca7 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 247ffed141f66a6a5a396a3e002995a9f00c70333271199200530066e77956c4 -R d5ab78b2216c6ac5e35f6ab8d6aa4867 +P 12dcd5425967b6306f3e3178babeac130962edb78e6298d545a429a1a20457cc +R 587bc730112047fe7757e9bed1469d73 U stephan -Z 836eb4f55d744876555cc67389157d1f +Z cdae14ec494ba9a4ab6ec30704fa3278 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 31efc532c3..3ea9463f59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -12dcd5425967b6306f3e3178babeac130962edb78e6298d545a429a1a20457cc +a9aecc987512d60f2663973f43c769cf086fc14149edfbcb18c0aec9f5aa3dbf From 57d0fe254a9628776eb081750543c8ce1c62fc83 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 7 Mar 2026 01:01:13 +0000 Subject: [PATCH 25/32] For backwards compatibility, ensure that the "opfs" VFS does not specifically require Atomics.waitAsync() (a new requirement of "opfs-wl"), but make use of it if available. Only apply jitter to the concurrency test runs at random intervals. FossilOrigin-Name: f2175f526c00cfe562e8f332eb197b5ef2c3d6be1fff2aab1566c2c533a293ac --- ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 78 ++++++++++++++++--- ext/wasm/tests/opfs/concurrency/index.html | 11 ++- ext/wasm/tests/opfs/concurrency/test.js | 9 ++- ext/wasm/tests/opfs/concurrency/worker.js | 9 ++- manifest | 18 ++--- manifest.uuid | 2 +- 6 files changed, 94 insertions(+), 33 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index f89f73c311..6362c6efd4 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -668,8 +668,7 @@ const installAsyncProxy = function(){ __implicitLocks.delete(fid); let rc = 0; try{ - /* Make only one attempt to get the handle. */ - await getSyncHandle(fh, 'xLock'); + await getSyncHandle(fh, 'xLock', state.asyncIdleWaitTime, 5); }catch(e){ fh.xLock = oldLockType; state.s11n.storeException(1, e); @@ -794,15 +793,64 @@ const installAsyncProxy = function(){ const opView = state.sabOPView; const slotWhichOp = opIds.whichOp; const idleWaitTime = state.asyncIdleWaitTime; + const hasWaitAsync = !!Atomics.waitAsync; while(!flagAsyncShutdown){ try { - const opId = Atomics.load(opView, slotWhichOp); - if( 0===opId ){ - const rv = Atomics.waitAsync(opView, slotWhichOp, 0, - idleWaitTime); - if( rv.async ) await rv.value; - await releaseImplicitLocks(); - continue; + let opId; + if( hasWaitAsync ){ + opId = Atomics.load(opView, slotWhichOp); + if( 0===opId ){ + const rv = Atomics.waitAsync(opView, slotWhichOp, 0, + idleWaitTime); + if( rv.async ) await rv.value; + await releaseImplicitLocks(); + continue; + } + }else{ + /** + For browsers without Atomics.waitAsync(), we require + the legacy implementation. Browser versions where + waitAsync() arrived: + + Chrome: 90 (2021-04-13) + Firefox: 145 (2025-11-11) + Safari: 16.4 (2023-03-27) + + The "opfs" VFS was not born until Chrome was somewhere in + the v104-108 range and did not work with Safari < v17 + (2023-09-18) due to a WebKit bug which restricted OPFS + access from sub-Workers. + + The waitAsync() counterpart of this block has shown to be + slightly more performant for the "opfs" VFS than this + block (whereas "opfs-wl" _requires_ that block), so we + enable it where it's available, regardless of the value + of isWebLocker, with the note that if isWebLocker is true + then the bootstrapping of this script will have failed if + waitAsync() is not available. + */ + if('not-equal'!==Atomics.wait( + state.sabOPView, slotWhichOp, 0, state.asyncIdleWaitTime + )){ + /* Maintenance note: we compare against 'not-equal' because + + https://github.com/tomayac/sqlite-wasm/issues/12 + + is reporting that this occasionally, under high loads, + returns 'ok', which leads to the whichOp being 0 (which + isn't a valid operation ID and leads to an exception, + along with a corresponding ugly console log + message). Unfortunately, the conditions for that cannot + be reliably reproduced. The only place in our code which + writes a 0 to the state.opIds.whichOp SharedArrayBuffer + index is a few lines down from here, and that instance + is required in order for clear communication between + the sync half of this proxy and this half. + */ + await releaseImplicitLocks(); + continue; + } + opId = Atomics.load(state.sabOPView, slotWhichOp); } Atomics.store(opView, slotWhichOp, 0); const hnd = f.opHandlers[opId]?.f ?? toss("No waitLoop handler for whichOp #",opId); @@ -812,6 +860,16 @@ const installAsyncProxy = function(){ operation */ ) || []; //error("waitLoop() whichOp =",opId, f.opHandlers[opId].key, args); +//#if nope + if( isWebLocker && (opId==opIds.xLock || opIds==opIds.xUnlock) ){ + /* An expert suggests that this introduces a race condition, + but my eyes aren't seeing it. The hope was that this + would improve the lock speed a tick, but it does not + appear to. */ + hnd(...args); + continue; + } +//#endif await hnd(...args); }catch(e){ error('in waitLoop():', e); @@ -865,7 +923,7 @@ if(globalThis.window === globalThis){ }else if(!globalThis.Atomics){ wPost('opfs-unavailable', "Missing Atomics API.", "The server must emit the COOP/COEP response headers to enable that."); -}else if('opfs-wl'===vfsName && !globalThis.Atomics.waitAsync){ +}else if(isWebLocker && !globalThis.Atomics.waitAsync){ wPost('opfs-unavailable',"Missing required Atomics.waitSync() for "+vfsName); }else if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index 54ed04a4f6..c5e929b010 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -25,15 +25,14 @@ the workers=N URL flag. Set the time between each workload with interval=N (milliseconds). Set the number of worker iterations with iterations=N. - Enable OPFS VFS verbosity with verbose=1-3 (output - goes to the dev console). Disable/enable "unlock ASAP" mode - (higher concurrency, lower speed) with unlock-asap=0-1. + Disable/enable "unlock ASAP" mode (potentially higher + concurrency but lower speed) with unlock-asap=0-1.

Achtung: if it does not start to do anything within a couple of seconds, check the dev console: Chrome sometimes fails to load - the wasm module due to "cannot allocate WasmMemory." Closing and - re-opening the tab usually resolves it, but sometimes restarting - the browser is required. + the wasm module due to "cannot allocate WasmMemory" when + reloading the page. Closing and re-opening the tab usually + resolves it, but sometimes restarting the browser is required.

Links for various testing options:

diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index bdc15a3c4b..f8d7f10399 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -139,12 +139,13 @@ return li.join('&'); }; for(const opt of [ - {interval: 500, workers: 1, iterations: 5, vfs: 'opfs-wl', opfsVerbose: 2}, - {interval: 500, workers: 2, iterations: 5, vfs: 'opfs-wl', opfsVerbose: 2}, {interval: 1000, workers: 5, iterations: 30}, {interval: 500, workers: 5, iterations: 30}, {interval: 250, workers: 3, iterations: 30}, - {interval: 600, workers: 5, iterations: 100} + {interval: 600, workers: 5, iterations: 100}, + {interval: 500, workers: 1, iterations: 5, vfs: 'opfs-wl'}, + {interval: 500, workers: 2, iterations: 10, vfs: 'opfs-wl'}, + {interval: 500, workers: 5, iterations: 20, vfs: 'opfs-wl'}, ]){ const li = document.createElement('li'); eTestLinks.appendChild(li); @@ -176,4 +177,4 @@ // Have to delay onmessage assignment until after the loop // to avoid that early workers get an undue head start. workers.forEach((w)=>w.onmessage = workers.onmessage); -})(self); +})(globalThis); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 75c2b8a64d..ccd1df932a 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -68,7 +68,7 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c' }); - sqlite3.capi.sqlite3_busy_timeout(db.pointer, 15000); + sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); } db.transaction((db)=>{ db.exec([ @@ -121,8 +121,11 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ if(interval.error || maxIterations === interval.count){ finish(); }else{ - const jitter = (Math.random()*150 - Math.random()*150) | 0; - setTimeout(timer, interval.delay + jitter); + const jitter = (Math.random()>=0.5) + ? ((Math.random()*100 - Math.random()*100) | 0) + : 0; + const n = interval.delay + jitter; + setTimeout(timer, n<50 ? interval.delay : n); } }, interval.delay); }/*run()*/; diff --git a/manifest b/manifest index 6c368d0569..ecc54ab16c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\sjitter\sand\sduration\sinfo\sto\sthe\sopfs\sconcurrency\stester.\sEnsure\sthat\sAtomics.waitAsync()\sis\savailable\sfor\sopfs-wl.\sFurther\stesting\shas\sshown\sthat\sWeb\sopfs-wl\sis\sconsistently\sfairer\sabout\sdoling\sout\scontested\slocks\sbut\sit's\sdog\sslow\scompared\sto\sthe\soriginal\sVFS\sunder\smoderate\sconcurrency.\sIn\ssingle-connection\suse\sthey're\seffectively\son\spar. -D 2026-03-06T23:50:44.079 +C For\sbackwards\scompatibility,\sensure\sthat\sthe\s"opfs"\sVFS\sdoes\snot\sspecifically\srequire\sAtomics.waitAsync()\s(a\snew\srequirement\sof\s"opfs-wl"),\sbut\smake\suse\sof\sit\sif\savailable.\sOnly\sapply\sjitter\sto\sthe\sconcurrency\stest\sruns\sat\srandom\sintervals. +D 2026-03-07T01:01:13.846 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -594,7 +594,7 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 341468dc0f322d98b032c6922c9865ef9028fafc8f713b950c5ff2c0305e2c87 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 296b3d3701c3b42918357e4d9e5d82c5321b371115fb51a8a405c2eaeebd9f0f F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 @@ -645,9 +645,9 @@ F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f80 F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb F ext/wasm/tester1.c-pp.js a4e79fbf63bb3255d2b8ffc1cd538c115d2f6b599bc324904c80f6644379a284 -F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e -F ext/wasm/tests/opfs/concurrency/test.js b315a5480197e8da80c626d21b7939a485784bc6a13ec681115d4f5384c9b971 -F ext/wasm/tests/opfs/concurrency/worker.js b247316330d2c11817b6debc7d16e4cbc51c689f56a4cb5d6c4ec4a2bca45ca7 +F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226 +F ext/wasm/tests/opfs/concurrency/test.js 520061e6a624fe0a7c07cf3438cb6b9b7c9dcbd606e644ccb5fb52518c48b342 +F ext/wasm/tests/opfs/concurrency/worker.js 90e51ff3e781a6e08f265cbaaa993bb10eb0b798893b78cf4d8a8a8b3b561392 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 12dcd5425967b6306f3e3178babeac130962edb78e6298d545a429a1a20457cc -R 587bc730112047fe7757e9bed1469d73 +P a9aecc987512d60f2663973f43c769cf086fc14149edfbcb18c0aec9f5aa3dbf +R 825063f48a3ce6fa65e33b9852885bae U stephan -Z cdae14ec494ba9a4ab6ec30704fa3278 +Z a8fffcaa59d459a1613276d07e6e1f6d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3ea9463f59..a5dd212704 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9aecc987512d60f2663973f43c769cf086fc14149edfbcb18c0aec9f5aa3dbf +f2175f526c00cfe562e8f332eb197b5ef2c3d6be1fff2aab1566c2c533a293ac From 9be458e61134fc0fb3a442b7c028cc12fc3b96d9 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 7 Mar 2026 02:19:23 +0000 Subject: [PATCH 26/32] Cleanups and docs in the new opfs code structure. FossilOrigin-Name: 3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311 --- ext/wasm/api/opfs-common-shared.c-pp.js | 55 +++++++------------ ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 51 +++++++++++------ manifest | 14 ++--- manifest.uuid | 2 +- 4 files changed, 63 insertions(+), 59 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 17916426ef..c0b028160e 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -14,7 +14,10 @@ This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js. It creates a private/internal sqlite3.opfs namespace common to the two and used (only) by them and the test framework. It is not part of - the public API. + the public API. The library deletes sqlite3.opfs in its final + bootstrapping steps unless it's specifically told to keep them (for + testing purposes only) using an undocumented and unsupported + mechanism. */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; @@ -461,49 +464,36 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ? (+urlParams.get('opfs-verbose') || 2) : 1; options.sanityChecks ??= urlParams.has('opfs-sanity-check'); - if( true ){ - /* Doing this from one scope up does not work */ - opfsUtil.proxyUri = "sqlite3-opfs-async-proxy.js"; - if( sqlite3.scriptInfo?.sqlite3Dir ){ - opfsUtil.proxyUri = ( - sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri - ); - } - //sqlite3.config.error("proxyUri =",opfsUtil.proxyUri, sqlite3.scriptInfo); + opfsUtil.proxyUri ??= "sqlite3-opfs-async-proxy.js"; + if( sqlite3.scriptInfo?.sqlite3Dir ){ + /* Doing this from one scope up, outside of this function, does + not work. */ + opfsUtil.proxyUri = ( + sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri + ); } - options.proxyUri ??= opfsUtil.proxyUri; if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } - if( false ){ - /* This ends up with the same values for all Worker instances. */ - callee.counter ??= 0; - ++callee.counter; - options.workerId ??= urlParams.get('opfs-async-proxy-id') ?? callee.counter; - }else{ - options.workerId ??= (Math.random() * 10000000) | 0; - } //sqlite3.config.warn("opfsUtil options =",JSON.stringify(options), 'urlParams =', urlParams); return opfsUtil.options = options; }; /** - Creates and populates the main state object used by "opfs" and "opfs-wl", and - transfered from those to their async counterpart. - - Returns an object containing state which we send to the async - proxy Worker. + Creates, populates, and returns the main state object used by the + "opfs" and "opfs-wl" VFSes, and transfered from those to their + async counterparts. The returned object's vfs property holds the fully-populated - capi.sqlite3_vfs instance. + capi.sqlite3_vfs instance, tagged with lots of extra state which + the current VFSes need to have exposed to them. - After setting up any local state needed, the caller must - call theVfs.bindVfs(X,Y), where X is an object containing - the sqlite3_io_methods to override and Y is a callback which - gets triggered if init succeeds, before the final Promise - decides whether or not to reject. The result of bindVfs() - must be returned from their main installation function. + After setting up any local state needed, the caller must call + theVfs.bindVfs(X,Y), where X is an object containing the + sqlite3_io_methods to override and Y is a callback which gets + triggered if init succeeds, before the final Promise decides + whether or not to reject. This object must, when it's passed to the async part, contain only cloneable or sharable objects. After the worker's "inited" @@ -1105,9 +1095,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ let proxyUri = options.proxyUri +( (options.proxyUri.indexOf('?')<0) ? '?' : '&' )+'vfs='+vfsName; - if( options.workerId ){ - proxyUri += '&opfs-async-proxy-id='+encodeURIComponent(options.workerId); - } //sqlite3.config.error("proxyUri",options.proxyUri, (new Error())); const W = opfsVfs.worker = //#if target:es6-bundler-friendly diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 6362c6efd4..d09dd2b350 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -55,8 +55,12 @@ const vfsName = urlParams.get('vfs'); if( !vfsName ){ throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); } -const workerId = urlParams.get('opfs-async-proxy-id') - ?? 'unnamed'; +/** + We use this to allow us to differentiate debug output from + multiple instances, e.g. multiple Workers to the "opfs" + VFS or both the "opfs" and "opfs-wl" VFSes. +*/ +const workerId = (Math.random() * 10000000) | 0; const isWebLocker = true; //'opfs-wl'===urlParams.get('vfs'); const wPost = (type,...args)=>postMessage({type, payload:args}); const installAsyncProxy = function(){ @@ -287,10 +291,11 @@ const installAsyncProxy = function(){ In order to help alleviate cross-tab contention for a dabase, if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to some fixed number of - times. If acquisition still fails at that point it will give up - and propagate the exception. Client-level code will see that as - an I/O error. + will wait briefly and try again, up to `maxTries` of times. If + acquisition still fails at that point it will give up and + propagate the exception. Client-level code will see that either + as an I/O error or SQLITE_BUSY, depending on the exception and + the context. 2024-06-12: there is a rare race condition here which has been reported a single time: @@ -306,6 +311,12 @@ const installAsyncProxy = function(){ creating a viable test for that condition has proven challenging so far. + Interface quirk: if fh.xLock is falsy and the handle is acquired + then fh.fid is added to __implicitLocks(). If fh.xLock is truthy, + it is not added as an implicit lock. i.e. xLock() impls must set + fh.xLock immediately _before_ calling this and must arrange to + restore it to its previous value if this function throws. + 2026-03-06: - baseWaitTime is the number of milliseconds to wait for the @@ -314,12 +325,11 @@ const installAsyncProxy = function(){ - maxTries is the number of attempt to make, each one spaced out by one additional factor of the baseWaitTime (e.g. 300, then 600, - then 900, the 1200...). This MUST be an integer >0 and defaults - to 6. + then 900, the 1200...). This MUST be an integer >0. Only the Web Locks impl should use the 3rd and 4th parameters. */ - const getSyncHandle = async (fh,opName, baseWaitTime, maxTries)=>{ + const getSyncHandle = async (fh, opName, baseWaitTime, maxTries = 6)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); @@ -610,8 +620,15 @@ const installAsyncProxy = function(){ if( isWebLocker ){ /* We require separate xLock() and xUnlock() implementations for the original and Web Lock implementations. The ones in this block - are for the WebLock impl. */ + are for the WebLock impl. + The Golden Rule for this impl is: if we have a web lock, we + must also hold the SAH. When "upgrading" an implicit lock to a + requested (explicit) lock, we must remove the SAH from the + __implicitLocks set. When we unlock, we release both the web + lock and the SAH. That invariant must be kept intact or race + conditions on SAHs will ensue. + */ /** Registry of active Web Locks: fid -> { mode, resolveRelease } */ const __activeWebLocks = Object.create(null); @@ -632,8 +649,7 @@ const installAsyncProxy = function(){ storeAndNotify(whichOp, 0); /* Don't do this: existing.mode = requestedMode; - Paraphrased from advice given by a consultanting - developer: + Paraphrased from advice given by a consulting developer: If you hold an exclusive lock and SQLite requests shared, you should keep exiting.mode as exclusive in because the @@ -648,10 +664,11 @@ const installAsyncProxy = function(){ Upgrade path: we must release shared and acquire exclusive. This transition is NOT atomic in Web Locks API. - Except that it _effectively_ is atomic if we don't call - closeSyncHandle(fh), as no other worker can lock that - until we let it go. But we can't do that without leading - to a deadly embrace, so... + It _effectively_ is atomic if we don't call + closeSyncHandle(fh), as no other worker can lock that until + we let it go. But we can't do that without eventually + leading to deadly embrace situations, so we don't do that. + (That's not a hypothetical, it has happened.) */ await closeSyncHandle(fh); existing.resolveRelease(); @@ -664,10 +681,10 @@ const installAsyncProxy = function(){ //error("xLock() initial promise entered..."); navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => { //error("xLock() Web Lock entered.", fh); - fh.xLock = lockType/*must be set before getSyncHandle() is called!*/; __implicitLocks.delete(fid); let rc = 0; try{ + fh.xLock = lockType/*must be set before getSyncHandle() is called!*/; await getSyncHandle(fh, 'xLock', state.asyncIdleWaitTime, 5); }catch(e){ fh.xLock = oldLockType; diff --git a/manifest b/manifest index ecc54ab16c..c12974dd15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sbackwards\scompatibility,\sensure\sthat\sthe\s"opfs"\sVFS\sdoes\snot\sspecifically\srequire\sAtomics.waitAsync()\s(a\snew\srequirement\sof\s"opfs-wl"),\sbut\smake\suse\sof\sit\sif\savailable.\sOnly\sapply\sjitter\sto\sthe\sconcurrency\stest\sruns\sat\srandom\sintervals. -D 2026-03-07T01:01:13.846 +C Cleanups\sand\sdocs\sin\sthe\snew\sopfs\scode\sstructure. +D 2026-03-07T02:19:23.636 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 612df64a2a159db30786c742c70a09d75b993d034f360367125747d2eca531a5 +F ext/wasm/api/opfs-common-shared.c-pp.js 81a2429027b76df8691e6c134d3fedbf7cb804585ac28165b857ecb2260a99f3 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -594,7 +594,7 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 296b3d3701c3b42918357e4d9e5d82c5321b371115fb51a8a405c2eaeebd9f0f +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 5c84524885393b90b57d27446872fe980a48e8b9f5f2bb728e7ba63e905feb3f F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P a9aecc987512d60f2663973f43c769cf086fc14149edfbcb18c0aec9f5aa3dbf -R 825063f48a3ce6fa65e33b9852885bae +P f2175f526c00cfe562e8f332eb197b5ef2c3d6be1fff2aab1566c2c533a293ac +R 226e3a09fddd6452554266d606d70e1e U stephan -Z a8fffcaa59d459a1613276d07e6e1f6d +Z 2eb7dcbdcae5965bf20b0b5469f42aa6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a5dd212704..d013e9cb8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2175f526c00cfe562e8f332eb197b5ef2c3d6be1fff2aab1566c2c533a293ac +3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311 From f20c4e650dbde03c90d2f49366f14e2cbd7f4957 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 7 Mar 2026 03:29:26 +0000 Subject: [PATCH 27/32] Elide more of the 64-bit was stuff from the default build target. FossilOrigin-Name: 3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f --- ext/wasm/mkwasmbuilds.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 022a926397..1f60a89bb7 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -246,7 +246,7 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_ALL | F_64BIT + .flags = CP_ALL | F_64BIT | F_NOT_IN_ALL }, /* The canonical esm build. */ @@ -274,7 +274,7 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_JS | F_ESM | F_64BIT + .flags = CP_JS | F_ESM | F_64BIT | F_NOT_IN_ALL }, /* speedtest1, our primary benchmarking tool */ diff --git a/manifest b/manifest index c12974dd15..640a34ab1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cleanups\sand\sdocs\sin\sthe\snew\sopfs\scode\sstructure. -D 2026-03-07T02:19:23.636 +C Elide\smore\sof\sthe\s64-bit\swas\sstuff\sfrom\sthe\sdefault\sbuild\starget. +D 2026-03-07T03:29:26.758 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -628,7 +628,7 @@ F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x -F ext/wasm/mkwasmbuilds.c 5961177b9d6ad2b49efbdbb3076527e6886fecef7adf99d71cce58bfca89be08 +F ext/wasm/mkwasmbuilds.c d4479af2774a43104b819d9286961b7c09793ca39440e20191e693ceff21f911 F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P f2175f526c00cfe562e8f332eb197b5ef2c3d6be1fff2aab1566c2c533a293ac -R 226e3a09fddd6452554266d606d70e1e +P 3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311 +R fd6275af0aa6341e83c30c2e83b929c6 U stephan -Z 2eb7dcbdcae5965bf20b0b5469f42aa6 +Z 401ca0507277c8f3e4ba9832533ffae5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d013e9cb8f..cc74046192 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311 +3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f From 7d10030d75122cc26cf5e71b75b258835b3afd72 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 7 Mar 2026 03:32:17 +0000 Subject: [PATCH 28/32] Cleanups and docs in the opfs vfs and its concurrency tester. Experimentally add sqlite3.capi.sqlite3_js_retry_busy(), which runs a callback repeatedly until it _stops_ returning (or throwing) an SQLITE_BUSY error. FossilOrigin-Name: 45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea --- ext/wasm/api/opfs-common-shared.c-pp.js | 3 +- ext/wasm/api/sqlite3-api-prologue.js | 56 +++++++++++++++++++ ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js | 36 +++++++----- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 1 - ext/wasm/tests/opfs/concurrency/test.js | 8 ++- ext/wasm/tests/opfs/concurrency/worker.js | 49 ++++++++-------- manifest | 22 ++++---- manifest.uuid | 2 +- 8 files changed, 118 insertions(+), 59 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index c0b028160e..15eec1ff8d 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -107,6 +107,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return false; } }; + /** Checks whether the given OPFS filesystem entry exists, returning true if it does, false if it doesn't or if an @@ -461,7 +462,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ options = util.nu(options); options.vfsName = vfsName; options.verbose ??= urlParams.has('opfs-verbose') - ? (+urlParams.get('opfs-verbose') || 2) : 1; + ? +urlParams.get('opfs-verbose') : 1; options.sanityChecks ??= urlParams.has('opfs-sanity-check'); opfsUtil.proxyUri ??= "sqlite3-opfs-async-proxy.js"; diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 7bb31dc60b..d1055c1cfb 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1939,6 +1939,62 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( 'sqlite3changeset_old'); }/*changeset/preupdate additions*/ + /** + EXPERIMENTAL. For tentative addition in 3.53.0. + + sqlite3_js_retry_busy(maxTimes,callback[,beforeRetry]) + + Calls the given _synchronous_ callback function. If that function + returns sqlite3.capi.SQLITE_BUSY _or_ throws an SQLite3Error + which a resultCode property of that value then it will suppress + that error and try again, up to the given maximum number of + times. If the callback returns any other value than that, + it is returned. If the maximum number of retries has been + reached, an SQLite3Error with a resultCode value of + sqlite3.capi.SQLITE_BUSY is thrown. If the callback throws any + exception other than the aforementioned BUSY exception, it is + propagated. If it throws a BUSY exception on its final attempt, + that is propagated as well. + + If the beforeRetry argument is given, it must be a _synchronous_ + function. It is called immediately before each retry of the + callback (not for the initial call), passed the attempt number + (so it starts with 2, not 1). If it throws, the exception is + handled as described above. Its result value is ignored. + + To effectively retry "forever", pass a negative maxTimes value, + with the caveat that there is no recovery from that if the code + gets stuck in a deadlock situation. + + TODO: an async variant of this. + */ + capi.sqlite3_js_retry_busy = function(maxTimes, callback, beforeRetry){ + for(let n = 1; n <= maxTimes; ++n){ + try{ + if( beforeRetry && n>1 ) beforeRetry(n); + const rc = callback(); + if( capi.SQLITE_BUSY===rc ){ + if( n===maxTimes ){ + throw new SQLite3Error(rc, [ + "sqlite3_js_retry_busy() max retry attempts (", + maxTimes, + ") reached." + ].join('')); + } + continue; + } + return rc; + }catch(e){ + if( npostMessage({type, payload:args}); const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; @@ -101,7 +101,7 @@ const installAsyncProxy = function(){ 2:console.log.bind(console) }; const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]('opfs-async-proxy',workerId+":",...args); + if(state.verbose>level) loggers[level](vfsName+' async-proxy',workerId+":",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); @@ -563,7 +563,7 @@ const installAsyncProxy = function(){ rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ - error("xRead() failed",e,fh); + //error("xRead() failed",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } @@ -590,7 +590,7 @@ const installAsyncProxy = function(){ affirmNotRO('xTruncate', fh); await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ - error("xTruncate():",e,fh); + //error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } @@ -608,7 +608,7 @@ const installAsyncProxy = function(){ {at: Number(offset64)}) ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; }catch(e){ - error("xWrite():",e,fh); + //error("xWrite():",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } @@ -811,6 +811,11 @@ const installAsyncProxy = function(){ const slotWhichOp = opIds.whichOp; const idleWaitTime = state.asyncIdleWaitTime; const hasWaitAsync = !!Atomics.waitAsync; +//#if nope + error("waitLoop init: isWebLocker",isWebLocker, + "idleWaitTime",idleWaitTime, + "hasWaitAsync",hasWaitAsync); +//#endif while(!flagAsyncShutdown){ try { let opId; @@ -834,17 +839,14 @@ const installAsyncProxy = function(){ Safari: 16.4 (2023-03-27) The "opfs" VFS was not born until Chrome was somewhere in - the v104-108 range and did not work with Safari < v17 - (2023-09-18) due to a WebKit bug which restricted OPFS - access from sub-Workers. + the v104-108 range (Summer/Autumn 2022) and did not work + with Safari < v17 (2023-09-18) due to a WebKit bug which + restricted OPFS access from sub-Workers. - The waitAsync() counterpart of this block has shown to be - slightly more performant for the "opfs" VFS than this - block (whereas "opfs-wl" _requires_ that block), so we - enable it where it's available, regardless of the value - of isWebLocker, with the note that if isWebLocker is true - then the bootstrapping of this script will have failed if - waitAsync() is not available. + The waitAsync() counterpart of this block can be used by + both "opfs" and "opfs-wl", whereas this block can only be + used by "opfs". Performance comparisons between the two + in high-contention tests have been indecisive. */ if('not-equal'!==Atomics.wait( state.sabOPView, slotWhichOp, 0, state.asyncIdleWaitTime @@ -863,6 +865,9 @@ const installAsyncProxy = function(){ index is a few lines down from here, and that instance is required in order for clear communication between the sync half of this proxy and this half. + + Much later (2026-03-07): that phenomenon is apparently + called a spurious wakeup. */ await releaseImplicitLocks(); continue; @@ -913,6 +918,7 @@ const installAsyncProxy = function(){ } }); initS11n(); + //warn("verbosity =",opt.verbose, state.verbose); log("init state",state); wPost('opfs-async-inited'); waitLoop(); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index c1d34f192d..349819b0e1 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -59,7 +59,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const installOpfsWlVfs = async function(options){ options = opfsUtil.initOptions('opfs-wl',options); if( !options ) return sqlite3; - options.verbose = 2; const capi = sqlite3.capi, state = opfsUtil.createVfsState(), opfsVfs = state.vfs, diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index f8d7f10399..a73bc2e47d 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -8,6 +8,7 @@ return ''+v; default: break; } + if( v instanceof Date ) return v.toString(); if(null===v) return 'null'; if(v instanceof Error){ v = { @@ -84,11 +85,12 @@ if(workers.counts.passed + workers.counts.failed !== workers.length){ return; } - stdout("Total Worker run time:",calcTime(new Date()),"ms"); if(workers.counts.failed>0){ - logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s) in", + calcTime(new Date()),"ms"); }else{ - logCss('tests-pass',"All",workers.length,"workers finished."); + logCss('tests-pass',"All",workers.length,"workers finished in", + calcTime(new Date()),"ms"); } }; diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index ccd1df932a..a698fc96f7 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -63,29 +63,29 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ } while(true){ try{ - if( !db ){ - db = new ctor({ - filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c' - }); - sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); - } - db.transaction((db)=>{ - db.exec([ - "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", - "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" - ]); + db = new ctor({ + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, + flags: 'c' }); break; }catch(e){ if(e instanceof sqlite3.SQLite3Error && sqlite3.capi.SQLITE_BUSY===e.resultCode){ - stderr("Retrying",(db ? "db init" : "open()"),"for BUSY:",e.message); + stderr("Retrying open() for BUSY:",e.message); continue; } throw e; } } + sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); + sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{ + db.transaction((db)=>{ + db.exec([ + "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", + "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" + ]); + }); + }); const maxIterations = urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; @@ -95,24 +95,19 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ ++interval.count; const prefix = "v(#"+interval.count+")"; stdout("Setting",prefix,"=",tm); - while(true){ - try{ + try{ + sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{ db.exec({ sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", bind: [options.workerName, new Date().getTime()] }); - //stdout("Set",prefix); - break; - }catch(e){ - if(e instanceof sqlite3.SQLite3Error - && sqlite3.capi.SQLITE_BUSY===e.resultCode){ - stderr("Retrying interval #",interval.count,"for BUSY:",e.message); - continue; - } - stderr("Error: ",e.message); - interval.error = e; - throw e; - } + }, (n)=>{ + stderr("Attempt #",n,"for interval #",interval.count,"due to BUSY"); + }); + }catch(e){ + stderr("Error: ",e.message); + interval.error = e; + throw e; } //stdout("doWork()",prefix,"error ",interval.error); }; diff --git a/manifest b/manifest index 640a34ab1e..44ee1e7107 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Elide\smore\sof\sthe\s64-bit\swas\sstuff\sfrom\sthe\sdefault\sbuild\starget. -D 2026-03-07T03:29:26.758 +C Cleanups\sand\sdocs\sin\sthe\sopfs\svfs\sand\sits\sconcurrency\stester.\sExperimentally\sadd\ssqlite3.capi.sqlite3_js_retry_busy(),\swhich\sruns\sa\scallback\srepeatedly\suntil\sit\s_stops_\sreturning\s(or\sthrowing)\san\sSQLITE_BUSY\serror. +D 2026-03-07T03:32:17.361 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,20 +585,20 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 81a2429027b76df8691e6c134d3fedbf7cb804585ac28165b857ecb2260a99f3 +F ext/wasm/api/opfs-common-shared.c-pp.js d58f16b8bf00401d954249d154c91abc4d4f54221f755d774cf5c4e8f83a1a9f F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa -F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 +F ext/wasm/api/sqlite3-api-prologue.js c616bff3bdff1fcbb6a39bd429a96de9c3e5f1e8c0d287083cea5e98e8b0151c F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 5c84524885393b90b57d27446872fe980a48e8b9f5f2bb728e7ba63e905feb3f +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js c2236c9037ccac368be6eeed3e5f6ed3cefed0d20539c9092139f27a76aaf1ba F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js f65903e15f49a5475eaf8fd3f35971d4c4c4b43f0a00167f1a2c98389cf84900 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js c518aee3c2f537ae73aa3afd67463d1eb63db0559a41d64fe4d1361b9966b52f F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 872c29aa760b05c2703185368a39b4b2e1a8abf15c29ee7e23ee8ca7cdcbed9d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -646,8 +646,8 @@ F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c9 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb F ext/wasm/tester1.c-pp.js a4e79fbf63bb3255d2b8ffc1cd538c115d2f6b599bc324904c80f6644379a284 F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226 -F ext/wasm/tests/opfs/concurrency/test.js 520061e6a624fe0a7c07cf3438cb6b9b7c9dcbd606e644ccb5fb52518c48b342 -F ext/wasm/tests/opfs/concurrency/worker.js 90e51ff3e781a6e08f265cbaaa993bb10eb0b798893b78cf4d8a8a8b3b561392 +F ext/wasm/tests/opfs/concurrency/test.js 46c772bc18abb0fcbb058d57b5aaee9e7938f948ecdd802c6ca0850ad3519f92 +F ext/wasm/tests/opfs/concurrency/worker.js fc985ec86b70b057224e8caaa9c5a11892ea9b980a5c5da21b1efdf5f12bc3f6 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311 -R fd6275af0aa6341e83c30c2e83b929c6 +P 3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f +R 5251eca545de3a2827b98f8a138c1479 U stephan -Z 401ca0507277c8f3e4ba9832533ffae5 +Z e640850ae89db803157694e7fdf4c165 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cc74046192..d297924b93 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f +45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea From 78649bb4215fe180cd78beae3b137bdfbbe24388 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 7 Mar 2026 04:19:08 +0000 Subject: [PATCH 29/32] Fix the mangling of the opfs async proxy script's URI caused by both OPFS VFSes modifying it to accound for sqlite3.dir. Add a link to speedtest1 with opfw-wl to index.html. FossilOrigin-Name: c1cb61f473a11a325ef421ba8edfb20257203688565f7db62309044be183af5f --- ext/wasm/api/opfs-common-shared.c-pp.js | 16 +++++++++------- ext/wasm/index.html | 5 +++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 15eec1ff8d..ae750e75d3 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -465,13 +465,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ? +urlParams.get('opfs-verbose') : 1; options.sanityChecks ??= urlParams.has('opfs-sanity-check'); - opfsUtil.proxyUri ??= "sqlite3-opfs-async-proxy.js"; - if( sqlite3.scriptInfo?.sqlite3Dir ){ - /* Doing this from one scope up, outside of this function, does - not work. */ - opfsUtil.proxyUri = ( - sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri - ); + if( !opfsUtil.proxyUri ){ + opfsUtil.proxyUri = "sqlite3-opfs-async-proxy.js"; + if( sqlite3.scriptInfo?.sqlite3Dir ){ + /* Doing this from one scope up, outside of this function, does + not work. */ + opfsUtil.proxyUri = ( + sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri + ); + } } options.proxyUri ??= opfsUtil.proxyUri; if('function' === typeof options.proxyUri){ diff --git a/ext/wasm/index.html b/ext/wasm/index.html index fd65dbc339..6f22dc150b 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -115,6 +115,11 @@ 64-bit): speedtest1-worker with the OPFS VFS preselected and configured for a moderate workload. +
  • speedtest1-worker?vfs=opfs-wl + (32-bit, + 64-bit): + speedtest1-worker with the + OPFS Web Locks VFS preselected and configured for a moderate workload.
  • speedtest1-worker?vfs=opfs-sahpool (32-bit, 64-bit): diff --git a/manifest b/manifest index 44ee1e7107..313bebc506 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cleanups\sand\sdocs\sin\sthe\sopfs\svfs\sand\sits\sconcurrency\stester.\sExperimentally\sadd\ssqlite3.capi.sqlite3_js_retry_busy(),\swhich\sruns\sa\scallback\srepeatedly\suntil\sit\s_stops_\sreturning\s(or\sthrowing)\san\sSQLITE_BUSY\serror. -D 2026-03-07T03:32:17.361 +C Fix\sthe\smangling\sof\sthe\sopfs\sasync\sproxy\sscript's\sURI\scaused\sby\sboth\sOPFS\sVFSes\smodifying\sit\sto\saccound\sfor\ssqlite3.dir.\sAdd\sa\slink\sto\sspeedtest1\swith\sopfw-wl\sto\sindex.html. +D 2026-03-07T04:19:08.919 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js d58f16b8bf00401d954249d154c91abc4d4f54221f755d774cf5c4e8f83a1a9f +F ext/wasm/api/opfs-common-shared.c-pp.js 7ff4b6cfd486a9fff1781af9c1fe2e3f328fd2a6b3a2d7dd3d5f83979e2ca552 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -624,7 +624,7 @@ F ext/wasm/fiddle/fiddle-worker.js 6c72acac2d381480bc9f5eb538e3f2faf2c1f72dd4fcb F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893 F ext/wasm/fiddle/index.c-pp.html 72c7e5517217960b3809648429ea396a7cbad0ffb2c92f6a2f5703abecb27317 F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290 -F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb5f1756 +F ext/wasm/index.html 5bf6cf1b0a3c8b9f5f54d77f2219d7ae87a15162055ce308109c49b1dcab4239 F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f -R 5251eca545de3a2827b98f8a138c1479 +P 45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea +R a0b14fb61a3f361676cb6a84638efe48 U stephan -Z e640850ae89db803157694e7fdf4c165 +Z 3353e988da2dca384c44895dc97fbee8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d297924b93..988c55dc18 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea +c1cb61f473a11a325ef421ba8edfb20257203688565f7db62309044be183af5f From 237fce8205a8350dc933aa4d3a283a97f3ada829 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 7 Mar 2026 06:10:29 +0000 Subject: [PATCH 30/32] Extend the JS pre-bootstrapping configuration to include an option to disable inclusion of any given extension VFS (not the default VFSes, like memdb). The primary motivation for this is to give people who don't use OPFS to a way to keep the OPFS VFSes from loading their proxy workers, which are expensive. FossilOrigin-Name: 9a07eb7a941479510891d1444aacbeb440efaad3e9a13f186833618d8b60a8da --- ext/wasm/api/opfs-common-shared.c-pp.js | 4 +++ ext/wasm/api/sqlite3-api-prologue.js | 32 ++++++++++++------- ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js | 4 +++ ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 4 +++ ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 15 +++++---- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 3 ++ ext/wasm/tester1.c-pp.js | 28 ++++++++-------- manifest | 24 +++++++------- manifest.uuid | 2 +- 9 files changed, 70 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index ae750e75d3..11acf40bdc 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -21,6 +21,10 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; + if( sqlite3.config.disable?.vfs?.opfs && + sqlite3.config.disable.vfs['opfs-vfs'] ){ + return; + } const toss = sqlite3.util.toss, capi = sqlite3.capi, util = sqlite3.util, diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index d1055c1cfb..e71b6c9570 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -99,6 +99,13 @@ used in WASMFS-capable builds of the library (which the canonical builds do not include). + - `disable` (as of 3.53.0) may be an object with the following + properties: + - `vfs`, an object, may contain a map of VFS names to booleans. + Any mapping to falsy are disabled. The supported names + are: "kvvfs", "opfs", "opfs-sahpool", "opfs-wl". + - Other disabling options may be added in the future. + [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, enabling delayed evaluation. @@ -145,7 +152,8 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( ); return sqlite3ApiBootstrap.sqlite3; } - const config = Object.assign(Object.create(null),{ + const nu = (...obj)=>Object.assign(Object.create(null),...obj); + const config = nu({ exports: undefined, memory: undefined, bigIntEnabled: !!globalThis.BigInt64Array, @@ -162,7 +170,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( certain wasm.xWrap.resultAdapter()s. */ useStdAlloc: false - }, apiConfig || {}); + }, apiConfig); Object.assign(config, { allocExportName: config.useStdAlloc ? 'malloc' : 'sqlite3_malloc', @@ -195,7 +203,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( not documented are installed as 1-to-1 proxies for their C-side counterparts. */ - const capi = Object.create(null); + const capi = nu(); /** Holds state which are specific to the WASM-related infrastructure and glue code. @@ -204,7 +212,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( dynamically after the api object is fully constructed, so not all are documented in this file. */ - const wasm = Object.create(null); + const wasm = nu(); /** Internal helper for SQLite3Error ctor. */ const __rcStr = (rc)=>{ @@ -752,7 +760,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( toss: function(...args){throw new Error(args.join(' '))}, toss3, typedArrayPart: wasm.typedArrayPart, - nu: (...obj)=>Object.assign(Object.create(null),...obj), + nu, assert: function(arg,msg){ if( !arg ){ util.toss("Assertion failed:",msg); @@ -1009,7 +1017,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; }; } - const rc = Object.create(null), ov = [0,0]; + const rc = nu(), ov = [0,0]; let i = 0, k; while((k = capi.sqlite3_compileoption_get(i++))){ f._opt(k,ov); @@ -1017,7 +1025,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } return f._result = rc; }else if(Array.isArray(optName)){ - const rc = Object.create(null); + const rc = nu(); optName.forEach((v)=>{ rc[v] = capi.sqlite3_compileoption_used(v); }); @@ -1068,7 +1076,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( The memory lives in the WASM heap and can be used with routines such as wasm.poke() and wasm.heap8u().slice(). */ - wasm.pstack = Object.assign(Object.create(null),{ + wasm.pstack = nu({ /** Sets the current pstack position to the given pointer. Results are undefined if the passed-in value did not come from @@ -1290,7 +1298,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( // sqlite3__wasm_init_wasmfs() is not available return this.dir = ""; } - }.bind(Object.create(null)); + }.bind(nu()); /** Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a @@ -1665,7 +1673,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( default: return capi.SQLITE_MISUSE; } - }.bind(Object.create(null)); + }.bind(nu()); /** Given a (sqlite3_value*), this function attempts to convert it @@ -1899,7 +1907,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); const pv = wasm.peekPtr(this.ptr); return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; - }.bind(Object.create(null)); + }.bind(nu()); /** A wrapper around sqlite3_preupdate_new() which fetches the @@ -2013,7 +2021,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( This object is initially a placeholder which gets replaced by a build-generated object. */ - version: Object.create(null), + version: nu(), /** The library reserves the 'client' property for client-side use diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index 0fdd7cb4cc..88f81226a6 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -28,6 +28,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //#@policy error //#savepoint begin //#define kvvfs-v2-added-in=3.52.0 + /** kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates storage of its pages and metadata to a key-value store. @@ -85,6 +86,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ backups. */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( sqlite3.config.disable?.vfs?.kvvfs ){ + return; + } 'use strict'; const capi = sqlite3.capi, sqlite3_kvvfs_methods = capi.sqlite3_kvvfs_methods, diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index c1fee5e1ab..689f53d4a7 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -55,6 +55,10 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; + if( sqlite3.config.disable?.vfs?.['opfs-sahpool'] ){ + return; + } + const toss = sqlite3.util.toss; const toss3 = sqlite3.util.toss3; const initPromises = Object.create(null) /* cache of (name:result) of VFS init results */; diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 349819b0e1..dbd786410a 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -34,10 +34,13 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.['opfs-wl'] ){ + return; + } const util = sqlite3.util, toss = sqlite3.util.toss; - const opfsUtil = sqlite3.opfs || toss("Missing sqlite3.opfs"); - + const opfsUtil = sqlite3.opfs; + const vfsName = 'opfs-wl'; /** installOpfsWlVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs @@ -57,7 +60,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3-vfs-opfs.c-pp.js:installOpfsVfs(). */ const installOpfsWlVfs = async function(options){ - options = opfsUtil.initOptions('opfs-wl',options); + options = opfsUtil.initOptions(vfsName,options); if( !options ) return sqlite3; const capi = sqlite3.capi, state = opfsUtil.createVfsState(), @@ -66,8 +69,8 @@ const installOpfsWlVfs = async function(options){ mTimeStart = opfsVfs.mTimeStart, mTimeEnd = opfsVfs.mTimeEnd, opRun = opfsVfs.opRun, - debug = (...args)=>sqlite3.config.debug("opfs-wl:",...args), - warn = (...args)=>sqlite3.config.warn("opfs-wl:",...args), + debug = (...args)=>sqlite3.config.debug(vfsName+":",...args), + warn = (...args)=>sqlite3.config.warn(vfsName+":",...args), __openFiles = opfsVfs.__openFiles; //debug("state",JSON.stringify(options)); @@ -123,7 +126,7 @@ const installOpfsWlVfs = async function(options){ }/*installOpfsWlVfs()*/; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ return installOpfsWlVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install the 'opfs-wl' sqlite3_vfs:",e); + sqlite3.config.warn("Ignoring inability to install the",vfsName,"sqlite3_vfs:",e); }); }); }/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 05e6a3f09f..53619bfc92 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -20,6 +20,9 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.opfs ){ + return; + } const util = sqlite3.util, opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs"); /** diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 8c0ba19165..6ed66d7853 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -66,15 +66,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const haveWasmCTests = ()=>{ return !!wasm.exports.sqlite3__wasm_test_intptr; }; - const hasOpfs = ()=>{ - return globalThis.FileSystemHandle - && globalThis.FileSystemDirectoryHandle - && globalThis.FileSystemFileHandle - && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle - && navigator?.storage?.getDirectory; - }; let SQLite3 /* populated after module load */; + const hasOpfs = ()=>!!SQLite3?.oo1?.OpfsDb; { const mapToString = (v)=>{ @@ -3773,8 +3767,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('OPFS: Origin-Private File System', - (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs") - || 'requires "opfs" VFS')) + (sqlite3)=>(hasOpfs() || 'requires "opfs" VFS')) .t({ name: 'OPFS db sanity checks', test: async function(sqlite3){ @@ -3914,7 +3907,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('OPFS SyncAccessHandle Pool VFS', - (sqlite3)=>(hasOpfs() || "requires OPFS APIs")) + (sqlite3)=>(!!sqlite3.installOpfsSAHPoolVfs || "requires OPFS SAH Pool APIs")) .t({ name: 'SAH sanity checks', test: async function(sqlite3){ @@ -4430,12 +4423,17 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// log("Loading and initializing sqlite3 WASM module..."); - if(0){ + if(1){ globalThis.sqlite3ApiConfig = { - debug: ()=>{}, - log: ()=>{}, - warn: ()=>{}, - error: ()=>{} + //debug: ()=>{}, log: ()=>{}, warn: ()=>{}, error: ()=>{}, + disable: { + vfs: { + kvvfs: false, + opfs: false, + "opfs-sahpool": false, + "opfs-wl": false + } + } } } //#if not target:es6-module diff --git a/manifest b/manifest index 313bebc506..d2f8ed2331 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\smangling\sof\sthe\sopfs\sasync\sproxy\sscript's\sURI\scaused\sby\sboth\sOPFS\sVFSes\smodifying\sit\sto\saccound\sfor\ssqlite3.dir.\sAdd\sa\slink\sto\sspeedtest1\swith\sopfw-wl\sto\sindex.html. -D 2026-03-07T04:19:08.919 +C Extend\sthe\sJS\spre-bootstrapping\sconfiguration\sto\sinclude\san\soption\sto\sdisable\sinclusion\sof\sany\sgiven\sextension\sVFS\s(not\sthe\sdefault\sVFSes,\slike\smemdb).\sThe\sprimary\smotivation\sfor\sthis\sis\sto\sgive\speople\swho\sdon't\suse\sOPFS\sto\sa\sway\sto\skeep\sthe\sOPFS\sVFSes\sfrom\sloading\stheir\sproxy\sworkers,\swhich\sare\sexpensive. +D 2026-03-07T06:10:29.452 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,21 +585,21 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 7ff4b6cfd486a9fff1781af9c1fe2e3f328fd2a6b3a2d7dd3d5f83979e2ca552 +F ext/wasm/api/opfs-common-shared.c-pp.js 49f4b044dfb3becd8c926ef42b7494da680e5ac878c6d10ef510b66f74ad55a1 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa -F ext/wasm/api/sqlite3-api-prologue.js c616bff3bdff1fcbb6a39bd429a96de9c3e5f1e8c0d287083cea5e98e8b0151c +F ext/wasm/api/sqlite3-api-prologue.js df30ccb31affb2bbe9a2de63d6276c7ec1192ec732e79a0138449a19ec04b988 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js c2236c9037ccac368be6eeed3e5f6ed3cefed0d20539c9092139f27a76aaf1ba F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js c518aee3c2f537ae73aa3afd67463d1eb63db0559a41d64fe4d1361b9966b52f -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 872c29aa760b05c2703185368a39b4b2e1a8abf15c29ee7e23ee8ca7cdcbed9d +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js f907fa712d1c9d98a3c08393f3e97a664857782da11670e129d39e383f45f968 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js cf5fba74978d3e0983f68505d9695ee7836853855590a92ccc5a96e27db5350b +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 743650cfefed0df900171971065c2d618e5652177a98844ddb6695c70149968e +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4b02d7063d00e2d4f42d4ca79a8a0bca1b275257e135bc154d35b701ed1d234a F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -644,7 +644,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js a4e79fbf63bb3255d2b8ffc1cd538c115d2f6b599bc324904c80f6644379a284 +F ext/wasm/tester1.c-pp.js 31cde3c4db4b955d5554c21cb0aa6ec3a0743b03d7c5262a38bfa303e7955942 F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226 F ext/wasm/tests/opfs/concurrency/test.js 46c772bc18abb0fcbb058d57b5aaee9e7938f948ecdd802c6ca0850ad3519f92 F ext/wasm/tests/opfs/concurrency/worker.js fc985ec86b70b057224e8caaa9c5a11892ea9b980a5c5da21b1efdf5f12bc3f6 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea -R a0b14fb61a3f361676cb6a84638efe48 +P c1cb61f473a11a325ef421ba8edfb20257203688565f7db62309044be183af5f +R bd137899af2cd4094f0a4f5a1f4094af U stephan -Z 3353e988da2dca384c44895dc97fbee8 +Z 147d7dd643bc15130deceeb0e6d52d6f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 988c55dc18..06ded776dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1cb61f473a11a325ef421ba8edfb20257203688565f7db62309044be183af5f +9a07eb7a941479510891d1444aacbeb440efaad3e9a13f186833618d8b60a8da From 03b0f8fc9ceb46c3745d5ffadca089a78a8612c4 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 8 Mar 2026 07:58:02 +0000 Subject: [PATCH 31/32] Refactor tester1 OPFS test to support both the "opfs" and "opfs-wl" VFSes. FossilOrigin-Name: 17ac8d7bdfaf96510abea8c11019045c1703b7040655d9c4c8d7001636260b50 --- ext/wasm/tester1.c-pp.js | 288 +++++++++++++++++++++++---------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 179 insertions(+), 123 deletions(-) diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 6ed66d7853..81bc37e26d 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -451,9 +451,131 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert( !initDb ); tryKey('hexkey', hexFoo, 6); dbUnlink(); - }, + }; //#endif enable-see + /* Tests common to "opfs" and "opfs-wl". These tests manipulate + "this" and must run in order. + */ + T.opfsCommon = { + sanityChecks: async function(vfsName, oo1Ctor, sqlite3){ + T.assert(capi.sqlite3_vfs_find(vfsName)); + const opfs = sqlite3.opfs; + const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db'; + const fileUri = 'file://'+filename+'?delete-before-open=1'; + const initSql = [ + 'create table p(a);', + 'insert into p(a) values(1),(2),(3)' + ]; + let db = new oo1Ctor(fileUri); + try { + db.exec(initSql); + T.assert(3 === db.selectValue('select count(*) from p')); + db.close(); + db = new oo1Ctor(filename); + db.exec('insert into p(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from p')); + this.opfsDbExport = capi.sqlite3_js_db_export(db); + T.assert(this.opfsDbExport instanceof Uint8Array) + .assert(this.opfsDbExport.byteLength>0 + && 0===this.opfsDbExport.byteLength % 512); + }finally{ + db.close(); + db = null; + } + T.assert(await opfs.entryExists(filename)); + try { + db = new oo1Ctor(fileUri); + db.exec(initSql) /* will throw if delete-before-open did not work */; + T.assert(3 === db.selectValue('select count(*) from p')); + }finally{ + if(db) db.close(); + } + }, + + importer: async function(vfsName, oo1Ctor, sqlite3){ + let db; + const filename = this.opfsDbFile; + try { + const exp = this.opfsDbExport; + delete this.opfsDbExport; + this.opfsImportSize = await oo1Ctor.importDb(filename, exp); + db = new oo1Ctor(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')). + assert( this.opfsImportSize == exp.byteLength ); + db.close(); + db = null; + this.opfsUnlink = + (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink(vfsName, fn); + this.opfsUnlink(filename); + T.assert(!(await sqlite3.opfs.entryExists(filename))); + // Try again with a function as an input source: + let cursor = 0; + const blockSize = 512, end = exp.byteLength; + const reader = async function(){ + if(cursor >= exp.byteLength){ + return undefined; + } + const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize); + cursor += blockSize; + return rv; + }; + this.opfsImportSize = await oo1Ctor.importDb(filename, reader); + db = new oo1Ctor(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')). + assert( this.opfsImportSize == exp.byteLength ); + }finally{ + if(db) db.close(); + } + }, + + opfsUtil: async function(vfsName, oo1Ctor, sqlite3){ + const filename = this.opfsDbFile; + const unlink = this.opfsUnlink; + T.assert(filename && !!unlink); + delete this.opfsDbFile; + delete this.opfsUnlink; + /************************************************************** + ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended + for client-side use. It is only for this project's own + internal use. Its APIs are subject to change or removal at + any time. The sqlite3.opfs namespace is REMOVED from the + sqlite3 namespace in non-test runs of the library. + ***************************************************************/ + const opfs = sqlite3.opfs; + const fSize = this.opfsImportSize; + delete this.opfsImportSize; + let sh; + try{ + T.assert(await opfs.entryExists(filename)); + const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false); + const fh = await dirHandle.getFileHandle(filenamePart); + sh = await fh.createSyncAccessHandle(); + T.assert(fSize === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + T.assert(!(await opfs.entryExists(filename))); + }finally{ + if(sh) await sh.close(); + unlink(); + } + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } + }/*T.opfsCommon*/; + //////////////////////////////////////////////////////////////////////// // End of infrastructure setup. Now define the tests... //////////////////////////////////////////////////////////////////////// @@ -3767,130 +3889,29 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('OPFS: Origin-Private File System', - (sqlite3)=>(hasOpfs() || 'requires "opfs" VFS')) + (sqlite3)=>(!!sqlite3.oo1.OpfsDb || 'requires "opfs" VFS')) .t({ name: 'OPFS db sanity checks', - test: async function(sqlite3){ - T.assert(capi.sqlite3_vfs_find('opfs')); - const opfs = sqlite3.opfs; - const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db'; - const fileUri = 'file://'+filename+'?delete-before-open=1'; - const initSql = [ - 'create table p(a);', - 'insert into p(a) values(1),(2),(3)' - ]; - let db = new sqlite3.oo1.OpfsDb(fileUri); - try { - db.exec(initSql); - T.assert(3 === db.selectValue('select count(*) from p')); - db.close(); - db = new sqlite3.oo1.OpfsDb(filename); - db.exec('insert into p(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from p')); - this.opfsDbExport = capi.sqlite3_js_db_export(db); - T.assert(this.opfsDbExport instanceof Uint8Array) - .assert(this.opfsDbExport.byteLength>0 - && 0===this.opfsDbExport.byteLength % 512); - }finally{ - db.close(); - } - T.assert(await opfs.entryExists(filename)); - try { - db = new sqlite3.oo1.OpfsDb(fileUri); - db.exec(initSql) /* will throw if delete-before-open did not work */; - T.assert(3 === db.selectValue('select count(*) from p')); - }finally{ - if(db) db.close(); - } + test: async (sqlite3)=>{ + await T.opfsCommon.sanityChecks('opfs', sqlite3.oo1.OpfsDb, sqlite3); } - }/*OPFS db sanity checks*/) + }) .t({ - name: 'OPFS import', - test: async function(sqlite3){ - let db; - const filename = this.opfsDbFile; - try { - const exp = this.opfsDbExport; - delete this.opfsDbExport; - this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp); - db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); - T.assert(6 === db.selectValue('select count(*) from p')). - assert( this.opfsImportSize == exp.byteLength ); - db.close(); - this.opfsUnlink = - (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink("opfs",fn); - this.opfsUnlink(filename); - T.assert(!(await sqlite3.opfs.entryExists(filename))); - // Try again with a function as an input source: - let cursor = 0; - const blockSize = 512, end = exp.byteLength; - const reader = async function(){ - if(cursor >= exp.byteLength){ - return undefined; - } - const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize); - cursor += blockSize; - return rv; - }; - this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader); - db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); - T.assert(6 === db.selectValue('select count(*) from p')). - assert( this.opfsImportSize == exp.byteLength ); - }finally{ - if(db) db.close(); - } + name: 'OPFS import/export', + test: async (sqlite3)=>{ + await T.opfsCommon.importer('opfs', sqlite3.oo1.OpfsDb, sqlite3); } - }/*OPFS export/import*/) + }) .t({ name: '(Internal-use) OPFS utility APIs', - test: async function(sqlite3){ - const filename = this.opfsDbFile; - const unlink = this.opfsUnlink; - T.assert(filename && !!unlink); - delete this.opfsDbFile; - delete this.opfsUnlink; - /************************************************************** - ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended - for client-side use. It is only for this project's own - internal use. Its APIs are subject to change or removal at - any time. - ***************************************************************/ - const opfs = sqlite3.opfs; - const fSize = this.opfsImportSize; - delete this.opfsImportSize; - let sh; - try{ - T.assert(await opfs.entryExists(filename)); - const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false); - const fh = await dirHandle.getFileHandle(filenamePart); - sh = await fh.createSyncAccessHandle(); - T.assert(fSize === await sh.getSize()); - await sh.close(); - sh = undefined; - unlink(); - T.assert(!(await opfs.entryExists(filename))); - }finally{ - if(sh) await sh.close(); - unlink(); - } - - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); + test: async (sqlite3)=>{ + await T.opfsCommon.opfsUtil("opfs", sqlite3.oo1.OpfsDb, sqlite3); } - }/*OPFS util sanity checks*/) + }) //#if enable-see .t({ name: 'OPFS with SEE encryption', + predicate: ()=>!!sqlite3.oo1.OpfsDb, test: function(sqlite3){ T.seeBaseCheck( sqlite3.oo1.OpfsDb, @@ -3899,9 +3920,44 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if( isInit ) opt.filename += '?delete-before-open=1'; return opt; }, - ()=>{}); + ()=>{} + ); } - })/*OPFS with SEE*/ + }) +//#endif enable-see + ;/* end OPFS tests */ + + //////////////////////////////////////////////////////////////////////// + T.g('OPFS-WL: Origin-Private File System with Web Locks', + (sqlite3)=>(!!sqlite3.oo1.OpfsWlDb || 'requires "opfs-wl" VFS')) + .t({ + name: 'OPFS-WL db sanity checks', + test: async (sqlite3)=>{ + await T.opfsCommon.sanityChecks('opfs-wl', sqlite3.oo1.OpfsWlDb, sqlite3); + } + }) + .t({ + name: 'OPFS-WL import/export', + test: async (sqlite3)=>{ + await T.opfsCommon.importer('opfs-wl', sqlite3.oo1.OpfsWlDb, sqlite3); + } + }) +//#if enable-see + .t({ + name: 'OPFS-WL with SEE encryption', + predicate: ()=>!!sqlite3.oo1.OpfsWlDb, + test: function(sqlite3){ + T.seeBaseCheck( + sqlite3.oo1.OpfsWlDb, + function(isInit){ + const opt = {filename: 'file:///sqlite3-see.edb'}; + if( isInit ) opt.filename += '?delete-before-open=1'; + return opt; + }, + ()=>{} + ); + } + }) //#endif enable-see ;/* end OPFS tests */ diff --git a/manifest b/manifest index d2f8ed2331..852b2ca2e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extend\sthe\sJS\spre-bootstrapping\sconfiguration\sto\sinclude\san\soption\sto\sdisable\sinclusion\sof\sany\sgiven\sextension\sVFS\s(not\sthe\sdefault\sVFSes,\slike\smemdb).\sThe\sprimary\smotivation\sfor\sthis\sis\sto\sgive\speople\swho\sdon't\suse\sOPFS\sto\sa\sway\sto\skeep\sthe\sOPFS\sVFSes\sfrom\sloading\stheir\sproxy\sworkers,\swhich\sare\sexpensive. -D 2026-03-07T06:10:29.452 +C Refactor\stester1\sOPFS\stest\sto\ssupport\sboth\sthe\s"opfs"\sand\s"opfs-wl"\sVFSes. +D 2026-03-08T07:58:02.639 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -644,7 +644,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js 31cde3c4db4b955d5554c21cb0aa6ec3a0743b03d7c5262a38bfa303e7955942 +F ext/wasm/tester1.c-pp.js bccc66cdb09477ef09bf7c871bdba85f2c8b936a32d5dc07dc8c882517f73b50 F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226 F ext/wasm/tests/opfs/concurrency/test.js 46c772bc18abb0fcbb058d57b5aaee9e7938f948ecdd802c6ca0850ad3519f92 F ext/wasm/tests/opfs/concurrency/worker.js fc985ec86b70b057224e8caaa9c5a11892ea9b980a5c5da21b1efdf5f12bc3f6 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P c1cb61f473a11a325ef421ba8edfb20257203688565f7db62309044be183af5f -R bd137899af2cd4094f0a4f5a1f4094af +P 9a07eb7a941479510891d1444aacbeb440efaad3e9a13f186833618d8b60a8da +R 2c796a522d5a1762226997586fbd0bf5 U stephan -Z 147d7dd643bc15130deceeb0e6d52d6f +Z 681d2e8fff944e3c89259f16f4209f56 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 06ded776dc..7e39b0e048 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a07eb7a941479510891d1444aacbeb440efaad3e9a13f186833618d8b60a8da +17ac8d7bdfaf96510abea8c11019045c1703b7040655d9c4c8d7001636260b50 From 303fad1f91d63c05ff73e7f6ae9f225745eda731 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 8 Mar 2026 08:09:09 +0000 Subject: [PATCH 32/32] Update the tester1 SEE tests to account for opfs-wl. FossilOrigin-Name: 616d32c8380ef47c5a7c2eb765e7b6fbec9500d4e16c837f4ee11d71c239a55f --- ext/wasm/tester1.c-pp.js | 8 +++++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 81bc37e26d..c1fda13d76 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3599,7 +3599,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; test: function(sqlite3){ const JDb = sqlite3.oo1.JsStorageDb; T.seeBaseCheck(JDb, - (isInit)=>return {filename: "session"}, + (isInit)=>{ + return {filename: "session"}; + }, ()=>JDb.clearStorage('session')); } })/*kvvfs with SEE*/ @@ -3911,7 +3913,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //#if enable-see .t({ name: 'OPFS with SEE encryption', - predicate: ()=>!!sqlite3.oo1.OpfsDb, + predicate: (sqlite3)=>!!sqlite3.oo1.OpfsDb, test: function(sqlite3){ T.seeBaseCheck( sqlite3.oo1.OpfsDb, @@ -3945,7 +3947,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //#if enable-see .t({ name: 'OPFS-WL with SEE encryption', - predicate: ()=>!!sqlite3.oo1.OpfsWlDb, + predicate: (sqlite3)=>!!sqlite3.oo1.OpfsWlDb, test: function(sqlite3){ T.seeBaseCheck( sqlite3.oo1.OpfsWlDb, diff --git a/manifest b/manifest index 852b2ca2e8..7dd4119fca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\stester1\sOPFS\stest\sto\ssupport\sboth\sthe\s"opfs"\sand\s"opfs-wl"\sVFSes. -D 2026-03-08T07:58:02.639 +C Update\sthe\stester1\sSEE\stests\sto\saccount\sfor\sopfs-wl. +D 2026-03-08T08:09:09.808 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -644,7 +644,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js bccc66cdb09477ef09bf7c871bdba85f2c8b936a32d5dc07dc8c882517f73b50 +F ext/wasm/tester1.c-pp.js a84541ec9cfedcb41279a0da6b34c052a2c7b397c696b8a8a8f85964c8d1e9ca F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226 F ext/wasm/tests/opfs/concurrency/test.js 46c772bc18abb0fcbb058d57b5aaee9e7938f948ecdd802c6ca0850ad3519f92 F ext/wasm/tests/opfs/concurrency/worker.js fc985ec86b70b057224e8caaa9c5a11892ea9b980a5c5da21b1efdf5f12bc3f6 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 9a07eb7a941479510891d1444aacbeb440efaad3e9a13f186833618d8b60a8da -R 2c796a522d5a1762226997586fbd0bf5 +P 17ac8d7bdfaf96510abea8c11019045c1703b7040655d9c4c8d7001636260b50 +R 20c127d6b469936bc976ef9124b5bb85 U stephan -Z 681d2e8fff944e3c89259f16f4209f56 +Z c738f3c585f3f955ad9c9cce49c18902 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e39b0e048..4539686e52 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17ac8d7bdfaf96510abea8c11019045c1703b7040655d9c4c8d7001636260b50 +616d32c8380ef47c5a7c2eb765e7b6fbec9500d4e16c837f4ee11d71c239a55f