database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2026-10-23 21:49:02 +03:00
Code Activity

Merge opfs-wl into trunk, not so much for the "opfs-wl" VFS, which is stil tentative (working fine but with little perceived benefit), but for the "opfs" VFS cleanups which necessarily happened around it.

Browse Source 
FossilOrigin-Name: bcaa876c6dafee0995d239f59b6ce7810b6f5faa219b9c392966211ae3dec17a
This commit is contained in:
stephan
2026-03-08 08:32:22 +00:00
parent 84ad6f302f 303fad1f91
commit 976b50620a
18 changed files with 2530 additions and 1830 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
ext/wasm/GNUmakefile
View File

@@ -911,8 +911,10 @@ ifeq (0,$(wasm-bare-bones))
sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js
endif
sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js
sqlite3-api.jses += $(dir.api)/opfs-common-shared.c-pp.js
sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js
sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js
sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js
# Parallel builds can fail if $(sqlite3-license-version.js) is not
# created early enough, so make all files in $(sqlite-api.jses) except
@@ -1182,13 +1184,13 @@ all: demos
#######################################################################
#
# "SOAP" is a static file which is not part of the amalgamation but
# gets copied into the build output folder and into each of the fiddle
# builds.
# "SOAP" is not part of the amalgamation but gets copied into the
# build output folder and into each of the fiddle builds.
#
sqlite3.ext.js += $(dir.dout)/sqlite3-opfs-async-proxy.js
$(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js
@$(call b.cp,@,$<,$@)
$(eval $(call b.c-pp.target,soap,\
$(dir.api)/sqlite3-opfs-async-proxy.c-pp.js,\
$(dir.dout)/sqlite3-opfs-async-proxy.js))
#
# Add a dep of $(sqlite3.ext.js) on every individual build's JS file.

191
ext/wasm/api/opfs-common-inline.c-pp.js Normal file
View File

@@ -0,0 +1,191 @@
//#if nope
/**
This file is for preprocessor #include into the "opfs" and
"opfs-wl" impls, as well as their async-proxy part. It must be
inlined in those files, as opposed to being a shared copy in the
library, because (A) the async proxy does not load the library and
(B) it references an object which is local to each of those files
but which has a 99% identical structure for each.
*/
//#endif
//#// vfs.metrics.enable is a refactoring crutch.
//#define vfs.metrics.enable=0
const initS11n = function(){
/**
This proxy de/serializes cross-thread function arguments and
output-pointer values via the state.sabIO SharedArrayBuffer,
using the region defined by (state.sabS11nOffset,
state.sabS11nOffset + state.sabS11nSize]. Only one dataset is
recorded at a time.
This is not a general-purpose format. It only supports the
range of operations, and data sizes, needed by the
sqlite3_vfs and sqlite3_io_methods operations. Serialized
data are transient and this serialization algorithm may
change at any time.
The data format can be succinctly summarized as:
Nt...Td...D
Where:
- N = number of entries (1 byte)
- t = type ID of first argument (1 byte)
- ...T = type IDs of the 2nd and subsequent arguments (1 byte
each).
- d = raw bytes of first argument (per-type size).
- ...D = raw bytes of the 2nd and subsequent arguments (per-type
size).
All types except strings have fixed sizes. Strings are stored
using their TextEncoder/TextDecoder representations. It would
arguably make more sense to store them as Int16Arrays of
their JS character values, but how best/fastest to get that
in and out of string form is an open point. Initial
experimentation with that approach did not gain us any speed.
Historical note: this impl was initially about 1% this size by
using using JSON.stringify/parse(), but using fit-to-purpose
serialization saves considerable runtime.
*/
if(state.s11n) return state.s11n;
const textDecoder = new TextDecoder(),
textEncoder = new TextEncoder('utf-8'),
viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize),
viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
state.s11n = Object.create(null);
/* Only arguments and return values of these types may be
serialized. This covers the whole range of types needed by the
sqlite3_vfs API. */
const TypeIds = Object.create(null);
TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' };
TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' };
TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' };
TypeIds.string = { id: 4 };
const getTypeId = (v)=>(
TypeIds[typeof v]
|| toss("Maintenance required: this value type cannot be serialized.",v)
);
const getTypeIdById = (tid)=>{
switch(tid){
case TypeIds.number.id: return TypeIds.number;
case TypeIds.bigint.id: return TypeIds.bigint;
case TypeIds.boolean.id: return TypeIds.boolean;
case TypeIds.string.id: return TypeIds.string;
default: toss("Invalid type ID:",tid);
}
};
/**
Returns an array of the deserialized state stored by the most
recent serialize() operation (from this thread or the
counterpart thread), or null if the serialization buffer is
empty. If passed a truthy argument, the serialization buffer
is cleared after deserialization.
*/
state.s11n.deserialize = function(clear=false){
//#if vfs.metrics.enable
++metrics.s11n.deserialize.count;
//#endif
const t = performance.now();
const argc = viewU8[0];
const rc = argc ? [] : null;
if(argc){
const typeIds = [];
let offset = 1, i, n, v;
for(i = 0; i < argc; ++i, ++offset){
typeIds.push(getTypeIdById(viewU8[offset]));
}
for(i = 0; i < argc; ++i){
const t = typeIds[i];
if(t.getter){
v = viewDV[t.getter](offset, state.littleEndian);
offset += t.size;
}else{/*String*/
n = viewDV.getInt32(offset, state.littleEndian);
offset += 4;
v = textDecoder.decode(viewU8.slice(offset, offset+n));
offset += n;
}
rc.push(v);
}
}
if(clear) viewU8[0] = 0;
//log("deserialize:",argc, rc);
//#if vfs.metrics.enable
metrics.s11n.deserialize.time += performance.now() - t;
//#endif
return rc;
};
/**
Serializes all arguments to the shared buffer for consumption
by the counterpart thread.
This routine is only intended for serializing OPFS VFS
arguments and (in at least one special case) result values,
and the buffer is sized to be able to comfortably handle
those.
If passed no arguments then it zeroes out the serialization
state.
*/
state.s11n.serialize = function(...args){
const t = performance.now();
//#if vfs.metrics.enable
++metrics.s11n.serialize.count;
//#endif
if(args.length){
//log("serialize():",args);
const typeIds = [];
let i = 0, offset = 1;
viewU8[0] = args.length & 0xff /* header = # of args */;
for(; i < args.length; ++i, ++offset){
/* Write the TypeIds.id value into the next args.length
bytes. */
typeIds.push(getTypeId(args[i]));
viewU8[offset] = typeIds[i].id;
}
for(i = 0; i < args.length; ++i) {
/* Deserialize the following bytes based on their
corresponding TypeIds.id from the header. */
const t = typeIds[i];
if(t.setter){
viewDV[t.setter](offset, args[i], state.littleEndian);
offset += t.size;
}else{/*String*/
const s = textEncoder.encode(args[i]);
viewDV.setInt32(offset, s.byteLength, state.littleEndian);
offset += 4;
viewU8.set(s, offset);
offset += s.byteLength;
}
}
//log("serialize() result:",viewU8.slice(0,offset));
}else{
viewU8[0] = 0;
}
//#if vfs.metrics.enable
metrics.s11n.serialize.time += performance.now() - t;
//#endif
};
//#if defined opfs-async-proxy
state.s11n.storeException = state.asyncS11nExceptions
? ((priority,e)=>{
if(priority<=state.asyncS11nExceptions){
state.s11n.serialize([e.name,': ',e.message].join(""));
}
})
: ()=>{};
//#endif
return state.s11n;
//#undef vfs.metrics.enable
}/*initS11n()*/;

1301
ext/wasm/api/opfs-common-shared.c-pp.js Normal file
View File

File diff suppressed because it is too large Load Diff

1
ext/wasm/api/post-js-header.js
View File

@@ -34,6 +34,7 @@ Module.runSQLite3PostLoadInit = async function(
- sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls
- sqlite3-vfs-opfs.c-pp.js => OPFS VFS
- sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS
- sqlite3-vfs-opfs-wl.c-pp.js => WebLock-using OPFS VFS
- post-js-footer.js => this file's epilogue
And all of that gets sandwiched between extern-pre-js.js and

90
ext/wasm/api/sqlite3-api-prologue.js
View File

@@ -99,6 +99,13 @@
used in WASMFS-capable builds of the library (which the canonical
builds do not include).
- `disable` (as of 3.53.0) may be an object with the following
properties:
- `vfs`, an object, may contain a map of VFS names to booleans.
Any mapping to falsy are disabled. The supported names
are: "kvvfs", "opfs", "opfs-sahpool", "opfs-wl".
- Other disabling options may be added in the future.
[^1] = This property may optionally be a function, in which case
this function calls that function to fetch the value,
enabling delayed evaluation.
@@ -145,7 +152,8 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
);
return sqlite3ApiBootstrap.sqlite3;
}
const config = Object.assign(Object.create(null),{
const nu = (...obj)=>Object.assign(Object.create(null),...obj);
const config = nu({
exports: undefined,
memory: undefined,
bigIntEnabled: !!globalThis.BigInt64Array,
@@ -162,7 +170,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
certain wasm.xWrap.resultAdapter()s.
*/
useStdAlloc: false
}, apiConfig || {});
}, apiConfig);
Object.assign(config, {
allocExportName: config.useStdAlloc ? 'malloc' : 'sqlite3_malloc',
@@ -195,7 +203,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
not documented are installed as 1-to-1 proxies for their
C-side counterparts.
*/
const capi = Object.create(null);
const capi = nu();
/**
Holds state which are specific to the WASM-related
infrastructure and glue code.
@@ -204,7 +212,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
dynamically after the api object is fully constructed, so
not all are documented in this file.
*/
const wasm = Object.create(null);
const wasm = nu();
/** Internal helper for SQLite3Error ctor. */
const __rcStr = (rc)=>{
@@ -752,6 +760,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
toss: function(...args){throw new Error(args.join(' '))},
toss3,
typedArrayPart: wasm.typedArrayPart,
nu,
assert: function(arg,msg){
if( !arg ){
util.toss("Assertion failed:",msg);
@@ -1008,7 +1017,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true;
};
}
const rc = Object.create(null), ov = [0,0];
const rc = nu(), ov = [0,0];
let i = 0, k;
while((k = capi.sqlite3_compileoption_get(i++))){
f._opt(k,ov);
@@ -1016,7 +1025,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
}
return f._result = rc;
}else if(Array.isArray(optName)){
const rc = Object.create(null);
const rc = nu();
optName.forEach((v)=>{
rc[v] = capi.sqlite3_compileoption_used(v);
});
@@ -1067,7 +1076,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
The memory lives in the WASM heap and can be used with routines
such as wasm.poke() and wasm.heap8u().slice().
*/
wasm.pstack = Object.assign(Object.create(null),{
wasm.pstack = nu({
/**
Sets the current pstack position to the given pointer. Results
are undefined if the passed-in value did not come from
@@ -1289,7 +1298,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
// sqlite3__wasm_init_wasmfs() is not available
return this.dir = "";
}
}.bind(Object.create(null));
}.bind(nu());
/**
Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a
@@ -1665,7 +1674,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
default:
return capi.SQLITE_MISUSE;
}
}.bind(Object.create(null));
}.bind(nu());
/**
Given a (sqlite3_value*), this function attempts to convert it
@@ -1899,7 +1908,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc);
const pv = wasm.peekPtr(this.ptr);
return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined;
}.bind(Object.create(null));
}.bind(nu());
/**
A wrapper around sqlite3_preupdate_new() which fetches the
@@ -1939,6 +1948,62 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
'sqlite3changeset_old');
}/*changeset/preupdate additions*/
/**
EXPERIMENTAL. For tentative addition in 3.53.0.
sqlite3_js_retry_busy(maxTimes,callback[,beforeRetry])
Calls the given _synchronous_ callback function. If that function
returns sqlite3.capi.SQLITE_BUSY _or_ throws an SQLite3Error
which a resultCode property of that value then it will suppress
that error and try again, up to the given maximum number of
times. If the callback returns any other value than that,
it is returned. If the maximum number of retries has been
reached, an SQLite3Error with a resultCode value of
sqlite3.capi.SQLITE_BUSY is thrown. If the callback throws any
exception other than the aforementioned BUSY exception, it is
propagated. If it throws a BUSY exception on its final attempt,
that is propagated as well.
If the beforeRetry argument is given, it must be a _synchronous_
function. It is called immediately before each retry of the
callback (not for the initial call), passed the attempt number
(so it starts with 2, not 1). If it throws, the exception is
handled as described above. Its result value is ignored.
To effectively retry "forever", pass a negative maxTimes value,
with the caveat that there is no recovery from that if the code
gets stuck in a deadlock situation.
TODO: an async variant of this.
*/
capi.sqlite3_js_retry_busy = function(maxTimes, callback, beforeRetry){
for(let n = 1; n <= maxTimes; ++n){
try{
if( beforeRetry && n>1 ) beforeRetry(n);
const rc = callback();
if( capi.SQLITE_BUSY===rc ){
if( n===maxTimes ){
throw new SQLite3Error(rc, [
"sqlite3_js_retry_busy() max retry attempts (",
maxTimes,
") reached."
].join(''));
}
continue;
}
return rc;
}catch(e){
if( n<maxTimes
&& (e instanceof SQLite3Error)
&& e.resultCode===capi.SQLITE_BUSY ){
continue;
}
throw e;
}
}
};
/* The remainder of the API will be set up in later steps. */
const sqlite3 = {
WasmAllocError: WasmAllocError,
@@ -1957,7 +2022,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
This object is initially a placeholder which gets replaced by a
build-generated object.
*/
version: Object.create(null),
version: nu(),
/**
The library reserves the 'client' property for client-side use
@@ -2003,6 +2068,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
so that we can add tests for them. */
delete sqlite3.util;
delete sqlite3.StructBinder;
delete sqlite3.opfs;
}
return sqlite3;
};
@@ -2035,7 +2101,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
*/
scriptInfo: undefined
};
if( ('undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */) ){
if( 'undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */ ){
sqlite3.__isUnderTest = !!sqlite3IsUnderTest;
}
try{

535
ext/wasm/api/sqlite3-opfs-async-proxy.js → ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js
View File

@@ -46,10 +46,22 @@
theFunc().then(...) is not compatible with the change to
synchronous, but we do do not use those APIs that way. i.e. we don't
_need_ to change anything for this, but at some point (after Chrome
versions (approximately) 104-107 are extinct) should change our
versions (approximately) 104-107 are extinct) we should change our
usage of those methods to remove the "await".
*/
"use strict";
const urlParams = new URL(globalThis.location.href).searchParams;
const vfsName = urlParams.get('vfs');
if( !vfsName ){
throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker");
}
/**
We use this to allow us to differentiate debug output from
multiple instances, e.g. multiple Workers to the "opfs"
VFS or both the "opfs" and "opfs-wl" VFSes.
*/
const workerId = (Math.random() * 10000000) | 0;
const isWebLocker = 'opfs-wl'===urlParams.get('vfs');
const wPost = (type,...args)=>postMessage({type, payload:args});
const installAsyncProxy = function(){
const toss = function(...args){throw new Error(args.join(' '))};
@@ -66,6 +78,13 @@ const installAsyncProxy = function(){
*/
const state = Object.create(null);
/* initS11n() is preprocessor-injected so that we have identical
copies in the synchronous and async halves. This side does not
load the SQLite library, so does not have access to that copy. */
//#define opfs-async-proxy
//#include api/opfs-common-inline.c-pp.js
//#undef opfs-async-proxy
/**
verbose:
@@ -82,7 +101,7 @@ const installAsyncProxy = function(){
2:console.log.bind(console)
};
const logImpl = (level,...args)=>{
if(state.verbose>level) loggers[level]("OPFS asyncer:",...args);
if(state.verbose>level) loggers[level](vfsName+' async-proxy',workerId+":",...args);
};
const log = (...args)=>logImpl(2, ...args);
const warn = (...args)=>logImpl(1, ...args);
@@ -97,12 +116,13 @@ const installAsyncProxy = function(){
*/
const __openFiles = Object.create(null);
/**
__implicitLocks is a Set of sqlite3_file pointers (integers) which were
"auto-locked". i.e. those for which we obtained a sync access
handle without an explicit xLock() call. Such locks will be
released during db connection idle time, whereas a sync access
handle obtained via xLock(), or subsequently xLock()'d after
auto-acquisition, will not be released until xUnlock() is called.
__implicitLocks is a Set of sqlite3_file pointers (integers)
which were "auto-locked". i.e. those for which we necessarily
obtain a sync access handle without an explicit xLock() call
guarding access. Such locks will be released during
`waitLoop()`'s idle time, whereas a sync access handle obtained
via xLock(), or subsequently xLock()'d after auto-acquisition,
will not be released until xUnlock() is called.
Maintenance reminder: if we relinquish auto-locks at the end of the
operation which acquires them, we pay a massive performance
@@ -271,10 +291,11 @@ const installAsyncProxy = function(){
In order to help alleviate cross-tab contention for a dabase, if
an exception is thrown while acquiring the handle, this routine
will wait briefly and try again, up to some fixed number of
times. If acquisition still fails at that point it will give up
and propagate the exception. Client-level code will see that as
an I/O error.
will wait briefly and try again, up to `maxTries` of times. If
acquisition still fails at that point it will give up and
propagate the exception. Client-level code will see that either
as an I/O error or SQLITE_BUSY, depending on the exception and
the context.
2024-06-12: there is a rare race condition here which has been
reported a single time:
@@ -289,13 +310,31 @@ const installAsyncProxy = function(){
there's another race condition there). That's easy to say but
creating a viable test for that condition has proven challenging
so far.
Interface quirk: if fh.xLock is falsy and the handle is acquired
then fh.fid is added to __implicitLocks(). If fh.xLock is truthy,
it is not added as an implicit lock. i.e. xLock() impls must set
fh.xLock immediately _before_ calling this and must arrange to
restore it to its previous value if this function throws.
2026-03-06:
- baseWaitTime is the number of milliseconds to wait for the
first retry, increasing by one factor for each retry. It defaults
to (state.asyncIdleWaitTime*2).
- maxTries is the number of attempt to make, each one spaced out
by one additional factor of the baseWaitTime (e.g. 300, then 600,
then 900, the 1200...). This MUST be an integer >0.
Only the Web Locks impl should use the 3rd and 4th parameters.
*/
const getSyncHandle = async (fh,opName)=>{
const getSyncHandle = async (fh, opName, baseWaitTime, maxTries = 6)=>{
if(!fh.syncHandle){
const t = performance.now();
log("Acquiring sync handle for",fh.filenameAbs);
const maxTries = 6,
msBase = state.asyncIdleWaitTime * 2;
const msBase = baseWaitTime ?? (state.asyncIdleWaitTime * 2);
maxTries ??= 6;
let i = 1, ms = msBase;
for(; true; ms = msBase * ++i){
try {
@@ -329,6 +368,9 @@ const installAsyncProxy = function(){
/**
Stores the given value at state.sabOPView[state.opIds.rc] and then
Atomics.notify()'s it.
The opName is only used for logging and debugging - all result
codes are expected on the same state.sabOPView slot.
*/
const storeAndNotify = (opName, value)=>{
log(opName+"() => notify(",value,")");
@@ -458,24 +500,12 @@ const installAsyncProxy = function(){
await releaseImplicitLock(fh);
storeAndNotify('xFileSize', rc);
},
xLock: async function(fid/*sqlite3_file pointer*/,
lockType/*SQLITE_LOCK_...*/){
const fh = __openFiles[fid];
let rc = 0;
const oldLockType = fh.xLock;
fh.xLock = lockType;
if( !fh.syncHandle ){
try {
await getSyncHandle(fh,'xLock');
__implicitLocks.delete(fid);
}catch(e){
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK);
fh.xLock = oldLockType;
}
}
storeAndNotify('xLock',rc);
},
/**
The first argument is semantically invalid here - it's an
address in the synchronous side's heap. We can do nothing with
it here except use it as a unique-per-file identifier.
i.e. a lookup key.
*/
xOpen: async function(fid/*sqlite3_file pointer*/, filename,
flags/*SQLITE_OPEN_...*/,
opfsFlags/*OPFS_...*/){
@@ -533,7 +563,7 @@ const installAsyncProxy = function(){
rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
}
}catch(e){
error("xRead() failed",e,fh);
//error("xRead() failed",e,fh);
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ);
}
@@ -560,15 +590,192 @@ const installAsyncProxy = function(){
affirmNotRO('xTruncate', fh);
await (await getSyncHandle(fh,'xTruncate')).truncate(size);
}catch(e){
error("xTruncate():",e,fh);
//error("xTruncate():",e,fh);
state.s11n.storeException(2,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE);
}
await releaseImplicitLock(fh);
storeAndNotify('xTruncate',rc);
},
xUnlock: async function(fid/*sqlite3_file pointer*/,
lockType/*SQLITE_LOCK_...*/){
xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){
let rc;
const fh = __openFiles[fid];
try{
affirmNotRO('xWrite', fh);
rc = (
n === (await getSyncHandle(fh,'xWrite'))
.write(fh.sabView.subarray(0, n),
{at: Number(offset64)})
) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
}catch(e){
//error("xWrite():",e,fh);
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE);
}
await releaseImplicitLock(fh);
storeAndNotify('xWrite',rc);
}
}/*vfsAsyncImpls*/;
if( isWebLocker ){
/* We require separate xLock() and xUnlock() implementations for the
original and Web Lock implementations. The ones in this block
are for the WebLock impl.
The Golden Rule for this impl is: if we have a web lock, we
must also hold the SAH. When "upgrading" an implicit lock to a
requested (explicit) lock, we must remove the SAH from the
__implicitLocks set. When we unlock, we release both the web
lock and the SAH. That invariant must be kept intact or race
conditions on SAHs will ensue.
*/
/** Registry of active Web Locks: fid -> { mode, resolveRelease } */
const __activeWebLocks = Object.create(null);
vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/,
lockType/*SQLITE_LOCK_...*/,
isFromUnlock/*only if called from this.xUnlock()*/){
const whichOp = isFromUnlock ? 'xUnlock' : 'xLock';
const fh = __openFiles[fid];
//error("xLock()",fid, lockType, isFromUnlock, fh);
const requestedMode = (lockType >= state.sq3Codes.SQLITE_LOCK_RESERVED)
? 'exclusive' : 'shared';
const existing = __activeWebLocks[fid];
if( existing ){
if( existing.mode === requestedMode
|| (existing.mode === 'exclusive'
&& requestedMode === 'shared') ) {
fh.xLock = lockType;
storeAndNotify(whichOp, 0);
/* Don't do this: existing.mode = requestedMode;
Paraphrased from advice given by a consulting developer:
If you hold an exclusive lock and SQLite requests shared,
you should keep exiting.mode as exclusive in because the
underlying Web Lock is still exclusive. Changing it to
shared would trick xLock into thinking it needs to
perform a release/re-acquire dance if an exclusive is
later requested.
*/
return 0 /* Already held at required or higher level */;
}
/*
Upgrade path: we must release shared and acquire exclusive.
This transition is NOT atomic in Web Locks API.
It _effectively_ is atomic if we don't call
closeSyncHandle(fh), as no other worker can lock that until
we let it go. But we can't do that without eventually
leading to deadly embrace situations, so we don't do that.
(That's not a hypothetical, it has happened.)
*/
await closeSyncHandle(fh);
existing.resolveRelease();
delete __activeWebLocks[fid];
}
const lockName = "sqlite3-vfs-opfs:" + fh.filenameAbs;
const oldLockType = fh.xLock;
return new Promise((resolveWaitLoop) => {
//error("xLock() initial promise entered...");
navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => {
//error("xLock() Web Lock entered.", fh);
__implicitLocks.delete(fid);
let rc = 0;
try{
fh.xLock = lockType/*must be set before getSyncHandle() is called!*/;
await getSyncHandle(fh, 'xLock', state.asyncIdleWaitTime, 5);
}catch(e){
fh.xLock = oldLockType;
state.s11n.storeException(1, e);
rc = GetSyncHandleError.convertRc(e, state.sq3Codes.SQLITE_BUSY);
}
const releasePromise = rc
? undefined
: new Promise((resolveRelease) => {
__activeWebLocks[fid] = { mode: requestedMode, resolveRelease };
});
storeAndNotify(whichOp, rc) /* unblock the C side */;
resolveWaitLoop(0) /* unblock waitLoop() */;
await releasePromise /* hold the lock until xUnlock */;
});
});
};
/** Internal helper for the opfs-wl xUnlock() */
const wlCloseHandle = async(fh)=>{
let rc = 0;
try{
/* For the record, we've never once seen closeSyncHandle()
throw, nor should it because destructors do not throw. */
await closeSyncHandle(fh);
}catch(e){
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_UNLOCK;
}
return rc;
};
vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/,
lockType/*SQLITE_LOCK_...*/){
const fh = __openFiles[fid];
const existing = __activeWebLocks[fid];
if( !existing ){
const rc = await wlCloseHandle(fh);
storeAndNotify('xUnlock', rc);
return rc;
}
//error("xUnlock()",fid, lockType, fh);
let rc = 0;
if( lockType === state.sq3Codes.SQLITE_LOCK_NONE ){
/* SQLite usually unlocks all the way to NONE */
rc = await wlCloseHandle(fh);
existing.resolveRelease();
delete __activeWebLocks[fid];
fh.xLock = lockType;
}else if( lockType === state.sq3Codes.SQLITE_LOCK_SHARED
&& existing.mode === 'exclusive' ){
/* downgrade Exclusive -> Shared */
rc = await wlCloseHandle(fh);
if( 0===rc ){
fh.xLock = lockType;
existing.resolveRelease();
delete __activeWebLocks[fid];
return vfsAsyncImpls.xLock(fid, lockType, true);
}
}else{
/* ??? */
error("xUnlock() unhandled condition", fh);
}
storeAndNotify('xUnlock', rc);
return 0;
}
}else{
/* Original/"legacy" xLock() and xUnlock() */
vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/,
lockType/*SQLITE_LOCK_...*/){
const fh = __openFiles[fid];
let rc = 0;
const oldLockType = fh.xLock;
fh.xLock = lockType;
if( !fh.syncHandle ){
try {
await getSyncHandle(fh,'xLock');
__implicitLocks.delete(fid);
}catch(e){
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK);
fh.xLock = oldLockType;
}
}
storeAndNotify('xLock',rc);
};
vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/,
lockType/*SQLITE_LOCK_...*/){
let rc = 0;
const fh = __openFiles[fid];
if( fh.syncHandle
@@ -582,173 +789,112 @@ const installAsyncProxy = function(){
}
}
storeAndNotify('xUnlock',rc);
},
xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){
let rc;
const fh = __openFiles[fid];
try{
affirmNotRO('xWrite', fh);
rc = (
n === (await getSyncHandle(fh,'xWrite'))
.write(fh.sabView.subarray(0, n),
{at: Number(offset64)})
) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
}catch(e){
error("xWrite():",e,fh);
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE);
}
await releaseImplicitLock(fh);
storeAndNotify('xWrite',rc);
}
}/*vfsAsyncImpls*/;
const initS11n = ()=>{
/**
ACHTUNG: this code is 100% duplicated in the other half of this
proxy! The documentation is maintained in the "synchronous half".
*/
if(state.s11n) return state.s11n;
const textDecoder = new TextDecoder(),
textEncoder = new TextEncoder('utf-8'),
viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize),
viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
state.s11n = Object.create(null);
const TypeIds = Object.create(null);
TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' };
TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' };
TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' };
TypeIds.string = { id: 4 };
const getTypeId = (v)=>(
TypeIds[typeof v]
|| toss("Maintenance required: this value type cannot be serialized.",v)
);
const getTypeIdById = (tid)=>{
switch(tid){
case TypeIds.number.id: return TypeIds.number;
case TypeIds.bigint.id: return TypeIds.bigint;
case TypeIds.boolean.id: return TypeIds.boolean;
case TypeIds.string.id: return TypeIds.string;
default: toss("Invalid type ID:",tid);
}
};
state.s11n.deserialize = function(clear=false){
const argc = viewU8[0];
const rc = argc ? [] : null;
if(argc){
const typeIds = [];
let offset = 1, i, n, v;
for(i = 0; i < argc; ++i, ++offset){
typeIds.push(getTypeIdById(viewU8[offset]));
}
for(i = 0; i < argc; ++i){
const t = typeIds[i];
if(t.getter){
v = viewDV[t.getter](offset, state.littleEndian);
offset += t.size;
}else{/*String*/
n = viewDV.getInt32(offset, state.littleEndian);
offset += 4;
v = textDecoder.decode(viewU8.slice(offset, offset+n));
offset += n;
}
rc.push(v);
}
}
if(clear) viewU8[0] = 0;
//log("deserialize:",argc, rc);
return rc;
};
state.s11n.serialize = function(...args){
if(args.length){
//log("serialize():",args);
const typeIds = [];
let i = 0, offset = 1;
viewU8[0] = args.length & 0xff /* header = # of args */;
for(; i < args.length; ++i, ++offset){
/* Write the TypeIds.id value into the next args.length
bytes. */
typeIds.push(getTypeId(args[i]));
viewU8[offset] = typeIds[i].id;
}
for(i = 0; i < args.length; ++i) {
/* Deserialize the following bytes based on their
corresponding TypeIds.id from the header. */
const t = typeIds[i];
if(t.setter){
viewDV[t.setter](offset, args[i], state.littleEndian);
offset += t.size;
}else{/*String*/
const s = textEncoder.encode(args[i]);
viewDV.setInt32(offset, s.byteLength, state.littleEndian);
offset += 4;
viewU8.set(s, offset);
offset += s.byteLength;
}
}
//log("serialize() result:",viewU8.slice(0,offset));
}else{
viewU8[0] = 0;
}
};
state.s11n.storeException = state.asyncS11nExceptions
? ((priority,e)=>{
if(priority<=state.asyncS11nExceptions){
state.s11n.serialize([e.name,': ',e.message].join(""));
}
})
: ()=>{};
return state.s11n;
}/*initS11n()*/;
}/*xLock() and xUnlock() impls*/
const waitLoop = async function f(){
const opHandlers = Object.create(null);
for(let k of Object.keys(state.opIds)){
const vi = vfsAsyncImpls[k];
if(!vi) continue;
const o = Object.create(null);
opHandlers[state.opIds[k]] = o;
o.key = k;
o.f = vi;
if( !f.inited ){
f.inited = true;
f.opHandlers = Object.create(null);
for(let k of Object.keys(state.opIds)){
const vi = vfsAsyncImpls[k];
if(!vi) continue;
const o = Object.create(null);
f.opHandlers[state.opIds[k]] = o;
o.key = k;
o.f = vi;
}
}
const opIds = state.opIds;
const opView = state.sabOPView;
const slotWhichOp = opIds.whichOp;
const idleWaitTime = state.asyncIdleWaitTime;
const hasWaitAsync = !!Atomics.waitAsync;
//#if nope
error("waitLoop init: isWebLocker",isWebLocker,
"idleWaitTime",idleWaitTime,
"hasWaitAsync",hasWaitAsync);
//#endif
while(!flagAsyncShutdown){
try {
if('not-equal'!==Atomics.wait(
state.sabOPView, state.opIds.whichOp, 0, state.asyncIdleWaitTime
)){
/* Maintenance note: we compare against 'not-equal' because
let opId;
if( hasWaitAsync ){
opId = Atomics.load(opView, slotWhichOp);
if( 0===opId ){
const rv = Atomics.waitAsync(opView, slotWhichOp, 0,
idleWaitTime);
if( rv.async ) await rv.value;
await releaseImplicitLocks();
continue;
}
}else{
/**
For browsers without Atomics.waitAsync(), we require
the legacy implementation. Browser versions where
waitAsync() arrived:
https://github.com/tomayac/sqlite-wasm/issues/12
Chrome: 90 (2021-04-13)
Firefox: 145 (2025-11-11)
Safari: 16.4 (2023-03-27)
is reporting that this occasionally, under high loads,
returns 'ok', which leads to the whichOp being 0 (which
isn't a valid operation ID and leads to an exception,
along with a corresponding ugly console log
message). Unfortunately, the conditions for that cannot
be reliably reproduced. The only place in our code which
writes a 0 to the state.opIds.whichOp SharedArrayBuffer
index is a few lines down from here, and that instance
is required in order for clear communication between
the sync half of this proxy and this half.
The "opfs" VFS was not born until Chrome was somewhere in
the v104-108 range (Summer/Autumn 2022) and did not work
with Safari < v17 (2023-09-18) due to a WebKit bug which
restricted OPFS access from sub-Workers.
The waitAsync() counterpart of this block can be used by
both "opfs" and "opfs-wl", whereas this block can only be
used by "opfs". Performance comparisons between the two
in high-contention tests have been indecisive.
*/
await releaseImplicitLocks();
continue;
if('not-equal'!==Atomics.wait(
state.sabOPView, slotWhichOp, 0, state.asyncIdleWaitTime
)){
/* Maintenance note: we compare against 'not-equal' because
https://github.com/tomayac/sqlite-wasm/issues/12
is reporting that this occasionally, under high loads,
returns 'ok', which leads to the whichOp being 0 (which
isn't a valid operation ID and leads to an exception,
along with a corresponding ugly console log
message). Unfortunately, the conditions for that cannot
be reliably reproduced. The only place in our code which
writes a 0 to the state.opIds.whichOp SharedArrayBuffer
index is a few lines down from here, and that instance
is required in order for clear communication between
the sync half of this proxy and this half.
Much later (2026-03-07): that phenomenon is apparently
called a spurious wakeup.
*/
await releaseImplicitLocks();
continue;
}
opId = Atomics.load(state.sabOPView, slotWhichOp);
}
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
Atomics.store(opView, slotWhichOp, 0);
const hnd = f.opHandlers[opId]?.f ?? toss("No waitLoop handler for whichOp #",opId);
const args = state.s11n.deserialize(
true /* clear s11n to keep the caller from confusing this with
an exception string written by the upcoming
operation */
) || [];
//warn("waitLoop() whichOp =",opId, hnd, args);
if(hnd.f) await hnd.f(...args);
else error("Missing callback for opId",opId);
//error("waitLoop() whichOp =",opId, f.opHandlers[opId].key, args);
//#if nope
if( isWebLocker && (opId==opIds.xLock || opIds==opIds.xUnlock) ){
/* An expert suggests that this introduces a race condition,
but my eyes aren't seeing it. The hope was that this
would improve the lock speed a tick, but it does not
appear to. */
hnd(...args);
continue;
}
//#endif
await hnd(...args);
}catch(e){
error('in waitLoop():',e);
error('in waitLoop():', e);
}
}
};
@@ -756,6 +902,7 @@ const installAsyncProxy = function(){
navigator.storage.getDirectory().then(function(d){
state.rootDir = d;
globalThis.onmessage = function({data}){
//log(globalThis.location.href,"onmessage()",data);
switch(data.type){
case 'opfs-async-init':{
/* Receive shared state from synchronous partner */
@@ -771,6 +918,7 @@ const installAsyncProxy = function(){
}
});
initS11n();
//warn("verbosity =",opt.verbose, state.verbose);
log("init state",state);
wPost('opfs-async-inited');
waitLoop();
@@ -782,22 +930,27 @@ const installAsyncProxy = function(){
flagAsyncShutdown = false;
waitLoop();
}
break;
break;
}
};
wPost('opfs-async-loaded');
}).catch((e)=>error("error initializing OPFS asyncer:",e));
}/*installAsyncProxy()*/;
if(!globalThis.SharedArrayBuffer){
if(globalThis.window === globalThis){
wPost('opfs-unavailable',
"This code cannot run from the main thread.",
"Load it as a Worker from a separate Worker.");
}else if(!globalThis.SharedArrayBuffer){
wPost('opfs-unavailable', "Missing SharedArrayBuffer API.",
"The server must emit the COOP/COEP response headers to enable that.");
}else if(!globalThis.Atomics){
wPost('opfs-unavailable', "Missing Atomics API.",
"The server must emit the COOP/COEP response headers to enable that.");
}else if(isWebLocker && !globalThis.Atomics.waitAsync){
wPost('opfs-unavailable',"Missing required Atomics.waitSync() for "+vfsName);
}else if(!globalThis.FileSystemHandle ||
!globalThis.FileSystemDirectoryHandle ||
!globalThis.FileSystemFileHandle ||
!globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle ||
!globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle ||
!navigator?.storage?.getDirectory){
wPost('opfs-unavailable',"Missing required OPFS APIs.");
}else{

7
ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js
View File

@@ -28,6 +28,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
//#@policy error
//#savepoint begin
//#define kvvfs-v2-added-in=3.52.0
/**
kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates
storage of its pages and metadata to a key-value store.
@@ -85,6 +86,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
backups.
*/
globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
if( sqlite3.config.disable?.vfs?.kvvfs ){
return;
}
'use strict';
const capi = sqlite3.capi,
sqlite3_kvvfs_methods = capi.sqlite3_kvvfs_methods,
@@ -1137,9 +1141,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
}/*.ioJrnl*/
}/*methodOverrides*/;
//#if nope
debug("pVfs and friends", pVfs, pIoDb, pIoJrnl,
kvvfsMethods, capi.sqlite3_file.structInfo,
KVVfsFile.structInfo);
//#endif
try {
util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough"
/* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ );

4
ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
View File

@@ -55,6 +55,10 @@
*/
globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
'use strict';
if( sqlite3.config.disable?.vfs?.['opfs-sahpool'] ){
return;
}
const toss = sqlite3.util.toss;
const toss3 = sqlite3.util.toss3;
const initPromises = Object.create(null) /* cache of (name:result) of VFS init results */;

133
ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js Normal file
View File

@@ -0,0 +1,133 @@
//#if not target:node
/*
2026-02-20
The author disclaims copyright to this source code. In place of a
legal notice, here is a blessing:
* May you do good and not evil.
* May you find forgiveness for yourself and forgive others.
* May you share freely, never taking more than you give.
***********************************************************************
This file is a reimplementation of the "opfs" VFS (as distinct from
"opfs-sahpool") which uses WebLocks for locking instead of a bespoke
custom Atomics.wait()/notify() protocol. This file holds the
"synchronous half" of the VFS, whereas it shares the "asynchronous
half" of the "opfs" VFS.
This file is intended to be appended to the main sqlite3 JS
deliverable somewhere after sqlite3-api-oo1.js.
TODOs (2026-03-03):
- For purposes of tester1.js we need to figure out which of these
VFSes will install the (internal-use-only) sqlite3.opfs utility code
namespace. We need that in order to clean up OPFS files during test
runs. Alternately, move those into their own
sqlite3ApiBootstrap.initializers entry which precedes both of the
VFSes, so they'll have access to it during bootstrapping. The
sqlite3.opfs namespace is removed at the end of bootstrapping unless
the library is told to run in testing mode (which is not a
documented feature).
*/
'use strict';
globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.['opfs-wl'] ){
return;
}
const util = sqlite3.util,
toss = sqlite3.util.toss;
const opfsUtil = sqlite3.opfs;
const vfsName = 'opfs-wl';
/**
installOpfsWlVfs() returns a Promise which, on success, installs an
sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs
which accept a VFS. It is intended to be called via
sqlite3ApiBootstrap.initializers or an equivalent mechanism.
This VFS is essentially identical to the "opfs" VFS but uses
WebLocks for its xLock() and xUnlock() implementations.
Quirks specific to this VFS:
- The (officially undocumented) 'opfs-wl-disable' URL
argument will disable OPFS, making this function a no-op.
Aside from locking differences in the VFSes, this function
otherwise behaves the same as
sqlite3-vfs-opfs.c-pp.js:installOpfsVfs().
*/
const installOpfsWlVfs = async function(options){
options = opfsUtil.initOptions(vfsName,options);
if( !options ) return sqlite3;
const capi = sqlite3.capi,
state = opfsUtil.createVfsState(),
opfsVfs = state.vfs,
metrics = opfsVfs.metrics.counters,
mTimeStart = opfsVfs.mTimeStart,
mTimeEnd = opfsVfs.mTimeEnd,
opRun = opfsVfs.opRun,
debug = (...args)=>sqlite3.config.debug(vfsName+":",...args),
warn = (...args)=>sqlite3.config.warn(vfsName+":",...args),
__openFiles = opfsVfs.__openFiles;
//debug("state",JSON.stringify(options));
/*
At this point, createVfsState() has populated:
- state: the configuration object we share with the async proxy.
- opfsVfs: an sqlite3_vfs instance with lots of JS state attached
to it.
with any code common to both the "opfs" and "opfs-wl" VFSes. Now
comes the VFS-dependent work...
*/
return opfsVfs.bindVfs(util.nu({
xLock: function(pFile,lockType){
mTimeStart('xLock');
//debug("xLock()...");
const f = __openFiles[pFile];
const rc = opRun('xLock', pFile, lockType);
if( !rc ) f.lockType = lockType;
mTimeEnd();
return rc;
},
xUnlock: function(pFile,lockType){
mTimeStart('xUnlock');
const f = __openFiles[pFile];
const rc = opRun('xUnlock', pFile, lockType);
if( !rc ) f.lockType = lockType;
mTimeEnd();
return rc;
}
}), function(sqlite3, vfs){
/* Post-VFS-registration initialization... */
if(sqlite3.oo1){
const OpfsWlDb = function(...args){
const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args);
opt.vfs = vfs.$zName;
sqlite3.oo1.DB.dbCtorHelper.call(this, opt);
};
OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
sqlite3.oo1.OpfsWlDb = OpfsWlDb;
OpfsWlDb.importDb = opfsUtil.importDb;
/* The "opfs" VFS variant adds a
oo1.DB.dbCtorHelper.setVfsPostOpenCallback() callback to set
a high busy_timeout. That was a design mis-decision and is
inconsistent with sqlite3_open() and friends, but is retained
against the risk of introducing regressions if it's removed.
This variant does not repeat that mistake.
*/
}
})/*bindVfs()*/;
}/*installOpfsWlVfs()*/;
globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{
return installOpfsWlVfs().catch((e)=>{
sqlite3.config.warn("Ignoring inability to install the",vfsName,"sqlite3_vfs:",e);
});
});
}/*sqlite3ApiBootstrap.initializers.push()*/);
//#endif target:node

1454
ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
View File

File diff suppressed because it is too large Load Diff

5
ext/wasm/index.html
View File

@@ -115,6 +115,11 @@
<a href='speedtest1-worker-64bit.html?vfs=opfs&size=10'>64-bit</a>):
speedtest1-worker with the
OPFS VFS preselected and configured for a moderate workload.</li>
<li>speedtest1-worker?vfs=opfs-wl
(<a href='speedtest1-worker.html?vfs=opfs-wl&size=10'>32-bit</a>,
<a href='speedtest1-worker-64bit.html?vfs=opfs-wl&size=10'>64-bit</a>):
speedtest1-worker with the
OPFS Web Locks VFS preselected and configured for a moderate workload.</li>
<li>speedtest1-worker?vfs=opfs-sahpool
(<a href='speedtest1-worker.html?vfs=opfs-sahpool&size=10'>32-bit</a>,
<a href='speedtest1-worker-64bit.html?vfs=opfs-sahpool&size=10'>64-bit</a>):

17
ext/wasm/mkwasmbuilds.c
View File

@@ -246,7 +246,7 @@ const BuildDefs oBuildDefs = {
.zEnv = 0,
.zDeps = 0,
.zIfCond = 0,
.flags = CP_ALL | F_64BIT
.flags = CP_ALL | F_64BIT | F_NOT_IN_ALL
},
/* The canonical esm build. */
@@ -274,7 +274,7 @@ const BuildDefs oBuildDefs = {
.zEnv = 0,
.zDeps = 0,
.zIfCond = 0,
.flags = CP_JS | F_ESM | F_64BIT
.flags = CP_JS | F_ESM | F_64BIT | F_NOT_IN_ALL
},
/* speedtest1, our primary benchmarking tool */
@@ -747,6 +747,12 @@ static void emit_api_js(char const *zBuildName){
zBuildName, zBuildName, zBuildName);
pf("$(out.%s.js): $(sqlite3-api.%s.js)\n",
zBuildName, zBuildName);
pf("$(sqlite3-api.%s.js):"
/* Extra deps needed by the OPFS pieces... */
" $(dir.api)/opfs-common-shared.c-pp.js"
" $(dir.api)/opfs-common-inline.c-pp.js"
"\n",
zBuildName);
}
/*
@@ -994,7 +1000,8 @@ static void mk_fiddle(void){
pf("$(out.%s.js): $(MAKEFILE_LIST) "
"$(EXPORTED_FUNCTIONS.fiddle) "
"$(fiddle.c.in) "
"$(pre-post.%s.deps)",
"$(pre-post.%s.deps) "
"$(dir.dout)/sqlite3-opfs-async-proxy.js",
zBuildName, zBuildName);
if( isDebug ){
pf(" $(dir.fiddle)/fiddle-worker.js"
@@ -1014,10 +1021,6 @@ static void mk_fiddle(void){
pf("\t@$(call b.call.wasm-strip,%s)\n", zBuildName);
pf("\t@$(call b.strip-js-emcc-bindings,$(logtag.%s))\n",
zBuildName);
pf("\t@$(call b.cp,"
"%s,"
"$(dir.api)/sqlite3-opfs-async-proxy.js,"
"$(dir $@))\n", zBuildName);
if( isDebug ){
pf("\t@$(call b.cp,%s,"
"$(dir.fiddle)/index.html "

342
ext/wasm/tester1.c-pp.js
View File

@@ -66,15 +66,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
const haveWasmCTests = ()=>{
return !!wasm.exports.sqlite3__wasm_test_intptr;
};
const hasOpfs = ()=>{
return globalThis.FileSystemHandle
&& globalThis.FileSystemDirectoryHandle
&& globalThis.FileSystemFileHandle
&& globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle
&& navigator?.storage?.getDirectory;
};
let SQLite3 /* populated after module load */;
const hasOpfs = ()=>!!SQLite3?.oo1?.OpfsDb;
{
const mapToString = (v)=>{
@@ -129,17 +123,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
};
}
}
const reportFinalTestStatus = function(pass){
if(isUIThread()){
let e = document.querySelector('#color-target');
e.classList.add(pass ? 'tests-pass' : 'tests-fail');
e = document.querySelector('title');
e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText;
}else{
postMessage({type:'test-result', payload:{pass}});
}
TestUtil.checkHeapSize(true);
};
const log = (...args)=>{
//console.log(...args);
logClass('',...args);
@@ -157,6 +140,19 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
console.debug('tester1',...args);
};
const reportFinalTestStatus = function(pass){
debug("Final test status:",pass);
if(isUIThread()){
let e = document.querySelector('#color-target');
e.classList.add(pass ? 'tests-pass' : 'tests-fail');
e = document.querySelector('title');
e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText;
}else{
postMessage({type:'test-result', payload:{pass}});
}
TestUtil.checkHeapSize(true);
};
const toss = (...args)=>{
error(...args);
throw new Error(args.join(' '));
@@ -455,9 +451,131 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
T.assert( !initDb );
tryKey('hexkey', hexFoo, 6);
dbUnlink();
},
};
//#endif enable-see
/* Tests common to "opfs" and "opfs-wl". These tests manipulate
"this" and must run in order.
*/
T.opfsCommon = {
sanityChecks: async function(vfsName, oo1Ctor, sqlite3){
T.assert(capi.sqlite3_vfs_find(vfsName));
const opfs = sqlite3.opfs;
const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db';
const fileUri = 'file://'+filename+'?delete-before-open=1';
const initSql = [
'create table p(a);',
'insert into p(a) values(1),(2),(3)'
];
let db = new oo1Ctor(fileUri);
try {
db.exec(initSql);
T.assert(3 === db.selectValue('select count(*) from p'));
db.close();
db = new oo1Ctor(filename);
db.exec('insert into p(a) values(4),(5),(6)');
T.assert(6 === db.selectValue('select count(*) from p'));
this.opfsDbExport = capi.sqlite3_js_db_export(db);
T.assert(this.opfsDbExport instanceof Uint8Array)
.assert(this.opfsDbExport.byteLength>0
&& 0===this.opfsDbExport.byteLength % 512);
}finally{
db.close();
db = null;
}
T.assert(await opfs.entryExists(filename));
try {
db = new oo1Ctor(fileUri);
db.exec(initSql) /* will throw if delete-before-open did not work */;
T.assert(3 === db.selectValue('select count(*) from p'));
}finally{
if(db) db.close();
}
},
importer: async function(vfsName, oo1Ctor, sqlite3){
let db;
const filename = this.opfsDbFile;
try {
const exp = this.opfsDbExport;
delete this.opfsDbExport;
this.opfsImportSize = await oo1Ctor.importDb(filename, exp);
db = new oo1Ctor(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
db.close();
db = null;
this.opfsUnlink =
(fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink(vfsName, fn);
this.opfsUnlink(filename);
T.assert(!(await sqlite3.opfs.entryExists(filename)));
// Try again with a function as an input source:
let cursor = 0;
const blockSize = 512, end = exp.byteLength;
const reader = async function(){
if(cursor >= exp.byteLength){
return undefined;
}
const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
cursor += blockSize;
return rv;
};
this.opfsImportSize = await oo1Ctor.importDb(filename, reader);
db = new oo1Ctor(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
}finally{
if(db) db.close();
}
},
opfsUtil: async function(vfsName, oo1Ctor, sqlite3){
const filename = this.opfsDbFile;
const unlink = this.opfsUnlink;
T.assert(filename && !!unlink);
delete this.opfsDbFile;
delete this.opfsUnlink;
/**************************************************************
ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended
for client-side use. It is only for this project's own
internal use. Its APIs are subject to change or removal at
any time. The sqlite3.opfs namespace is REMOVED from the
sqlite3 namespace in non-test runs of the library.
***************************************************************/
const opfs = sqlite3.opfs;
const fSize = this.opfsImportSize;
delete this.opfsImportSize;
let sh;
try{
T.assert(await opfs.entryExists(filename));
const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false);
const fh = await dirHandle.getFileHandle(filenamePart);
sh = await fh.createSyncAccessHandle();
T.assert(fSize === await sh.getSize());
await sh.close();
sh = undefined;
unlink();
T.assert(!(await opfs.entryExists(filename)));
}finally{
if(sh) await sh.close();
unlink();
}
// Some sanity checks of the opfs utility functions...
const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12);
const aDir = testDir+'/test/dir';
T.assert(await opfs.mkdir(aDir), "mkdir failed")
.assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists")
.assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)")
.assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed")
.assert(!(await opfs.unlink(testDir+'/test/dir')),
"delete 2b should have failed (dir already deleted)")
.assert((await opfs.unlink(testDir, true)), "delete 3 failed")
.assert(!(await opfs.entryExists(testDir)),
"entryExists(",testDir,") should have failed");
}
}/*T.opfsCommon*/;
////////////////////////////////////////////////////////////////////////
// End of infrastructure setup. Now define the tests...
////////////////////////////////////////////////////////////////////////
@@ -3481,7 +3599,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
test: function(sqlite3){
const JDb = sqlite3.oo1.JsStorageDb;
T.seeBaseCheck(JDb,
(isInit)=>return {filename: "session"},
(isInit)=>{
return {filename: "session"};
},
()=>JDb.clearStorage('session'));
}
})/*kvvfs with SEE*/
@@ -3771,131 +3891,29 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
////////////////////////////////////////////////////////////////////////
T.g('OPFS: Origin-Private File System',
(sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs")
|| 'requires "opfs" VFS'))
(sqlite3)=>(!!sqlite3.oo1.OpfsDb || 'requires "opfs" VFS'))
.t({
name: 'OPFS db sanity checks',
test: async function(sqlite3){
T.assert(capi.sqlite3_vfs_find('opfs'));
const opfs = sqlite3.opfs;
const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db';
const fileUri = 'file://'+filename+'?delete-before-open=1';
const initSql = [
'create table p(a);',
'insert into p(a) values(1),(2),(3)'
];
let db = new sqlite3.oo1.OpfsDb(fileUri);
try {
db.exec(initSql);
T.assert(3 === db.selectValue('select count(*) from p'));
db.close();
db = new sqlite3.oo1.OpfsDb(filename);
db.exec('insert into p(a) values(4),(5),(6)');
T.assert(6 === db.selectValue('select count(*) from p'));
this.opfsDbExport = capi.sqlite3_js_db_export(db);
T.assert(this.opfsDbExport instanceof Uint8Array)
.assert(this.opfsDbExport.byteLength>0
&& 0===this.opfsDbExport.byteLength % 512);
}finally{
db.close();
}
T.assert(await opfs.entryExists(filename));
try {
db = new sqlite3.oo1.OpfsDb(fileUri);
db.exec(initSql) /* will throw if delete-before-open did not work */;
T.assert(3 === db.selectValue('select count(*) from p'));
}finally{
if(db) db.close();
}
test: async (sqlite3)=>{
await T.opfsCommon.sanityChecks('opfs', sqlite3.oo1.OpfsDb, sqlite3);
}
}/*OPFS db sanity checks*/)
})
.t({
name: 'OPFS import',
test: async function(sqlite3){
let db;
const filename = this.opfsDbFile;
try {
const exp = this.opfsDbExport;
delete this.opfsDbExport;
this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp);
db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
db.close();
this.opfsUnlink =
(fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink("opfs",fn);
this.opfsUnlink(filename);
T.assert(!(await sqlite3.opfs.entryExists(filename)));
// Try again with a function as an input source:
let cursor = 0;
const blockSize = 512, end = exp.byteLength;
const reader = async function(){
if(cursor >= exp.byteLength){
return undefined;
}
const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
cursor += blockSize;
return rv;
};
this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader);
db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
}finally{
if(db) db.close();
}
name: 'OPFS import/export',
test: async (sqlite3)=>{
await T.opfsCommon.importer('opfs', sqlite3.oo1.OpfsDb, sqlite3);
}
}/*OPFS export/import*/)
})
.t({
name: '(Internal-use) OPFS utility APIs',
test: async function(sqlite3){
const filename = this.opfsDbFile;
const unlink = this.opfsUnlink;
T.assert(filename && !!unlink);
delete this.opfsDbFile;
delete this.opfsUnlink;
/**************************************************************
ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended
for client-side use. It is only for this project's own
internal use. Its APIs are subject to change or removal at
any time.
***************************************************************/
const opfs = sqlite3.opfs;
const fSize = this.opfsImportSize;
delete this.opfsImportSize;
let sh;
try{
T.assert(await opfs.entryExists(filename));
const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false);
const fh = await dirHandle.getFileHandle(filenamePart);
sh = await fh.createSyncAccessHandle();
T.assert(fSize === await sh.getSize());
await sh.close();
sh = undefined;
unlink();
T.assert(!(await opfs.entryExists(filename)));
}finally{
if(sh) await sh.close();
unlink();
}
// Some sanity checks of the opfs utility functions...
const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12);
const aDir = testDir+'/test/dir';
T.assert(await opfs.mkdir(aDir), "mkdir failed")
.assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists")
.assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)")
.assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed")
.assert(!(await opfs.unlink(testDir+'/test/dir')),
"delete 2b should have failed (dir already deleted)")
.assert((await opfs.unlink(testDir, true)), "delete 3 failed")
.assert(!(await opfs.entryExists(testDir)),
"entryExists(",testDir,") should have failed");
test: async (sqlite3)=>{
await T.opfsCommon.opfsUtil("opfs", sqlite3.oo1.OpfsDb, sqlite3);
}
}/*OPFS util sanity checks*/)
})
//#if enable-see
.t({
name: 'OPFS with SEE encryption',
predicate: (sqlite3)=>!!sqlite3.oo1.OpfsDb,
test: function(sqlite3){
T.seeBaseCheck(
sqlite3.oo1.OpfsDb,
@@ -3904,15 +3922,50 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
if( isInit ) opt.filename += '?delete-before-open=1';
return opt;
},
()=>{});
()=>{}
);
}
})/*OPFS with SEE*/
})
//#endif enable-see
;/* end OPFS tests */
////////////////////////////////////////////////////////////////////////
T.g('OPFS-WL: Origin-Private File System with Web Locks',
(sqlite3)=>(!!sqlite3.oo1.OpfsWlDb || 'requires "opfs-wl" VFS'))
.t({
name: 'OPFS-WL db sanity checks',
test: async (sqlite3)=>{
await T.opfsCommon.sanityChecks('opfs-wl', sqlite3.oo1.OpfsWlDb, sqlite3);
}
})
.t({
name: 'OPFS-WL import/export',
test: async (sqlite3)=>{
await T.opfsCommon.importer('opfs-wl', sqlite3.oo1.OpfsWlDb, sqlite3);
}
})
//#if enable-see
.t({
name: 'OPFS-WL with SEE encryption',
predicate: (sqlite3)=>!!sqlite3.oo1.OpfsWlDb,
test: function(sqlite3){
T.seeBaseCheck(
sqlite3.oo1.OpfsWlDb,
function(isInit){
const opt = {filename: 'file:///sqlite3-see.edb'};
if( isInit ) opt.filename += '?delete-before-open=1';
return opt;
},
()=>{}
);
}
})
//#endif enable-see
;/* end OPFS tests */
////////////////////////////////////////////////////////////////////////
T.g('OPFS SyncAccessHandle Pool VFS',
(sqlite3)=>(hasOpfs() || "requires OPFS APIs"))
(sqlite3)=>(!!sqlite3.installOpfsSAHPoolVfs || "requires OPFS SAH Pool APIs"))
.t({
name: 'SAH sanity checks',
test: async function(sqlite3){
@@ -4438,12 +4491,17 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
////////////////////////////////////////////////////////////////////////
log("Loading and initializing sqlite3 WASM module...");
if(0){
if(1){
globalThis.sqlite3ApiConfig = {
debug: ()=>{},
log: ()=>{},
warn: ()=>{},
error: ()=>{}
//debug: ()=>{}, log: ()=>{}, warn: ()=>{}, error: ()=>{},
disable: {
vfs: {
kvvfs: false,
opfs: false,
"opfs-sahpool": false,
"opfs-wl": false
}
}
}
}
//#if not target:es6-module

11
ext/wasm/tests/opfs/concurrency/index.html
View File

@@ -25,15 +25,14 @@
the <code>workers=N</code> URL flag. Set the time between each
workload with <code>interval=N</code> (milliseconds). Set the
number of worker iterations with <code>iterations=N</code>.
Enable OPFS VFS verbosity with <code>verbose=1-3</code> (output
goes to the dev console). Disable/enable "unlock ASAP" mode
(higher concurrency, lower speed) with <code>unlock-asap=0-1</code>.
Disable/enable "unlock ASAP" mode (potentially higher
concurrency but lower speed) with <code>unlock-asap=0-1</code>.
</p>
<p>Achtung: if it does not start to do anything within a couple of
seconds, check the dev console: Chrome sometimes fails to load
the wasm module due to "cannot allocate WasmMemory." Closing and
re-opening the tab usually resolves it, but sometimes restarting
the browser is required.
the wasm module due to "cannot allocate WasmMemory" when
reloading the page. Closing and re-opening the tab usually
resolves it, but sometimes restarting the browser is required.
</p>
<p>
Links for various testing options: <ul id='testlinks'></ul>

81
ext/wasm/tests/opfs/concurrency/test.js
View File

@@ -8,6 +8,7 @@
return ''+v;
default: break;
}
if( v instanceof Date ) return v.toString();
if(null===v) return 'null';
if(v instanceof Error){
v = {
@@ -55,11 +56,11 @@
const options = Object.create(null);
options.sqlite3Dir = urlArgsJs.get('sqlite3.dir');
options.workerCount = (
urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3
) || 4;
urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 0
) || 3;
options.opfsVerbose = (
urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1
) || 1;
) || 0;
options.interval = (
urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 1000
) || 1000;
@@ -69,47 +70,60 @@
options.unlockAsap = (
urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0
) || 0;
options.vfs = urlArgsHtml.get('vfs') || 'opfs';
options.noUnlink = !!urlArgsHtml.has('no-unlink');
const workers = [];
workers.post = (type,...args)=>{
for(const w of workers) w.postMessage({type, payload:args});
};
workers.counts = {loaded: 0, passed: 0, failed: 0};
let timeStart;
const calcTime = (endDate)=>{
return endDate.getTime() - timeStart?.getTime?.();
};
const checkFinished = function(){
if(workers.counts.passed + workers.counts.failed !== workers.length){
return;
}
if(workers.counts.failed>0){
logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s).");
logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s) in",
calcTime(new Date()),"ms");
}else{
logCss('tests-pass',"All",workers.length,"workers finished.");
logCss('tests-pass',"All",workers.length,"workers finished in",
calcTime(new Date()),"ms");
}
};
workers.onmessage = function(msg){
msg = msg.data;
const prefix = 'Worker #'+msg.worker+':';
switch(msg.type){
case 'loaded':
stdout(prefix,"loaded");
if(++workers.counts.loaded === workers.length){
stdout("All",workers.length,"workers loaded. Telling them to run...");
workers.post('run');
}
break;
case 'stdout': stdout(prefix,...msg.payload); break;
case 'stderr': stderr(prefix,...msg.payload); break;
case 'error': stderr(prefix,"ERROR:",...msg.payload); break;
case 'finished':
++workers.counts.passed;
logCss('tests-pass',prefix,...msg.payload);
checkFinished();
break;
case 'failed':
++workers.counts.failed;
logCss('tests-fail',prefix,"FAILED:",...msg.payload);
checkFinished();
break;
default: logCss('error',"Unhandled message type:",msg); break;
case 'loaded':
stdout(prefix,"loaded");
if(++workers.counts.loaded === workers.length){
timeStart = new Date();
stdout(timeStart,"All",workers.length,"workers loaded. Telling them to run...");
workers.post('run');
}
break;
case 'stdout': stdout(prefix,...msg.payload); break;
case 'stderr': stderr(prefix,...msg.payload); break;
case 'error': stderr(prefix,"ERROR:",...msg.payload); break;
case 'finished': {
++workers.counts.passed;
const timeEnd = new Date();
logCss('tests-pass',prefix,timeEnd,"("+calcTime(timeEnd)+"ms) ", ...msg.payload);
checkFinished();
break;
}
case 'failed': {
++workers.counts.failed;
const timeEnd = new Date();
logCss('tests-fail',prefix,timeEnd,"("+calcTime(timeEnd)+"ms) FAILED:",...msg.payload);
checkFinished();
break;
}
default: logCss('error',"Unhandled message type:",msg); break;
}
};
@@ -118,7 +132,10 @@
const eTestLinks = document.querySelector('#testlinks');
const optArgs = function(obj){
const li = [];
for(const k of ['interval','iterations','workers','verbose','unlock-asap']){
for(const k of [
'interval', 'iterations', 'unlock-asap',
'opfsVerbose', 'vfs', 'workers'
]){
if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]);
}
return li.join('&');
@@ -127,7 +144,10 @@
{interval: 1000, workers: 5, iterations: 30},
{interval: 500, workers: 5, iterations: 30},
{interval: 250, workers: 3, iterations: 30},
{interval: 600, workers: 5, iterations: 100}
{interval: 600, workers: 5, iterations: 100},
{interval: 500, workers: 1, iterations: 5, vfs: 'opfs-wl'},
{interval: 500, workers: 2, iterations: 10, vfs: 'opfs-wl'},
{interval: 500, workers: 5, iterations: 20, vfs: 'opfs-wl'},
]){
const li = document.createElement('li');
eTestLinks.appendChild(li);
@@ -142,13 +162,14 @@
workers.uri = (
'worker.js?'
+ 'sqlite3.dir='+options.sqlite3Dir
+ '&vfs='+options.vfs
+ '&interval='+options.interval
+ '&iterations='+options.iterations
+ '&opfs-verbose='+options.opfsVerbose
+ '&opfs-unlock-asap='+options.unlockAsap
);
for(let i = 0; i < options.workerCount; ++i){
stdout("Launching worker...");
stdout("Launching worker...", i, );
workers.push(new Worker(
workers.uri+'&workerId='+(i+1)+(
(i || options.noUnlink) ? '' : '&unlink-db'
@@ -158,4 +179,4 @@
// Have to delay onmessage assignment until after the loop
// to avoid that early workers get an undue head start.
workers.forEach((w)=>w.onmessage = workers.onmessage);
})(self);
})(globalThis);

134
ext/wasm/tests/opfs/concurrency/worker.js
View File

@@ -1,19 +1,24 @@
importScripts(
(new URL(self.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js'
);
self.sqlite3InitModule().then(async function(sqlite3){
const urlArgs = new URL(self.location.href).searchParams;
const options = {
workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000),
unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/
};
'use strict';
const urlArgs = new URL(globalThis.location.href).searchParams;
const options = {
workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000),
unlockAsap: urlArgs.get('opfs-unlock-asap') || 0,
vfs: urlArgs.get('vfs')
};
const jsSqlite = urlArgs.get('sqlite3.dir')
+'/sqlite3.js?opfs-async-proxy-id='
+options.workerName;
importScripts(jsSqlite)/*Sigh - URL args are not propagated this way*/;
//const sqlite3InitModule = (await import(jsSqlite)).default;
globalThis.sqlite3InitModule.__isUnderTest = true;
globalThis.sqlite3InitModule().then(async function(sqlite3){
const wPost = (type,...payload)=>{
postMessage({type, worker: options.workerName, payload});
};
const stdout = (...args)=>wPost('stdout',...args);
const stderr = (...args)=>wPost('stderr',...args);
if(!sqlite3.opfs){
stderr("OPFS support not detected. Aborting.");
stderr("This code requires the (private) sqlite3.opfs object. Aborting.");
return;
}
@@ -34,80 +39,107 @@ self.sqlite3InitModule().then(async function(sqlite3){
count: 0
});
const finish = ()=>{
if(db){
if(!db.pointer) return;
if(db?.pointer){
db.close();
}
if(interval.error){
wPost('failed',"Ending work after interval #"+interval.count,
stderr("Ending work at interval #"+interval.count,
"due to error:", interval.error);
wPost('failed', "at interval #"+interval.count,
"due to error:",interval.error);
}else{
wPost('finished',"Ending work after",interval.count,"intervals.");
}
};
const run = async function(){
db = new sqlite3.oo1.OpfsDb({
filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap,
flags: 'c'
const Ctors = Object.assign(Object.create(null),{
opfs: sqlite3.oo1.OpfsDb,
'opfs-wl': sqlite3.oo1.OpfsWlDb
});
const ctor = Ctors[options.vfs];
if( !ctor ){
stderr("Invalid VFS name:",vfs);
return;
}
stdout("ctor:", options.vfs);
while(true){
try{
db = new ctor({
filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap,
flags: 'c'
});
break;
}catch(e){
if(e instanceof sqlite3.SQLite3Error
&& sqlite3.capi.SQLITE_BUSY===e.resultCode){
stderr("Retrying open() for BUSY:",e.message);
continue;
}
throw e;
}
}
sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000);
db.transaction((db)=>{
db.exec([
"create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);",
"create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);"
]);
sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{
db.transaction((db)=>{
db.exec([
"create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);",
"create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);"
]);
});
});
const maxIterations =
urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10;
stdout("Starting interval-based db updates with delay of",interval.delay,"ms.");
stdout("Starting",maxIterations,"interval-based db updates with delay of",interval.delay,"ms.");
const doWork = async ()=>{
const tm = new Date().getTime();
++interval.count;
const prefix = "v(#"+interval.count+")";
stdout("Setting",prefix,"=",tm);
try{
db.exec({
sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)",
bind: [options.workerName, new Date().getTime()]
sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{
db.exec({
sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)",
bind: [options.workerName, new Date().getTime()]
});
}, (n)=>{
stderr("Attempt #",n,"for interval #",interval.count,"due to BUSY");
});
//stdout("Set",prefix);
}catch(e){
stderr("Error: ",e.message);
interval.error = e;
throw e;
}
//stdout("doWork()",prefix,"error ",interval.error);
};
if(1){/*use setInterval()*/
setTimeout(async function timer(){
await doWork();
if(interval.error || maxIterations === interval.count){
finish();
}else{
setTimeout(timer, interval.delay);
}
}, interval.delay);
}else{
/*This approach provides no concurrency whatsoever: each worker
is run to completion before any others can work.*/
let i;
for(i = 0; i < maxIterations; ++i){
await doWork();
if(interval.error) break;
await wait(interval.ms);
setTimeout(async function timer(){
await doWork();
if(interval.error || maxIterations === interval.count){
finish();
}else{
const jitter = (Math.random()>=0.5)
? ((Math.random()*100 - Math.random()*100) | 0)
: 0;
const n = interval.delay + jitter;
setTimeout(timer, n<50 ? interval.delay : n);
}
finish();
}
}, interval.delay);
}/*run()*/;
self.onmessage = function({data}){
globalThis.onmessage = function({data}){
switch(data.type){
case 'run': run().catch((e)=>{
case 'run':
run().catch((e)=>{
if(!interval.error) interval.error = e;
}).catch(e=>{
/* Don't do this in finally() - it fires too soon. */
finish();
throw e;
});
break;
default:
stderr("Unhandled message type '"+data.type+"'.");
break;
break;
default:
stderr("Unhandled message type '"+data.type+"'.");
break;
}
};
});

40
manifest
View File

@@ -1,5 +1,5 @@
C Add\sparens\saround\san\sexpression\sin\sthe\sCLI\sshell\sto\saccount\sfor\soperator\sprecedence,\sas\sreported\sin\s[forum:856ff7a2f1|forum\spost\s856ff7a2f1].
D 2026-03-08T07:02:46.194
C Merge\sopfs-wl\sinto\strunk,\snot\sso\smuch\sfor\sthe\s"opfs-wl"\sVFS,\swhich\sis\sstil\stentative\s(working\sfine\sbut\swith\slittle\sperceived\sbenefit),\sbut\sfor\sthe\s"opfs"\sVFS\scleanups\swhich\snecessarily\shappened\saround\sit.
D 2026-03-08T08:32:22.078
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -572,7 +572,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e
F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07
F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a
F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6
F ext/wasm/GNUmakefile 79236447d750609aa6beda30feec1314180c5462a493ad94214122887232bfd4
F ext/wasm/GNUmakefile 4496669ca5785c7bfc7af76565f1dd45e8f9ec7529b58c485fd374088162bef1
F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a
F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259
F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff
@@ -584,19 +584,22 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 6ad5ace0a16b3300e8d40d4e596eee1c773424e22
F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81
F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930
F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41
F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76
F ext/wasm/api/opfs-common-shared.c-pp.js 49f4b044dfb3becd8c926ef42b7494da680e5ac878c6d10ef510b66f74ad55a1
F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b
F ext/wasm/api/post-js-header.js d24bd0d065f3489c8b78ddf3ead6321e5d047187a162cd503c41700e03dd1f06
F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55
F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f
F ext/wasm/api/sqlite3-api-glue.c-pp.js 7afb3da3510facafd94ce31133ec847d0d4db5b2b5e4325941803fd3bca07c16
F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa
F ext/wasm/api/sqlite3-api-prologue.js 2eac3d601c8a824fda1529f9eb894518672d3e575b49c0601094afda442b294f
F ext/wasm/api/sqlite3-api-prologue.js db1ba33ca7092f12d5bb6cb615c78c457432be4a1d26785686993a8909950d72
F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938
F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc
F ext/wasm/api/sqlite3-opfs-async-proxy.js 92d6d327a862f1627ff3e88e60fdfea9def06ad539d98929ba46490e64372736
F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js c2236c9037ccac368be6eeed3e5f6ed3cefed0d20539c9092139f27a76aaf1ba w ext/wasm/api/sqlite3-opfs-async-proxy.js
F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js f907fa712d1c9d98a3c08393f3e97a664857782da11670e129d39e383f45f968
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js cf5fba74978d3e0983f68505d9695ee7836853855590a92ccc5a96e27db5350b
F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 743650cfefed0df900171971065c2d618e5652177a98844ddb6695c70149968e
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4b02d7063d00e2d4f42d4ca79a8a0bca1b275257e135bc154d35b701ed1d234a
F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81
F ext/wasm/api/sqlite3-wasm.c ddf9d435b2e901eaceb805ff694e9609c2f32b5cf89a4f164e734a6fa303fdd2
F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66
@@ -621,11 +624,11 @@ F ext/wasm/fiddle/fiddle-worker.js 6c72acac2d381480bc9f5eb538e3f2faf2c1f72dd4fcb
F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893
F ext/wasm/fiddle/index.c-pp.html 72c7e5517217960b3809648429ea396a7cbad0ffb2c92f6a2f5703abecb27317
F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290
F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb5f1756
F ext/wasm/index.html 5bf6cf1b0a3c8b9f5f54d77f2219d7ae87a15162055ce308109c49b1dcab4239
F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b
F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41
F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x
F ext/wasm/mkwasmbuilds.c 0e9198eb90acae4bcf57cf62d7186f6af5aaac02efdb075a1aded33614b3805a
F ext/wasm/mkwasmbuilds.c d4479af2774a43104b819d9286961b7c09793ca39440e20191e693ceff21f911
F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2
F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96
F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
@@ -641,10 +644,10 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c
F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244
F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb
F ext/wasm/tester1.c-pp.js 1c81b2b3f6c02adbb50ff48e07645946cd066cd215c0b8b500574fefecacf5c8
F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e
F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88
F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
F ext/wasm/tester1.c-pp.js e26e78fe450375b43a88a5da41946b8997612201a592770f19ec827b2bf96cc0
F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226
F ext/wasm/tests/opfs/concurrency/test.js 46c772bc18abb0fcbb058d57b5aaee9e7938f948ecdd802c6ca0850ad3519f92
F ext/wasm/tests/opfs/concurrency/worker.js 4edeec8d88f14c9603c9d4e32ce379ce1d5394f3c5e820776e26a603d600b3e3
F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79
F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8
F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328
@@ -2189,8 +2192,9 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee
F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c
P 0c5d4fe0bd322b4c8dbd0800001317a21cd511b00244de74b4db3fd051153879
R d722ffeb4d76b155994400ff67e06b8f
P da3ad170aebf57f2202e56673ac149bc51077d126ab5928f1db16492e403d5b9 616d32c8380ef47c5a7c2eb765e7b6fbec9500d4e16c837f4ee11d71c239a55f
R 7fa9d9bc620afcdc086430a72b380676
T +closed 616d32c8380ef47c5a7c2eb765e7b6fbec9500d4e16c837f4ee11d71c239a55f Closed\sby\sintegrate-merge.
U stephan
Z f680be3a93bb03be191e09867e313221
Z eb8fc8dc10da8c389057ea48d94a3dd6
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@@ -1 +1 @@
da3ad170aebf57f2202e56673ac149bc51077d126ab5928f1db16492e403d5b9
bcaa876c6dafee0995d239f59b6ce7810b6f5faa219b9c392966211ae3dec17a