database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2026-10-23 21:49:02 +03:00
Code Activity

Consolidate much of the OPFS utility code into a new file for use by two of the OPFS VFSes.

Browse Source 
FossilOrigin-Name: db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7
This commit is contained in:
stephan
2026-03-04 11:37:39 +00:00
parent 6d6bf30735
commit 96daad10fa
8 changed files with 544 additions and 892 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ext/wasm/GNUmakefile
View File

@@ -911,6 +911,7 @@ ifeq (0,$(wasm-bare-bones))
sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js
endif
sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js
sqlite3-api.jses += $(dir.api)/opfs-common-shared.c-pp.js
sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js
sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js
sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js

22
ext/wasm/api/opfs-common.c-pp.js → ext/wasm/api/opfs-common-inline.c-pp.js
View File

@@ -1,26 +1,14 @@
//#if nope
/**
This file is for preprocessor #include into the "opfs" and
"opfs-wl" impls, as well as their async-proxy part.
"opfs-wl" impls, as well as their async-proxy part. It must be
inlined in those files, as opposed to being a shared copy in the
library, because (A) the async proxy does not load the library and
(B) it references an object which is local to each of those files
but which has a 99% identical structure for each.
*/
//#endif
//#if not defined opfs-async-proxy
/**
TODO: move the sqlite3.opfs (private/internal) namespace object
init from sqlite3-vfs-opfs*.js into here. That namespace gets
removed from the sqlite3 namespace in the final stages of library
bootstrapping except in test runs, where it's retained so that
tests can clean up OPFS so their test cases work (the down-side of
them being persistent).
*/
//#endif not defined opfs-async-proxy
// This function won't work as-is if we #include it, but the
// missing elements have not yet been identified.
const initS11n = function(){
/* This function is needed by the "opfs" and "opfs-wl" VFSes
and sqlite-opfs-async-proxy.js (used by those of those). */
/**
This proxy de/serializes cross-thread function arguments and
output-pointer values via the state.sabIO SharedArrayBuffer,

430
ext/wasm/api/opfs-common-shared.c-pp.js Normal file
View File

@@ -0,0 +1,430 @@
//#if not target:node
/*
2026-03-04
The author disclaims copyright to this source code. In place of a
legal notice, here is a blessing:
* May you do good and not evil.
* May you find forgiveness for yourself and forgive others.
* May you share freely, never taking more than you give.
***********************************************************************
This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js. It
creates a private/internal sqlite3.opfs namespace common to the two
and used (only) by them and the test framework. It is not part of
the public API.
*/
globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
'use strict';
const toss = sqlite3.util.toss;
const toss3 = sqlite3.util.toss3;
const capi = sqlite3.capi;
const util = sqlite3.util;
const wasm = sqlite3.wasm;
/**
Generic utilities for working with OPFS. This will get filled out
by the Promise setup and, on success, installed as sqlite3.opfs.
This is an internal/private namespace intended for use solely
by the OPFS VFSes and test code for them. The library bootstrapping
process removes this object in non-testing contexts.
*/
const opfsUtil = sqlite3.opfs = Object.create(null);
/**
Returns true if _this_ thread has access to the OPFS APIs.
*/
opfsUtil.thisThreadHasOPFS = ()=>{
return globalThis.FileSystemHandle &&
globalThis.FileSystemDirectoryHandle &&
globalThis.FileSystemFileHandle &&
globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle &&
navigator?.storage?.getDirectory;
};
/**
Must be called by the OPFS VFSes immediately after they determine
whether OPFS is available by calling
thisThreadHasOPFS(). Resolves to the OPFS storage root directory
and sets opfsUtil.rootDirectory to that value.
*/
opfsUtil.getRootDir = async function f(){
return f.promise ??= navigator.storage.getDirectory().then(d=>{
opfsUtil.rootDirectory = d;
return d;
}).catch(e=>{
delete f.promise;
throw e;
});
};
/**
Expects an OPFS file path. It gets resolved, such that ".."
components are properly expanded, and returned. If the 2nd arg
is true, the result is returned as an array of path elements,
else an absolute path string is returned.
*/
opfsUtil.getResolvedPath = function(filename,splitIt){
const p = new URL(filename, "file://irrelevant").pathname;
return splitIt ? p.split('/').filter((v)=>!!v) : p;
};
/**
Takes the absolute path to a filesystem element. Returns an
array of [handleOfContainingDir, filename]. If the 2nd argument
is truthy then each directory element leading to the file is
created along the way. Throws if any creation or resolution
fails.
*/
opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){
const path = opfsUtil.getResolvedPath(absFilename, true);
const filename = path.pop();
let dh = await opfsUtil.getRootDir();
for(const dirName of path){
if(dirName){
dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
}
}
return [dh, filename];
};
/**
Creates the given directory name, recursively, in
the OPFS filesystem. Returns true if it succeeds or the
directory already exists, else false.
*/
opfsUtil.mkdir = async function(absDirName){
try {
await opfsUtil.getDirForFilename(absDirName+"/filepart", true);
return true;
}catch(e){
//sqlite3.config.warn("mkdir(",absDirName,") failed:",e);
return false;
}
};
/**
Checks whether the given OPFS filesystem entry exists,
returning true if it does, false if it doesn't or if an
exception is intercepted while trying to make the
determination.
*/
opfsUtil.entryExists = async function(fsEntryName){
try {
const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName);
await dh.getFileHandle(fn);
return true;
}catch(e){
return false;
}
};
/**
Generates a random ASCII string len characters long, intended for
use as a temporary file name.
*/
opfsUtil.randomFilename = function f(len=16){
if(!f._chars){
f._chars = "abcdefghijklmnopqrstuvwxyz"+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
"012346789";
f._n = f._chars.length;
}
const a = [];
let i = 0;
for( ; i < len; ++i){
const ndx = Math.random() * (f._n * 64) % f._n | 0;
a[i] = f._chars[ndx];
}
return a.join("");
/*
An alternative impl. with an unpredictable length
but much simpler:
Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36)
*/
};
/**
Returns a promise which resolves to an object which represents
all files and directories in the OPFS tree. The top-most object
has two properties: `dirs` is an array of directory entries
(described below) and `files` is a list of file names for all
files in that directory.
Traversal starts at sqlite3.opfs.rootDirectory.
Each `dirs` entry is an object in this form:
```
{ name: directoryName,
dirs: [...subdirs],
files: [...file names]
}
```
The `files` and `subdirs` entries are always set but may be
empty arrays.
The returned object has the same structure but its `name` is
an empty string. All returned objects are created with
Object.create(null), so have no prototype.
Design note: the entries do not contain more information,
e.g. file sizes, because getting such info is not only
expensive but is subject to locking-related errors.
*/
opfsUtil.treeList = async function(){
const doDir = async function callee(dirHandle,tgt){
tgt.name = dirHandle.name;
tgt.dirs = [];
tgt.files = [];
for await (const handle of dirHandle.values()){
if('directory' === handle.kind){
const subDir = Object.create(null);
tgt.dirs.push(subDir);
await callee(handle, subDir);
}else{
tgt.files.push(handle.name);
}
}
};
const root = Object.create(null);
const dir = await opfsUtil.getRootDir();
await doDir(dir, root);
return root;
};
/**
Irrevocably deletes _all_ files in the current origin's OPFS.
Obviously, this must be used with great caution. It may throw
an exception if removal of anything fails (e.g. a file is
locked), but the precise conditions under which the underlying
APIs will throw are not documented (so we cannot tell you what
they are).
*/
opfsUtil.rmfr = async function(){
const rd = await opfsUtil.getRootDir();
const dir = rd, opt = {recurse: true};
for await (const handle of dir.values()){
dir.removeEntry(handle.name, opt);
}
};
/**
Deletes the given OPFS filesystem entry. As this environment
has no notion of "current directory", the given name must be an
absolute path. If the 2nd argument is truthy, deletion is
recursive (use with caution!).
The returned Promise resolves to true if the deletion was
successful, else false (but...). The OPFS API reports the
reason for the failure only in human-readable form, not
exceptions which can be type-checked to determine the
failure. Because of that...
If the final argument is truthy then this function will
propagate any exception on error, rather than returning false.
*/
opfsUtil.unlink = async function(fsEntryName, recursive = false,
throwOnError = false){
try {
const [hDir, filenamePart] =
await opfsUtil.getDirForFilename(fsEntryName, false);
await hDir.removeEntry(filenamePart, {recursive});
return true;
}catch(e){
if(throwOnError){
throw new Error("unlink(",arguments[0],") failed: "+e.message,{
cause: e
});
}
return false;
}
};
/**
Traverses the OPFS filesystem, calling a callback for each
entry. The argument may be either a callback function or an
options object with any of the following properties:
- `callback`: function which gets called for each filesystem
entry. It gets passed 3 arguments: 1) the
FileSystemFileHandle or FileSystemDirectoryHandle of each
entry (noting that both are instanceof FileSystemHandle). 2)
the FileSystemDirectoryHandle of the parent directory. 3) the
current depth level, with 0 being at the top of the tree
relative to the starting directory. If the callback returns a
literal false, as opposed to any other falsy value, traversal
stops without an error. Any exceptions it throws are
propagated. Results are undefined if the callback manipulate
the filesystem (e.g. removing or adding entries) because the
how OPFS iterators behave in the face of such changes is
undocumented.
- `recursive` [bool=true]: specifies whether to recurse into
subdirectories or not. Whether recursion is depth-first or
breadth-first is unspecified!
- `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory]
specifies the starting directory.
If this function is passed a function, it is assumed to be the
callback.
Returns a promise because it has to (by virtue of being async)
but that promise has no specific meaning: the traversal it
performs is synchronous. The promise must be used to catch any
exceptions propagated by the callback, however.
*/
opfsUtil.traverse = async function(opt){
const defaultOpt = {
recursive: true,
directory: await opfsUtil.getRootDir()
};
if('function'===typeof opt){
opt = {callback:opt};
}
opt = Object.assign(defaultOpt, opt||{});
const doDir = async function callee(dirHandle, depth){
for await (const handle of dirHandle.values()){
if(false === opt.callback(handle, dirHandle, depth)) return false;
else if(opt.recursive && 'directory' === handle.kind){
if(false === await callee(handle, depth + 1)) break;
}
}
};
doDir(opt.directory, 0);
};
/**
Impl of opfsUtil.importDb() when it's given a function as its
second argument.
*/
const importDbChunked = async function(filename, callback){
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
let sah = await hFile.createSyncAccessHandle();
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
sah.truncate(0);
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/);
return nWrote;
}catch(e){
await sah.close();
sah = undefined;
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally {
if( sah ) await sah.close();
}
};
/**
Asynchronously imports the given bytes (a byte array or
ArrayBuffer) into the given database file.
Results are undefined if the given db name refers to an opened
db.
If passed a function for its second argument, its behaviour
changes: imports its data in chunks fed to it by the given
callback function. It calls the callback (which may be async)
repeatedly, expecting either a Uint8Array or ArrayBuffer (to
denote new input) or undefined (to denote EOF). For so long as
the callback continues to return non-undefined, it will append
incoming data to the given VFS-hosted database file. When
called this way, the resolved value of the returned Promise is
the number of bytes written to the target file.
It very specifically requires the input to be an SQLite3
database and throws if that's not the case. It does so in
order to prevent this function from taking on a larger scope
than it is specifically intended to. i.e. we do not want it to
become a convenience for importing arbitrary files into OPFS.
This routine rewrites the database header bytes in the output
file (not the input array) to force disabling of WAL mode.
On error this throws and the state of the input file is
undefined (it depends on where the exception was triggered).
On success, resolves to the number of bytes written.
*/
opfsUtil.importDb = async function(filename, bytes){
if( bytes instanceof Function ){
return importDbChunked(filename, bytes);
}
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
util.affirmIsDb(bytes);
const n = bytes.byteLength;
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
let sah, err, nWrote = 0;
try {
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
nWrote = sah.write(bytes, {at: 0});
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}
sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */;
return nWrote;
}catch(e){
if( sah ){ await sah.close(); sah = undefined; }
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally{
if( sah ) await sah.close();
}
};
/**
Checks for features required for OPFS VFSes and throws with a
descriptive error message if they're not found. This is intended
to be run as part of async VFS installation steps.
*/
opfsUtil.vfsInstallationFeatureCheck = function(){
if(!globalThis.SharedArrayBuffer
|| !globalThis.Atomics){
throw new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+
"The server must emit the COOP/COEP response headers to enable those. "+
"See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep");
}else if('undefined'===typeof WorkerGlobalScope){
throw new Error("The OPFS sqlite3_vfs cannot run in the main thread "+
"because it requires Atomics.wait().");
}else if(!globalThis.FileSystemHandle ||
!globalThis.FileSystemDirectoryHandle ||
!globalThis.FileSystemFileHandle ||
!globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle ||
!navigator?.storage?.getDirectory){
throw new newError("Missing required OPFS APIs.");
}
};
}/*sqlite3ApiBootstrap.initializers*/);
//#else
/*
The OPFS SAH Pool parts are elided from builds targeting node.js.
*/
//#endif target:node

13
ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js
View File

@@ -46,19 +46,16 @@
theFunc().then(...) is not compatible with the change to
synchronous, but we do do not use those APIs that way. i.e. we don't
_need_ to change anything for this, but at some point (after Chrome
versions (approximately) 104-107 are extinct) should change our
versions (approximately) 104-107 are extinct) we should change our
usage of those methods to remove the "await".
*/
"use strict";
const wPost = (type,...args)=>postMessage({type, payload:args});
//#if nope
const urlParams = new URL(globalThis.location.href).searchParams;
if( !urlParams.has('vfs') ){
throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker");
}
const isWebLocker = 'opfs-wl'===urlParams.get('vfs');
const msgKeyPrefix = 'opfs-'; //isWebLocker ? 'opfs-wl-' : 'opfs-';
//#endif
const installAsyncProxy = function(){
const toss = function(...args){throw new Error(args.join(' '))};
if(globalThis.window === globalThis){
@@ -73,6 +70,9 @@ const installAsyncProxy = function(){
this API.
*/
const state = Object.create(null);
//#define opfs-async-proxy
//#include api/opfs-common-inline.c-pp.js
//#undef opfs-async-proxy
/**
verbose:
@@ -326,7 +326,7 @@ const installAsyncProxy = function(){
}
log("Got",opName+"() sync handle for",fh.filenameAbs,
'in',performance.now() - t,'ms');
if(!fh.xLock && !state.lock/*set by opfs-wl*/){
if(!isWebLocker && !fh.xLock){
__implicitLocks.add(fh.fid);
log("Acquired implicit lock for",opName+"()",fh.fid,fh.filenameAbs);
}
@@ -611,9 +611,6 @@ const installAsyncProxy = function(){
}
}/*vfsAsyncImpls*/;
//#define opfs-async-proxy
//#include api/opfs-common.c-pp.js
/**
Starts a new WebLock request.
*/

481
ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js
View File

@@ -20,12 +20,7 @@
This file is intended to be appended to the main sqlite3 JS
deliverable somewhere after sqlite3-api-oo1.js.
TODOs (2026-0303):
- Move pieces of this file which are common to it,
sqlite3-vfs-opfs.c-pp.js, and/or sqlite3-opfs-async-proxy.js into
separate files and #include them in each using the preprocessor.
e.g. the s11n namespace object is duplicated in all three files.
TODOs (2026-03-03):
- For purposes of tester1.js we need to figure out which of these
VFSes will install the (internal-use-only) sqlite3.opfs utility code
@@ -39,8 +34,11 @@
*/
'use strict';
globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs")
/* Gets removed from sqlite3 during bootstrap, so we need an
early reference to it. */;
/**
installOpfsVfs() returns a Promise which, on success, installs an
installOpfsWlVfs() returns a Promise which, on success, installs an
sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs
which accept a VFS. It is intended to be called via
sqlite3ApiBootstrap.initializers or an equivalent mechanism.
@@ -67,27 +65,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
On success, the Promise resolves to the top-most sqlite3 namespace
object.
*/
const installOpfsVfs = function callee(options){
if(!globalThis.SharedArrayBuffer
|| !globalThis.Atomics){
return Promise.reject(
new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+
"The server must emit the COOP/COEP response headers to enable those. "+
"See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep")
);
}else if('undefined'===typeof WorkerGlobalScope){
return Promise.reject(
new Error("The OPFS sqlite3_vfs cannot run in the main thread "+
"because it requires Atomics.wait().")
);
}else if(!globalThis.FileSystemHandle ||
!globalThis.FileSystemDirectoryHandle ||
!globalThis.FileSystemFileHandle ||
!globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle ||
!navigator?.storage?.getDirectory){
return Promise.reject(
new Error("Missing required OPFS APIs.")
);
const installOpfsWlVfs = function callee(options){
try{
opfsUtil.vfsInstallationFeatureCheck();
}catch(e){
return Promise.reject(e);
}
const nu = (...obj)=>Object.assign(Object.create(null),...obj);
options = nu(options);
@@ -129,31 +111,34 @@ const installOpfsVfs = function callee(options){
const sqlite3_file = capi.sqlite3_file;
const sqlite3_io_methods = capi.sqlite3_io_methods;
/**
Generic utilities for working with OPFS. This will get filled out
by the Promise setup and, on success, installed as sqlite3.opfs.
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
objects. After the worker's "inited" message arrives, other types
of data may be added to it.
ACHTUNG: do not rely on these APIs in client code. They are
experimental and subject to change or removal as the
OPFS-specific sqlite3_vfs evolves.
*/
const opfsUtil = Object.create(null);
For purposes of Atomics.wait() and Atomics.notify(), we use a
SharedArrayBuffer with one slot reserved for each of the API
proxy's methods. The sync side of the API uses Atomics.wait()
on the corresponding slot and the async side uses
Atomics.notify() on that slot.
/**
Returns true if _this_ thread has access to the OPFS APIs.
The approach of using a single SAB to serialize comms for all
instances might(?) lead to deadlock situations in multi-db
cases. We should probably have one SAB here with a single slot
for locking a per-file initialization step and then allocate a
separate SAB like the above one for each file. That will
require a bit of acrobatics but should be feasible. The most
problematic part is that xOpen() would have to use
postMessage() to communicate its SharedArrayBuffer, and mixing
that approach with Atomics.wait/notify() gets a bit messy.
*/
const thisThreadHasOPFS = ()=>{
return globalThis.FileSystemHandle &&
globalThis.FileSystemDirectoryHandle &&
globalThis.FileSystemFileHandle &&
globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle &&
navigator?.storage?.getDirectory;
};
const state = Object.create(null);
const metrics = Object.create(null);
//#define opfs-has-metrics
//#include api/opfs-common-inline.c-pp.js
//#undef opfs-has-metrics
/**
Not part of the public API. Solely for internal/development
use.
*/
opfsUtil.metrics = {
const vfsMetrics = {
dump: function(){
let k, n = 0, t = 0, w = 0;
for(k in state.opIds){
@@ -254,29 +239,7 @@ const installOpfsVfs = function callee(options){
environment or the other when sqlite3_os_end() is called (_if_ it
gets called at all in a wasm build, which is undefined).
*/
/**
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
objects. After the worker's "inited" message arrives, other types
of data may be added to it.
For purposes of Atomics.wait() and Atomics.notify(), we use a
SharedArrayBuffer with one slot reserved for each of the API
proxy's methods. The sync side of the API uses Atomics.wait()
on the corresponding slot and the async side uses
Atomics.notify() on that slot.
The approach of using a single SAB to serialize comms for all
instances might(?) lead to deadlock situations in multi-db
cases. We should probably have one SAB here with a single slot
for locking a per-file initialization step and then allocate a
separate SAB like the above one for each file. That will
require a bit of acrobatics but should be feasible. The most
problematic part is that xOpen() would have to use
postMessage() to communicate its SharedArrayBuffer, and mixing
that approach with Atomics.wait/notify() gets a bit messy.
*/
const state = Object.create(null);
state.verbose = options.verbose;
state.littleEndian = (()=>{
const buffer = new ArrayBuffer(2);
@@ -331,7 +294,6 @@ const installOpfsVfs = function callee(options){
+ state.sabS11nSize/* argument/result serialization block */
);
state.opIds = Object.create(null);
const metrics = Object.create(null);
{
/* Indexes for use in our SharedArrayBuffer... */
let i = 0;
@@ -378,7 +340,7 @@ const installOpfsVfs = function callee(options){
state.sabOP = new SharedArrayBuffer(
i * 4/* ==sizeof int32, noting that Atomics.wait() and friends
can only function on Int32Array views of an SAB. */);
opfsUtil.metrics.reset();
vfsMetrics.reset();
}
/**
SQLITE_xxx constants to export to the async worker
@@ -499,6 +461,7 @@ const installOpfsVfs = function callee(options){
return rc;
};
//#if nope
/**
Not part of the public API. Only for test/development use.
*/
@@ -512,35 +475,7 @@ const installOpfsVfs = function callee(options){
W.postMessage({type: 'opfs-async-restart'});
}
};
//#define opfs-has-metrics
//#include api/opfs-common.c-pp.js
/**
Generates a random ASCII string len characters long, intended for
use as a temporary file name.
*/
const randomFilename = function f(len=16){
if(!f._chars){
f._chars = "abcdefghijklmnopqrstuvwxyz"+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
"012346789";
f._n = f._chars.length;
}
const a = [];
let i = 0;
for( ; i < len; ++i){
const ndx = Math.random() * (f._n * 64) % f._n | 0;
a[i] = f._chars[ndx];
}
return a.join("");
/*
An alternative impl. with an unpredictable length
but much simpler:
Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36)
*/
};
//#endif
/**
Map of sqlite3_file pointers to objects constructed by xOpen().
@@ -770,7 +705,7 @@ const installOpfsVfs = function callee(options){
mTimeStart('xOpen');
let opfsFlags = 0;
if(0===zName){
zName = randomFilename();
zName = opfsUtil.randomFilename();
}else if(wasm.isPtr(zName)){
if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){
/* -----------------------^^^^^ MUST pass the untranslated
@@ -832,322 +767,6 @@ const installOpfsVfs = function callee(options){
};
}
/**
Expects an OPFS file path. It gets resolved, such that ".."
components are properly expanded, and returned. If the 2nd arg
is true, the result is returned as an array of path elements,
else an absolute path string is returned.
*/
opfsUtil.getResolvedPath = function(filename,splitIt){
const p = new URL(filename, "file://irrelevant").pathname;
return splitIt ? p.split('/').filter((v)=>!!v) : p;
};
/**
Takes the absolute path to a filesystem element. Returns an
array of [handleOfContainingDir, filename]. If the 2nd argument
is truthy then each directory element leading to the file is
created along the way. Throws if any creation or resolution
fails.
*/
opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){
const path = opfsUtil.getResolvedPath(absFilename, true);
const filename = path.pop();
let dh = opfsUtil.rootDirectory;
for(const dirName of path){
if(dirName){
dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
}
}
return [dh, filename];
};
/**
Creates the given directory name, recursively, in
the OPFS filesystem. Returns true if it succeeds or the
directory already exists, else false.
*/
opfsUtil.mkdir = async function(absDirName){
try {
await opfsUtil.getDirForFilename(absDirName+"/filepart", true);
return true;
}catch(e){
//sqlite3.config.warn("mkdir(",absDirName,") failed:",e);
return false;
}
};
/**
Checks whether the given OPFS filesystem entry exists,
returning true if it does, false if it doesn't or if an
exception is intercepted while trying to make the
determination.
*/
opfsUtil.entryExists = async function(fsEntryName){
try {
const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName);
await dh.getFileHandle(fn);
return true;
}catch(e){
return false;
}
};
/**
Generates a random ASCII string, intended for use as a
temporary file name. Its argument is the length of the string,
defaulting to 16.
*/
opfsUtil.randomFilename = randomFilename;
/**
Returns a promise which resolves to an object which represents
all files and directories in the OPFS tree. The top-most object
has two properties: `dirs` is an array of directory entries
(described below) and `files` is a list of file names for all
files in that directory.
Traversal starts at sqlite3.opfs.rootDirectory.
Each `dirs` entry is an object in this form:
```
{ name: directoryName,
dirs: [...subdirs],
files: [...file names]
}
```
The `files` and `subdirs` entries are always set but may be
empty arrays.
The returned object has the same structure but its `name` is
an empty string. All returned objects are created with
Object.create(null), so have no prototype.
Design note: the entries do not contain more information,
e.g. file sizes, because getting such info is not only
expensive but is subject to locking-related errors.
*/
opfsUtil.treeList = async function(){
const doDir = async function callee(dirHandle,tgt){
tgt.name = dirHandle.name;
tgt.dirs = [];
tgt.files = [];
for await (const handle of dirHandle.values()){
if('directory' === handle.kind){
const subDir = Object.create(null);
tgt.dirs.push(subDir);
await callee(handle, subDir);
}else{
tgt.files.push(handle.name);
}
}
};
const root = Object.create(null);
await doDir(opfsUtil.rootDirectory, root);
return root;
};
/**
Irrevocably deletes _all_ files in the current origin's OPFS.
Obviously, this must be used with great caution. It may throw
an exception if removal of anything fails (e.g. a file is
locked), but the precise conditions under which the underlying
APIs will throw are not documented (so we cannot tell you what
they are).
*/
opfsUtil.rmfr = async function(){
const dir = opfsUtil.rootDirectory, opt = {recurse: true};
for await (const handle of dir.values()){
dir.removeEntry(handle.name, opt);
}
};
/**
Deletes the given OPFS filesystem entry. As this environment
has no notion of "current directory", the given name must be an
absolute path. If the 2nd argument is truthy, deletion is
recursive (use with caution!).
The returned Promise resolves to true if the deletion was
successful, else false (but...). The OPFS API reports the
reason for the failure only in human-readable form, not
exceptions which can be type-checked to determine the
failure. Because of that...
If the final argument is truthy then this function will
propagate any exception on error, rather than returning false.
*/
opfsUtil.unlink = async function(fsEntryName, recursive = false,
throwOnError = false){
try {
const [hDir, filenamePart] =
await opfsUtil.getDirForFilename(fsEntryName, false);
await hDir.removeEntry(filenamePart, {recursive});
return true;
}catch(e){
if(throwOnError){
throw new Error("unlink(",arguments[0],") failed: "+e.message,{
cause: e
});
}
return false;
}
};
/**
Traverses the OPFS filesystem, calling a callback for each
entry. The argument may be either a callback function or an
options object with any of the following properties:
- `callback`: function which gets called for each filesystem
entry. It gets passed 3 arguments: 1) the
FileSystemFileHandle or FileSystemDirectoryHandle of each
entry (noting that both are instanceof FileSystemHandle). 2)
the FileSystemDirectoryHandle of the parent directory. 3) the
current depth level, with 0 being at the top of the tree
relative to the starting directory. If the callback returns a
literal false, as opposed to any other falsy value, traversal
stops without an error. Any exceptions it throws are
propagated. Results are undefined if the callback manipulate
the filesystem (e.g. removing or adding entries) because the
how OPFS iterators behave in the face of such changes is
undocumented.
- `recursive` [bool=true]: specifies whether to recurse into
subdirectories or not. Whether recursion is depth-first or
breadth-first is unspecified!
- `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory]
specifies the starting directory.
If this function is passed a function, it is assumed to be the
callback.
Returns a promise because it has to (by virtue of being async)
but that promise has no specific meaning: the traversal it
performs is synchronous. The promise must be used to catch any
exceptions propagated by the callback, however.
*/
opfsUtil.traverse = async function(opt){
const defaultOpt = {
recursive: true,
directory: opfsUtil.rootDirectory
};
if('function'===typeof opt){
opt = {callback:opt};
}
opt = Object.assign(defaultOpt, opt||{});
const doDir = async function callee(dirHandle, depth){
for await (const handle of dirHandle.values()){
if(false === opt.callback(handle, dirHandle, depth)) return false;
else if(opt.recursive && 'directory' === handle.kind){
if(false === await callee(handle, depth + 1)) break;
}
}
};
doDir(opt.directory, 0);
};
/**
impl of importDb() when it's given a function as its second
argument.
*/
const importDbChunked = async function(filename, callback){
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
let sah = await hFile.createSyncAccessHandle();
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
sah.truncate(0);
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/);
return nWrote;
}catch(e){
await sah.close();
sah = undefined;
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally {
if( sah ) await sah.close();
}
};
/**
Asynchronously imports the given bytes (a byte array or
ArrayBuffer) into the given database file.
Results are undefined if the given db name refers to an opened
db.
If passed a function for its second argument, its behaviour
changes: imports its data in chunks fed to it by the given
callback function. It calls the callback (which may be async)
repeatedly, expecting either a Uint8Array or ArrayBuffer (to
denote new input) or undefined (to denote EOF). For so long as
the callback continues to return non-undefined, it will append
incoming data to the given VFS-hosted database file. When
called this way, the resolved value of the returned Promise is
the number of bytes written to the target file.
It very specifically requires the input to be an SQLite3
database and throws if that's not the case. It does so in
order to prevent this function from taking on a larger scope
than it is specifically intended to. i.e. we do not want it to
become a convenience for importing arbitrary files into OPFS.
This routine rewrites the database header bytes in the output
file (not the input array) to force disabling of WAL mode.
On error this throws and the state of the input file is
undefined (it depends on where the exception was triggered).
On success, resolves to the number of bytes written.
*/
opfsUtil.importDb = async function(filename, bytes){
if( bytes instanceof Function ){
return importDbChunked(filename, bytes);
}
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
util.affirmIsDb(bytes);
const n = bytes.byteLength;
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
let sah, err, nWrote = 0;
try {
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
nWrote = sah.write(bytes, {at: 0});
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}
sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */;
return nWrote;
}catch(e){
if( sah ){ await sah.close(); sah = undefined; }
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally{
if( sah ) await sah.close();
}
};
if(sqlite3.oo1){
const OpfsWlDb = function(...args){
const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args);
@@ -1270,13 +889,11 @@ const installOpfsVfs = function callee(options){
warn("Running sanity checks because of opfs-sanity-check URL arg...");
sanityCheck();
}
if(thisThreadHasOPFS()){
navigator.storage.getDirectory().then((d)=>{
if(opfsUtil.thisThreadHasOPFS()){
opfsUtil.getRootDir().then((d)=>{
W.onerror = W._originalOnError;
delete W._originalOnError;
//sqlite3.opfs = opfsUtil;
opfsUtil.rootDirectory = d;
log("End of OPFS sqlite3_vfs setup.", opfsVfs);
log("End of OPFS-WL sqlite3_vfs setup.", opfsVfs);
promiseResolve();
}).catch(promiseReject);
}else{
@@ -1301,22 +918,22 @@ const installOpfsVfs = function callee(options){
}/*W.onmessage()*/;
})/*thePromise*/;
return thePromise;
}/*installOpfsVfs()*/;
installOpfsVfs.defaultProxyUri =
}/*installOpfsWlVfs()*/;
installOpfsWlVfs.defaultProxyUri =
"sqlite3-opfs-async-proxy.js";
globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{
try{
let proxyJs = installOpfsVfs.defaultProxyUri;
let proxyJs = installOpfsWlVfs.defaultProxyUri;
if( sqlite3?.scriptInfo?.sqlite3Dir ){
installOpfsVfs.defaultProxyUri =
installOpfsWlVfs.defaultProxyUri =
sqlite3.scriptInfo.sqlite3Dir + proxyJs;
//sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri);
//sqlite3.config.warn("installOpfsWlVfs.defaultProxyUri =",installOpfsWlVfs.defaultProxyUri);
}
return installOpfsVfs().catch((e)=>{
sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message);
return installOpfsWlVfs().catch((e)=>{
sqlite3.config.warn("Ignoring inability to install OPFS-WL sqlite3_vfs:",e.message);
});
}catch(e){
sqlite3.config.error("installOpfsVfs() exception:",e);
sqlite3.config.error("installOpfsWlVfs() exception:",e);
return Promise.reject(e);
}
});

466
ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
View File

@@ -20,7 +20,10 @@
*/
'use strict';
globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
/**
const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs")
/* Gets removed from sqlite3 during bootstrap, so we need an
early reference to it. */;
/**
installOpfsVfs() returns a Promise which, on success, installs an
sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs
which accept a VFS. It is intended to be called via
@@ -74,30 +77,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
object.
*/
const installOpfsVfs = function callee(options){
if(!globalThis.SharedArrayBuffer
|| !globalThis.Atomics){
return Promise.reject(
new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+
"The server must emit the COOP/COEP response headers to enable those. "+
"See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep")
);
}else if('undefined'===typeof WorkerGlobalScope){
return Promise.reject(
new Error("The OPFS sqlite3_vfs cannot run in the main thread "+
"because it requires Atomics.wait().")
);
}else if(!globalThis.FileSystemHandle ||
!globalThis.FileSystemDirectoryHandle ||
!globalThis.FileSystemFileHandle ||
!globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle ||
!navigator?.storage?.getDirectory){
return Promise.reject(
new Error("Missing required OPFS APIs.")
);
}
if(!options || 'object'!==typeof options){
options = Object.create(null);
try{
opfsUtil.vfsInstallationFeatureCheck();
}catch(e){
return Promise.reject(e);
}
options = Object.assign(Object.create(null), options);
const urlParams = new URL(globalThis.location.href).searchParams;
if(urlParams.has('opfs-disable')){
//sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.');
@@ -113,12 +98,10 @@ const installOpfsVfs = function callee(options){
if(undefined===options.proxyUri){
options.proxyUri = callee.defaultProxyUri;
}
//sqlite3.config.warn("OPFS options =",options,globalThis.location);
if('function' === typeof options.proxyUri){
options.proxyUri = options.proxyUri();
}
//sqlite3.config.warn("OPFS options =",options,globalThis.location);
const thePromise = new Promise(function(promiseResolve_, promiseReject_){
const loggers = [
sqlite3.config.error,
@@ -138,32 +121,36 @@ const installOpfsVfs = function callee(options){
const sqlite3_vfs = capi.sqlite3_vfs;
const sqlite3_file = capi.sqlite3_file;
const sqlite3_io_methods = capi.sqlite3_io_methods;
/**
Generic utilities for working with OPFS. This will get filled out
by the Promise setup and, on success, installed as sqlite3.opfs.
ACHTUNG: do not rely on these APIs in client code. They are
experimental and subject to change or removal as the
OPFS-specific sqlite3_vfs evolves.
*/
const opfsUtil = Object.create(null);
/**
Returns true if _this_ thread has access to the OPFS APIs.
*/
const thisThreadHasOPFS = ()=>{
return globalThis.FileSystemHandle &&
globalThis.FileSystemDirectoryHandle &&
globalThis.FileSystemFileHandle &&
globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle &&
navigator?.storage?.getDirectory;
};
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
objects. After the worker's "inited" message arrives, other types
of data may be added to it.
/**
Not part of the public API. Solely for internal/development
use.
For purposes of Atomics.wait() and Atomics.notify(), we use a
SharedArrayBuffer with one slot reserved for each of the API
proxy's methods. The sync side of the API uses Atomics.wait()
on the corresponding slot and the async side uses
Atomics.notify() on that slot.
The approach of using a single SAB to serialize comms for all
instances might(?) lead to deadlock situations in multi-db
cases. We should probably have one SAB here with a single slot
for locking a per-file initialization step and then allocate a
separate SAB like the above one for each file. That will
require a bit of acrobatics but should be feasible. The most
problematic part is that xOpen() would have to use
postMessage() to communicate its SharedArrayBuffer, and mixing
that approach with Atomics.wait/notify() gets a bit messy.
*/
opfsUtil.metrics = {
const state = Object.create(null);
const metrics = Object.create(null);
//#define opfs-has-metrics
//#include api/opfs-common-inline.c-pp.js
//#undef opfs-has-metrics
const vfsMetrics = {
dump: function(){
let k, n = 0, t = 0, w = 0;
for(k in state.opIds){
@@ -264,29 +251,6 @@ const installOpfsVfs = function callee(options){
environment or the other when sqlite3_os_end() is called (_if_ it
gets called at all in a wasm build, which is undefined).
*/
/**
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
objects. After the worker's "inited" message arrives, other types
of data may be added to it.
For purposes of Atomics.wait() and Atomics.notify(), we use a
SharedArrayBuffer with one slot reserved for each of the API
proxy's methods. The sync side of the API uses Atomics.wait()
on the corresponding slot and the async side uses
Atomics.notify() on that slot.
The approach of using a single SAB to serialize comms for all
instances might(?) lead to deadlock situations in multi-db
cases. We should probably have one SAB here with a single slot
for locking a per-file initialization step and then allocate a
separate SAB like the above one for each file. That will
require a bit of acrobatics but should be feasible. The most
problematic part is that xOpen() would have to use
postMessage() to communicate its SharedArrayBuffer, and mixing
that approach with Atomics.wait/notify() gets a bit messy.
*/
const state = Object.create(null);
state.verbose = options.verbose;
state.littleEndian = (()=>{
const buffer = new ArrayBuffer(2);
@@ -341,7 +305,6 @@ const installOpfsVfs = function callee(options){
+ state.sabS11nSize/* argument/result serialization block */
);
state.opIds = Object.create(null);
const metrics = Object.create(null);
{
/* Indexes for use in our SharedArrayBuffer... */
let i = 0;
@@ -379,7 +342,7 @@ const installOpfsVfs = function callee(options){
state.sabOP = new SharedArrayBuffer(
i * 4/* ==sizeof int32, noting that Atomics.wait() and friends
can only function on Int32Array views of an SAB. */);
opfsUtil.metrics.reset();
vfsMetrics.reset();
}
/**
SQLITE_xxx constants to export to the async worker
@@ -495,6 +458,7 @@ const installOpfsVfs = function callee(options){
return rc;
};
//#if nope
/**
Not part of the public API. Only for test/development use.
*/
@@ -508,35 +472,7 @@ const installOpfsVfs = function callee(options){
W.postMessage({type: 'opfs-async-restart'});
}
};
//#define opfs-has-metrics
//#include api/opfs-common.c-pp.js
/**
Generates a random ASCII string len characters long, intended for
use as a temporary file name.
*/
const randomFilename = function f(len=16){
if(!f._chars){
f._chars = "abcdefghijklmnopqrstuvwxyz"+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
"012346789";
f._n = f._chars.length;
}
const a = [];
let i = 0;
for( ; i < len; ++i){
const ndx = Math.random() * (f._n * 64) % f._n | 0;
a[i] = f._chars[ndx];
}
return a.join("");
/*
An alternative impl. with an unpredictable length
but much simpler:
Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36)
*/
};
//#endif
/**
Map of sqlite3_file pointers to objects constructed by xOpen().
@@ -743,7 +679,7 @@ const installOpfsVfs = function callee(options){
mTimeStart('xOpen');
let opfsFlags = 0;
if(0===zName){
zName = randomFilename();
zName = opfsUtil.randomFilename();
}else if(wasm.isPtr(zName)){
if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){
/* -----------------------^^^^^ MUST pass the untranslated
@@ -805,322 +741,6 @@ const installOpfsVfs = function callee(options){
};
}
/**
Expects an OPFS file path. It gets resolved, such that ".."
components are properly expanded, and returned. If the 2nd arg
is true, the result is returned as an array of path elements,
else an absolute path string is returned.
*/
opfsUtil.getResolvedPath = function(filename,splitIt){
const p = new URL(filename, "file://irrelevant").pathname;
return splitIt ? p.split('/').filter((v)=>!!v) : p;
};
/**
Takes the absolute path to a filesystem element. Returns an
array of [handleOfContainingDir, filename]. If the 2nd argument
is truthy then each directory element leading to the file is
created along the way. Throws if any creation or resolution
fails.
*/
opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){
const path = opfsUtil.getResolvedPath(absFilename, true);
const filename = path.pop();
let dh = opfsUtil.rootDirectory;
for(const dirName of path){
if(dirName){
dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
}
}
return [dh, filename];
};
/**
Creates the given directory name, recursively, in
the OPFS filesystem. Returns true if it succeeds or the
directory already exists, else false.
*/
opfsUtil.mkdir = async function(absDirName){
try {
await opfsUtil.getDirForFilename(absDirName+"/filepart", true);
return true;
}catch(e){
//sqlite3.config.warn("mkdir(",absDirName,") failed:",e);
return false;
}
};
/**
Checks whether the given OPFS filesystem entry exists,
returning true if it does, false if it doesn't or if an
exception is intercepted while trying to make the
determination.
*/
opfsUtil.entryExists = async function(fsEntryName){
try {
const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName);
await dh.getFileHandle(fn);
return true;
}catch(e){
return false;
}
};
/**
Generates a random ASCII string, intended for use as a
temporary file name. Its argument is the length of the string,
defaulting to 16.
*/
opfsUtil.randomFilename = randomFilename;
/**
Returns a promise which resolves to an object which represents
all files and directories in the OPFS tree. The top-most object
has two properties: `dirs` is an array of directory entries
(described below) and `files` is a list of file names for all
files in that directory.
Traversal starts at sqlite3.opfs.rootDirectory.
Each `dirs` entry is an object in this form:
```
{ name: directoryName,
dirs: [...subdirs],
files: [...file names]
}
```
The `files` and `subdirs` entries are always set but may be
empty arrays.
The returned object has the same structure but its `name` is
an empty string. All returned objects are created with
Object.create(null), so have no prototype.
Design note: the entries do not contain more information,
e.g. file sizes, because getting such info is not only
expensive but is subject to locking-related errors.
*/
opfsUtil.treeList = async function(){
const doDir = async function callee(dirHandle,tgt){
tgt.name = dirHandle.name;
tgt.dirs = [];
tgt.files = [];
for await (const handle of dirHandle.values()){
if('directory' === handle.kind){
const subDir = Object.create(null);
tgt.dirs.push(subDir);
await callee(handle, subDir);
}else{
tgt.files.push(handle.name);
}
}
};
const root = Object.create(null);
await doDir(opfsUtil.rootDirectory, root);
return root;
};
/**
Irrevocably deletes _all_ files in the current origin's OPFS.
Obviously, this must be used with great caution. It may throw
an exception if removal of anything fails (e.g. a file is
locked), but the precise conditions under which the underlying
APIs will throw are not documented (so we cannot tell you what
they are).
*/
opfsUtil.rmfr = async function(){
const dir = opfsUtil.rootDirectory, opt = {recurse: true};
for await (const handle of dir.values()){
dir.removeEntry(handle.name, opt);
}
};
/**
Deletes the given OPFS filesystem entry. As this environment
has no notion of "current directory", the given name must be an
absolute path. If the 2nd argument is truthy, deletion is
recursive (use with caution!).
The returned Promise resolves to true if the deletion was
successful, else false (but...). The OPFS API reports the
reason for the failure only in human-readable form, not
exceptions which can be type-checked to determine the
failure. Because of that...
If the final argument is truthy then this function will
propagate any exception on error, rather than returning false.
*/
opfsUtil.unlink = async function(fsEntryName, recursive = false,
throwOnError = false){
try {
const [hDir, filenamePart] =
await opfsUtil.getDirForFilename(fsEntryName, false);
await hDir.removeEntry(filenamePart, {recursive});
return true;
}catch(e){
if(throwOnError){
throw new Error("unlink(",arguments[0],") failed: "+e.message,{
cause: e
});
}
return false;
}
};
/**
Traverses the OPFS filesystem, calling a callback for each
entry. The argument may be either a callback function or an
options object with any of the following properties:
- `callback`: function which gets called for each filesystem
entry. It gets passed 3 arguments: 1) the
FileSystemFileHandle or FileSystemDirectoryHandle of each
entry (noting that both are instanceof FileSystemHandle). 2)
the FileSystemDirectoryHandle of the parent directory. 3) the
current depth level, with 0 being at the top of the tree
relative to the starting directory. If the callback returns a
literal false, as opposed to any other falsy value, traversal
stops without an error. Any exceptions it throws are
propagated. Results are undefined if the callback manipulate
the filesystem (e.g. removing or adding entries) because the
how OPFS iterators behave in the face of such changes is
undocumented.
- `recursive` [bool=true]: specifies whether to recurse into
subdirectories or not. Whether recursion is depth-first or
breadth-first is unspecified!
- `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory]
specifies the starting directory.
If this function is passed a function, it is assumed to be the
callback.
Returns a promise because it has to (by virtue of being async)
but that promise has no specific meaning: the traversal it
performs is synchronous. The promise must be used to catch any
exceptions propagated by the callback, however.
*/
opfsUtil.traverse = async function(opt){
const defaultOpt = {
recursive: true,
directory: opfsUtil.rootDirectory
};
if('function'===typeof opt){
opt = {callback:opt};
}
opt = Object.assign(defaultOpt, opt||{});
const doDir = async function callee(dirHandle, depth){
for await (const handle of dirHandle.values()){
if(false === opt.callback(handle, dirHandle, depth)) return false;
else if(opt.recursive && 'directory' === handle.kind){
if(false === await callee(handle, depth + 1)) break;
}
}
};
doDir(opt.directory, 0);
};
/**
impl of importDb() when it's given a function as its second
argument.
*/
const importDbChunked = async function(filename, callback){
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
let sah = await hFile.createSyncAccessHandle();
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
sah.truncate(0);
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/);
return nWrote;
}catch(e){
await sah.close();
sah = undefined;
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally {
if( sah ) await sah.close();
}
};
/**
Asynchronously imports the given bytes (a byte array or
ArrayBuffer) into the given database file.
Results are undefined if the given db name refers to an opened
db.
If passed a function for its second argument, its behaviour
changes: imports its data in chunks fed to it by the given
callback function. It calls the callback (which may be async)
repeatedly, expecting either a Uint8Array or ArrayBuffer (to
denote new input) or undefined (to denote EOF). For so long as
the callback continues to return non-undefined, it will append
incoming data to the given VFS-hosted database file. When
called this way, the resolved value of the returned Promise is
the number of bytes written to the target file.
It very specifically requires the input to be an SQLite3
database and throws if that's not the case. It does so in
order to prevent this function from taking on a larger scope
than it is specifically intended to. i.e. we do not want it to
become a convenience for importing arbitrary files into OPFS.
This routine rewrites the database header bytes in the output
file (not the input array) to force disabling of WAL mode.
On error this throws and the state of the input file is
undefined (it depends on where the exception was triggered).
On success, resolves to the number of bytes written.
*/
opfsUtil.importDb = async function(filename, bytes){
if( bytes instanceof Function ){
return importDbChunked(filename, bytes);
}
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
util.affirmIsDb(bytes);
const n = bytes.byteLength;
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
let sah, err, nWrote = 0;
try {
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
nWrote = sah.write(bytes, {at: 0});
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}
sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */;
return nWrote;
}catch(e){
if( sah ){ await sah.close(); sah = undefined; }
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally{
if( sah ) await sah.close();
}
};
if(sqlite3.oo1){
const OpfsDb = function(...args){
const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args);
@@ -1241,12 +861,10 @@ const installOpfsVfs = function callee(options){
warn("Running sanity checks because of opfs-sanity-check URL arg...");
sanityCheck();
}
if(thisThreadHasOPFS()){
navigator.storage.getDirectory().then((d)=>{
if(opfsUtil.thisThreadHasOPFS()){
opfsUtil.getRootDir().then((d)=>{
W.onerror = W._originalOnError;
delete W._originalOnError;
sqlite3.opfs = opfsUtil;
opfsUtil.rootDirectory = d;
log("End of OPFS sqlite3_vfs setup.", opfsVfs);
promiseResolve();
}).catch(promiseReject);

21
manifest
View File

@@ -1,5 +1,5 @@
C A\spotential\sfix\sfor\sthe\sprobable\sbreakage\sof\sbundler-friendly\sbuilds\sin\sthe\sprevious\scheck-in.\sPending\sreview\sfrom\ssomeone\swho\suses\sthose\stools.
D 2026-03-04T00:48:03.758
C Consolidate\smuch\sof\sthe\sOPFS\sutility\scode\sinto\sa\snew\sfile\sfor\suse\sby\stwo\sof\sthe\sOPFS\sVFSes.
D 2026-03-04T11:37:39.256
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -572,7 +572,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e
F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07
F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a
F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6
F ext/wasm/GNUmakefile a0afdb3e6306997de1e123e8b8270affb744519c9e384dbc277d0af6a6ba1c91
F ext/wasm/GNUmakefile 4496669ca5785c7bfc7af76565f1dd45e8f9ec7529b58c485fd374088162bef1
F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a
F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259
F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff
@@ -584,7 +584,8 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 7ba933e8f1290cc65459dd371c0c9a031d96bdf14
F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81
F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930
F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41
F ext/wasm/api/opfs-common.c-pp.js 5540c58aee8fbbf2c67984e0a35e2b680a7603052f2cc442d407da3cc81c4819
F ext/wasm/api/opfs-common-inline.c-pp.js b9c4e080698792cbc04ce9dd9dda7d8316c6db0262f74820706f98b352b949d5 w ext/wasm/api/opfs-common.c-pp.js
F ext/wasm/api/opfs-common-shared.c-pp.js 3f8f3f2ab4790fdd1e6d1d9224e232cef07f1c8753827c18bbba965dbe98795f
F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b
F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55
F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f
@@ -593,12 +594,12 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c
F ext/wasm/api/sqlite3-api-prologue.js ccd8ece4b4580d2a70996218f28e810d70a86f5e2795f4d4a75f0603af24aef6
F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938
F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc
F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 041f24cd61d470072303b0cd1bb73dd2928bf2c96c0fbf3d3f4a39024c4e9ae7
F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 25b856508ac94336419133c6ec10594f576b469f85cc69cde4c09cfa06a8e1c7
F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9
F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 8c79bf3ea89102772225e6539c030787ea85203b6cc0354c545df69d81a2d910
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b2b4c7570ec816c49e7b3293f451992bd7fb7154dcd4cec6fb05d032c6cedf52
F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 54f71e563dda30af73ed84ff9de03441537b2e8fb8d2ae2a0b0c8187f51db67a
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 2e2a72a40e2ad6ea92f52eb7adbd925d4acd874ffeecaa00b85234ad49862655
F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81
F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29
F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66
@@ -2190,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee
F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c
P 9a471f7491a371052bf1785098ba966dd0d03503e7d8b9fbcd65f07b038e5021
R 8aa8acc5e3e3610acfaa4dc12ceb0c5d
P 8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec
R 82203270087631af34878be94f4d1594
U stephan
Z dc633fdeffefdb99fdb4ea173110ea63
Z 1548da853f7a79cab3a86616c1bf35cc
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@@ -1 +1 @@
8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec
db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7