database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-12-24 14:17:58 +03:00
Code Activity

Numerous cleanups in the JS bits. Removed some now-defunct wasm test files. Expose sqlite3.opfs object containing various OPFS-specific utilities.

Browse Source 
FossilOrigin-Name: 26e625d05d9820033b23536f18ad3ddc59ed712ad507d4b0c7fe88abd15d2be8
This commit is contained in:
stephan
2022-09-18 17:32:35 +00:00
parent 0db3089576
commit f386012069
15 changed files with 337 additions and 679 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
ext/wasm/api/sqlite3-api-oo1.js
View File

@@ -85,9 +85,24 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
/**
A proxy for DB class constructors. It must be called with the
being-construct DB object as its "this".
being-construct DB object as its "this". See the DB constructor
for the argument docs. This is split into a separate function
in order to enable simple creation of special-case DB constructors,
e.g. a hypothetical LocalStorageDB or OpfsDB.
Expects to be passed a configuration object with the following
properties:
- `.filename`: the db filename. It may be a special name like ":memory:"
or "".
- `.flags`: as documented in the DB constructor.
- `.vfs`: as documented in the DB constructor.
It also accepts those as the first 3 arguments.
*/
const dbCtorHelper = function ctor(fn=':memory:', flags='c', vfsName){
const dbCtorHelper = function ctor(...args){
if(!ctor._name2vfs){
// Map special filenames which we handle here (instead of in C)
// to some helpful metadata...
@@ -104,25 +119,33 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
filename: isWorkerThread || (()=>'session')
};
}
if('string'!==typeof fn){
toss3("Invalid filename for DB constructor.");
const opt = ctor.normalizeArgs(...args);
let fn = opt.filename, vfsName = opt.vfs, flagsStr = opt.flags;
if(('string'!==typeof fn && 'number'!==typeof fn)
|| 'string'!==typeof flagsStr
|| (vfsName && ('string'!==typeof vfsName && 'number'!==typeof vfsName))){
console.error("Invalid DB ctor args",opt,arguments);
toss3("Invalid arguments for DB constructor.");
}
const vfsCheck = ctor._name2vfs[fn];
let fnJs = ('number'===typeof fn) ? capi.wasm.cstringToJs(fn) : fn;
const vfsCheck = ctor._name2vfs[fnJs];
if(vfsCheck){
vfsName = vfsCheck.vfs;
fn = vfsCheck.filename(fn);
fn = fnJs = vfsCheck.filename(fnJs);
}
let ptr, oflags = 0;
if( flags.indexOf('c')>=0 ){
if( flagsStr.indexOf('c')>=0 ){
oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE;
}
if( flags.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE;
if( flagsStr.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE;
if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY;
oflags |= capi.SQLITE_OPEN_EXRESCODE;
const stack = capi.wasm.scopedAllocPush();
try {
const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */;
const pVfsName = vfsName ? capi.wasm.scopedAllocCString(vfsName) : 0;
const pVfsName = vfsName ? (
('number'===typeof vfsName ? vfsName : capi.wasm.scopedAllocCString(vfsName))
): 0;
const rc = capi.sqlite3_open_v2(fn, ppDb, oflags, pVfsName);
ptr = capi.wasm.getPtrValue(ppDb);
checkSqlite3Rc(ptr, rc);
@@ -132,11 +155,36 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
}finally{
capi.wasm.scopedAllocPop(stack);
}
this.filename = fn;
this.filename = fnJs;
__ptrMap.set(this, ptr);
__stmtMap.set(this, Object.create(null));
__udfMap.set(this, Object.create(null));
};
/**
A helper for DB constructors. It accepts either a single
config-style object or up to 3 arguments (filename, dbOpenFlags,
dbVfsName). It returns a new object containing:
{ filename: ..., flags: ..., vfs: ... }
If passed an object, any additional properties it has are copied
as-is into the new object.
*/
dbCtorHelper.normalizeArgs = function(filename,flags = 'c',vfs = null){
const arg = {};
if(1===arguments.length && 'object'===typeof arguments[0]){
const x = arguments[0];
Object.keys(x).forEach((k)=>arg[k] = x[k]);
if(undefined===arg.flags) arg.flags = 'c';
if(undefined===arg.vfs) arg.vfs = null;
}else{
arg.filename = filename;
arg.flags = flags;
arg.vfs = vfs;
}
return arg;
};
/**
The DB class provides a high-level OO wrapper around an sqlite3
@@ -175,6 +223,17 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
or not at all, to use the default. If passed a value, it must
be the string name of a VFS
The constructor optionally (and preferably) takes its arguments
in the form of a single configuration object with the following
properties:
- `.filename`: database file name
- `.flags`: open-mode flags
- `.vfs`: the VFS fname
The `filename` and `vfs` arguments may be either JS strings or
C-strings allocated via WASM.
For purposes of passing a DB instance to C-style sqlite3
functions, the DB object's read-only `pointer` property holds its
`sqlite3*` pointer value. That property can also be used to check
@@ -187,12 +246,12 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
the database. In this mode, only a single database is permitted
in each storage object. This feature is experimental and subject
to any number of changes (including outright removal). This
support requires a specific build of sqlite3, the existence of
which can be determined at runtime by checking for a non-0 return
value from sqlite3.capi.sqlite3_vfs_find("kvvfs").
support requires the kvvfs sqlite3 VFS, the existence of which
can be determined at runtime by checking for a non-0 return value
from sqlite3.capi.sqlite3_vfs_find("kvvfs").
*/
const DB = function ctor(fn=':memory:', flags='c', vfsName){
dbCtorHelper.apply(this, Array.prototype.slice.call(arguments));
const DB = function(...args){
dbCtorHelper.apply(this, args);
};
/**
@@ -361,12 +420,31 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
closed. After calling close(), `this.pointer` will resolve to
`undefined`, so that can be used to check whether the db
instance is still opened.
If this.onclose.before is a function then it is called before
any close-related cleanup.
If this.onclose.after is a function then it is called after the
db is closed but before auxiliary state like this.filename is
cleared.
Both onclose handlers are passed this object. If this db is not
opened, neither of the handlers are called. Any exceptions the
handlers throw are ignored because "destructors must not
throw."
Note that garbage collection of a db handle, if it happens at
all, will never trigger close(), so onclose handlers are not a
reliable way to implement close-time cleanup or maintenance of
a db.
*/
close: function(){
if(this.pointer){
if(this.onclose && (this.onclose.before instanceof Function)){
try{this.onclose.before(this)}
catch(e){/*ignore*/}
}
const pDb = this.pointer;
let s;
const that = this;
Object.keys(__stmtMap.get(this)).forEach((k,s)=>{
if(s && s.pointer) s.finalize();
});
@@ -377,6 +455,10 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
__stmtMap.delete(this);
__udfMap.delete(this);
capi.sqlite3_close_v2(pDb);
if(this.onclose && (this.onclose.after instanceof Function)){
try{this.onclose.after(this)}
catch(e){/*ignore*/}
}
delete this.filename;
}
},
@@ -401,13 +483,13 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
}
},
/**
Similar to this.filename but will return NULL for special names
like ":memory:". Not of much use until we have filesystem
support. Throws if the DB has been closed. If passed an
argument it then it will return the filename of the ATTACHEd db
with that name, else it assumes a name of `main`.
Similar to this.filename but will return a falsy value for
special names like ":memory:". Throws if the DB has been
closed. If passed an argument it then it will return the
filename of the ATTACHEd db with that name, else it assumes a
name of `main`.
*/
fileName: function(dbName='main'){
getFilename: function(dbName='main'){
return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName);
},
/**
@@ -1591,7 +1673,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
ooApi: "0.1"
},
DB,
Stmt
Stmt,
dbCtorHelper
}/*oo1 object*/;
});

128
ext/wasm/api/sqlite3-api-opfs.js
View File

@@ -42,8 +42,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
that number will increase as the OPFS API matures).
- The OPFS features used here are only available in dedicated Worker
threads. This file tries to detect that case and becomes a no-op
if those features do not seem to be available.
threads. This file tries to detect that case, resulting in a
rejected Promise if those features do not seem to be available.
- It requires the SharedArrayBuffer and Atomics classes, and the
former is only available if the HTTP server emits the so-called
@@ -72,7 +72,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
returned Promise resolves.
On success, the Promise resolves to the top-most sqlite3 namespace
object.
object and that object gets a new object installed in its
`opfs` property, containing several OPFS-specific utilities.
*/
sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
const options = (asyncProxyUri && 'object'===asyncProxyUri) ? asyncProxyUri : {
@@ -89,6 +90,13 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
options.proxyUri = callee.defaultProxyUri;
}
delete sqlite3.installOpfsVfs;
/**
Generic utilities for working with OPFS. This will get filled out
by the Promise setup and, on success, installed as sqlite3.opfs.
*/
const opfsUtil = Object.create(null);
const thePromise = new Promise(function(promiseResolve, promiseReject){
const logPrefix = "OPFS syncer:";
const warn = (...args)=>{
@@ -118,9 +126,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
const sqlite3_vfs = capi.sqlite3_vfs;
const sqlite3_file = capi.sqlite3_file;
const sqlite3_io_methods = capi.sqlite3_io_methods;
const StructBinder = sqlite3.StructBinder;
const W = new Worker(options.proxyUri);
const workerOrigOnError = W.onrror;
W._originalOnError = W.onerror /* will be restored later */;
W.onerror = function(err){
promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
};
@@ -131,17 +138,37 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
This object must initially contain only cloneable or sharable
objects. After the worker's "inited" message arrives, other types
of data may be added to it.
For purposes of Atomics.wait() and Atomics.notify(), we use a
SharedArrayBuffer with one slot reserved for each of the API
proxy's methods. The sync side of the API uses Atomics.wait()
on the corresponding slot and the async side uses
Atomics.notify() on that slot.
The approach of using a single SAB to serialize comms for all
instances might(?) lead to deadlock situations in multi-db
cases. We should probably have one SAB here with a single slot
for locking a per-file initialization step and then allocate a
separate SAB like the above one for each file. That will
require a bit of acrobatics but should be feasible.
*/
const state = Object.create(null);
state.verbose = options.verbose;
state.fileBufferSize = 1024 * 64 + 8 /* size of fileHandle.sab. 64k = max sqlite3 page size */;
state.fbInt64Offset = state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64*/;
state.fileBufferSize =
1024 * 64 + 8 /* size of aFileHandle.sab. 64k = max sqlite3 page
size. The additional bytes are space for
holding BigInt results, since we cannot store
those via the Atomics API (which only works on
an Int32Array). */;
state.fbInt64Offset =
state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */;
state.opIds = Object.create(null);
{
let i = 0;
state.opIds.xAccess = i++;
state.opIds.xClose = i++;
state.opIds.xDelete = i++;
state.opIds.xDeleteNoWait = i++;
state.opIds.xFileSize = i++;
state.opIds.xOpen = i++;
state.opIds.xRead = i++;
@@ -149,14 +176,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
state.opIds.xSync = i++;
state.opIds.xTruncate = i++;
state.opIds.xWrite = i++;
state.opIds.mkdir = i++;
state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/);
/* The approach of using a single SAB to serialize comms for all
instances may(?) lead to deadlock situations in multi-db
cases. We should probably have one SAB here with a single slot
for locking a per-file initialization step and then allocate a
separate SAB like the above one for each file. That will
require a bit of acrobatics but should be feasible.
*/
}
state.sq3Codes = Object.create(null);
@@ -167,15 +188,14 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ',
'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE'
'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
'SQLITE_IOERR_DELETE'
].forEach(function(k){
state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
state.sq3Codes._reverse[capi[k]] = k;
});
const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n];
const opStore = (op,val=-1)=>Atomics.store(state.opSABView, state.opIds[op], val);
const opWait = (op,val=-1)=>Atomics.wait(state.opSABView, state.opIds[op], val);
/**
Runs the given operation in the async worker counterpart, waits
@@ -185,9 +205,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
given operation's signature in the async API counterpart.
*/
const opRun = (op,args)=>{
opStore(op);
Atomics.store(state.opSABView, state.opIds[op], -1);
wMsg(op, args);
opWait(op);
Atomics.wait(state.opSABView, state.opIds[op], -1);
return Atomics.load(state.opSABView, state.opIds[op]);
};
@@ -268,7 +288,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
func with the same signature as described above.
*/
const installMethod = function callee(tgt, name, func){
if(!(tgt instanceof StructBinder.StructType)){
if(!(tgt instanceof sqlite3.StructBinder.StructType)){
toss("Usage error: target object is-not-a StructType.");
}
if(1===arguments.length){
@@ -429,7 +449,10 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
return 0;
},
xDelete: function(pVfs, zName, doSyncDir){
return opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
/* We're ignoring errors because we cannot yet differentiate
between harmless and non-harmless failures. */
return 0;
},
xFullPathname: function(pVfs,zName,nOut,pOut){
/* Until/unless we have some notion of "current dir"
@@ -521,6 +544,65 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
for(let k of Object.keys(ioSyncWrappers)) inst(k, ioSyncWrappers[k]);
inst = installMethod(opfsVfs);
for(let k of Object.keys(vfsSyncWrappers)) inst(k, vfsSyncWrappers[k]);
/**
Syncronously deletes the given OPFS filesystem entry, ignoring
any errors. As this environment has no notion of "current
directory", the given name must be an absolute path. If the 2nd
argument is truthy, deletion is recursive (use with caution!).
Returns true if the deletion succeeded and fails if it fails,
but cannot report the nature of the failure.
*/
opfsUtil.deleteEntry = function(fsEntryName,recursive){
return 0===opRun('xDelete', {filename:fsEntryName, recursive});
};
/**
Exactly like deleteEntry() but runs asynchronously.
*/
opfsUtil.deleteEntryAsync = async function(fsEntryName,recursive){
wMsg('xDeleteNoWait', {filename: fsEntryName, recursive});
};
/**
Synchronously creates the given directory name, recursively, in
the OPFS filesystem. Returns true if it succeeds or the
directory already exists, else false.
*/
opfsUtil.mkdir = async function(absDirName){
return 0===opRun('mkdir', absDirName);
};
/**
Synchronously checks whether the given OPFS filesystem exists,
returning true if it does, false if it doesn't.
*/
opfsUtil.entryExists = function(fsEntryName){
return 0===opRun('xAccess', fsEntryName);
};
/**
Generates a random ASCII string, intended for use as a
temporary file name. Its argument is the length of the string,
defaulting to 16.
*/
opfsUtil.randomFilename = randomFilename;
if(sqlite3.oo1){
opfsUtil.OpfsDb = function(...args){
const opt = sqlite3.oo1.dbCtorHelper.normalizeArgs(...args);
opt.vfs = opfsVfs.$zName;
sqlite3.oo1.dbCtorHelper.call(this, opt);
};
opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
}
/**
Potential TODOs:
- Expose one or both of the Worker objects via opfsUtil and
publish an interface for proxying the higher-level OPFS
features like getting a directory listing.
*/
const sanityCheck = async function(){
const scope = wasm.scopedAllocPush();
@@ -605,7 +687,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
warn("Running sanity checks because of opfs-sanity-check URL arg...");
sanityCheck();
}
W.onerror = workerOrigOnError;
W.onerror = W._originalOnError;
delete W._originalOnError;
sqlite3.opfs = opfsUtil;
promiseResolve(sqlite3);
log("End of OPFS sqlite3_vfs setup.", opfsVfs);
}catch(e){

87
ext/wasm/api/sqlite3-api-prologue.js
View File

@@ -689,7 +689,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
/** State for sqlite3_web_persistent_dir(). */
let __persistentDir;
/**
An experiment. Do not use.
An experiment. Do not use in client code.
If the wasm environment has a persistent storage directory,
its path is returned by this function. If it does not then
@@ -699,14 +699,18 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
environment to determine whether persistence filesystem support
is available and, if it is, enables it (if needed).
This function currently only recognizes the WASMFS/OPFS storage
combination. "Plain" OPFS is provided via a separate VFS which
can optionally be installed (if OPFS is available on the system)
using sqlite3.installOpfsVfs().
TODOs and caveats:
- If persistent storage is available at the root of the virtual
filesystem, this interface cannot currently distinguish that
from the lack of persistence. That case cannot currently (with
WASMFS/OPFS) happen, but it is conceivably possible in future
environments or non-browser runtimes (none of which are yet
supported targets).
from the lack of persistence. That can (in the mean time)
happen when using the JS-native "opfs" VFS, as opposed to the
WASMFS/OPFS combination.
*/
capi.sqlite3_web_persistent_dir = function(){
if(undefined !== __persistentDir) return __persistentDir;
@@ -764,6 +768,49 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
capi.wasm.exports.sqlite3_initialize();
}
/**
Given an `sqlite3*` and an sqlite3_vfs name, returns a truthy
value (see below) if that db handle uses that VFS, else returns
false. If pDb is falsy then this function returns a truthy value
if the default VFS is that VFS. Results are undefined if pDb is
truthy but refers to an invalid pointer.
The 2nd argument may either be a JS string or a C-string
allocated from the wasm environment.
The truthy value it returns is a pointer to the `sqlite3_vfs`
object.
To permit safe use of this function from APIs which may be called
via the C stack (like SQL UDFs), this function does not throw: if
bad arguments cause a conversion error when passing into
wasm-space, false is returned.
*/
capi.sqlite3_web_db_uses_vfs = function(pDb,vfsName){
try{
const pK = ('number'===vfsName)
? capi.wasm.exports.sqlite3_vfs_find(vfsName)
: capi.sqlite3_vfs_find(vfsName);
if(!pK) return false;
else if(!pDb){
return capi.sqlite3_vfs_find(0)===pK ? pK : false;
}
const ppVfs = capi.wasm.allocPtr();
try{
return (
(0===capi.sqlite3_file_control(
pDb, "main", capi.SQLITE_FCNTL_VFS_POINTER, ppVfs
)) && (capi.wasm.getPtrValue(ppVfs) === pK)
) ? pK : false;
}finally{
capi.wasm.dealloc(ppVfs);
}
}catch(e){
/* Ignore - probably bad args to a wasm-bound function. */
return false;
}
};
if( self.window===self ){
/* Features specific to the main window thread... */
@@ -812,7 +859,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
/**
This routine guesses the approximate amount of
window.localStorage and/or window.sessionStorage in use by the
kvvfs database backend. Its argument must be one of
kvvfs database backend. Its argument must be one of
('session', 'local', ''). In the first two cases, only
sessionStorage resp. localStorage is counted. If it's an empty
string (the default) then both are counted. Only storage keys
@@ -842,34 +889,6 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
return sz * 2 /* because JS uses UC16 encoding */;
};
/**
Given an `sqlite3*`, returns a truthy value (see below) if that
db handle uses the "kvvfs" VFS, else returns false. If pDb is
NULL then this function returns true if the default VFS is
"kvvfs". Results are undefined if pDb is truthy but refers to
an invalid pointer.
The truthy value it returns is a pointer to the kvvfs
`sqlite3_vfs` object.
*/
capi.sqlite3_web_db_is_kvvfs = function(pDb){
const pK = capi.sqlite3_vfs_find("kvvfs");
if(!pK) return false;
else if(!pDb){
return capi.sqlite3_vfs_find(0) && pK;
}
const scope = capi.wasm.scopedAllocPush();
try{
const ppVfs = capi.wasm.scopedAllocPtr();
return (
(0===capi.sqlite3_file_control(
pDb, "main", capi.SQLITE_FCNTL_VFS_POINTER, ppVfs
)) && (capi.wasm.getPtrValue(ppVfs) === pK)
) ? pK : false;
}finally{
capi.wasm.scopedAllocPop(scope);
}
};
}/* main-window-only bits */
/* The remainder of the API will be set up in later steps. */

6
ext/wasm/api/sqlite3-api-worker1.js
View File

@@ -371,10 +371,14 @@ sqlite3.initWorker1API = function(){
close: function(db,alsoUnlink){
if(db){
delete this.dbs[getDbId(db)];
const filename = db.fileName();
const filename = db.getFilename();
db.close();
if(db===this.defaultDb) this.defaultDb = undefined;
if(alsoUnlink && filename){
/* This isn't necessarily correct: the db might be using a
VFS other than the default. How do we best resolve this
without having to special-case the kvvfs and opfs
VFSes? */
sqlite3.capi.wasm.sqlite3_wasm_vfs_unlink(filename);
}
}