database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Merge trunk into js-cpp branch.

Browse Source 
FossilOrigin-Name: e047b33d1fb7d6a32e967f03f9952249cd2da4d21dc301fe92bd7baa0da5d6a9
This commit is contained in:
stephan
2022-11-17 15:21:49 +00:00
parent b0ab21dee4 3c1572ddb4
commit c7c15d1b83
57 changed files with 648 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
ext/wasm/GNUmakefile
View File

@ -110,7 +110,7 @@ emcc.WASM_BIGINT ?= 1
sqlite3.c := $(dir.top)/sqlite3.c
sqlite3.h := $(dir.top)/sqlite3.h
SQLITE_OPT = \
-DSQLITE_ENABLE_FTS4 \
-DSQLITE_ENABLE_FTS5 \
-DSQLITE_ENABLE_RTREE \
-DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \
@ -465,7 +465,20 @@ emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT)
# debugging info, _huge_.
########################################################################
sqlite3.js := $(dir.dout)/sqlite3.js
########################################################################
# AN EXPERIMENT: undocumented Emscripten feature: if the target file
# extension is "mjs", it defaults to ES6 module builds:
# https://github.com/emscripten-core/emscripten/issues/14383
ifeq (,$(filter esm,$(MAKECMDGOALS)))
sqlite3.js.ext := js
else
esm.deps := $(filter-out esm,$(MAKECMDGOALS))
esm: $(if $(esm.deps),$(esm.deps),all)
sqlite3.js.ext := mjs
endif
# /esm
########################################################################
sqlite3.js := $(dir.dout)/sqlite3.$(sqlite3.js.ext)
sqlite3.wasm := $(dir.dout)/sqlite3.wasm
sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c
# sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter

5
ext/wasm/api/extern-post-js.js
View File

@ -15,7 +15,10 @@
impls which Emscripten installs at some point in the file above
this.
*/
const originalInit = self.sqlite3InitModule;
const originalInit =
/*Maintenance reminde: DO NOT use `self.` here. It's correct
for non-ES6 Module cases but wrong for ES6 modules because those
resolve this symbol differently! */ sqlite3InitModule;
if(!originalInit){
throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build.");
}

7
ext/wasm/api/sqlite3-api-opfs.js
View File

@ -472,9 +472,11 @@ const installOpfsVfs = function callee(options){
/**
Returns an array of the deserialized state stored by the most
recent serialize() operation (from from this thread or the
counterpart thread), or null if the serialization buffer is empty.
counterpart thread), or null if the serialization buffer is
empty. If passed a truthy argument, the serialization buffer
is cleared after deserialization.
*/
state.s11n.deserialize = function(){
state.s11n.deserialize = function(clear=false){
++metrics.s11n.deserialize.count;
const t = performance.now();
const argc = viewU8[0];
@ -499,6 +501,7 @@ const installOpfsVfs = function callee(options){
rc.push(v);
}
}
if(clear) viewU8[0] = 0;
//log("deserialize:",argc, rc);
metrics.s11n.deserialize.time += performance.now() - t;
return rc;

93
ext/wasm/api/sqlite3-api-prologue.js
View File

@ -17,22 +17,29 @@
conventions, and build process are very much under construction and
will be (re)documented once they've stopped fluctuating so much.
Specific goals of this project:
Project home page: https://sqlite.org
Documentation home page: https://sqlite.org/wasm
Specific goals of this subproject:
- Except where noted in the non-goals, provide a more-or-less
feature-complete wrapper to the sqlite3 C API, insofar as WASM
feature parity with C allows for. In fact, provide at least 3
feature parity with C allows for. In fact, provide at least 4
APIs...
1) Bind a low-level sqlite3 API which is as close to the native
one as feasible in terms of usage.
1) 1-to-1 bindings as exported from WASM, with no automatic
type conversions between JS and C.
2) A binding of (1) which provides certain JS/C type conversions
to greatly simplify its use.
2) A higher-level API, more akin to sql.js and node.js-style
3) A higher-level API, more akin to sql.js and node.js-style
implementations. This one speaks directly to the low-level
API. This API must be used from the same thread as the
low-level API.
3) A second higher-level API which speaks to the previous APIs via
4) A second higher-level API which speaks to the previous APIs via
worker messages. This one is intended for use in the main
thread, with the lower-level APIs installed in a Worker thread,
and talking to them via Worker messages. Because Workers are
@ -90,11 +97,13 @@
config object is only honored the first time this is
called. Subsequent calls ignore the argument and return the same
(configured) object which gets initialized by the first call.
This function will throw if any of the required config options are
missing.
The config object properties include:
- `exports`[^1]: the "exports" object for the current WASM
environment. In an Emscripten build, this should be set to
environment. In an Emscripten-based build, this should be set to
`Module['asm']`.
- `memory`[^1]: optional WebAssembly.Memory object, defaulting to
@ -104,7 +113,7 @@
WASM-exported memory.
- `bigIntEnabled`: true if BigInt support is enabled. Defaults to
true if self.BigInt64Array is available, else false. Some APIs
true if `self.BigInt64Array` is available, else false. Some APIs
will throw exceptions if called without BigInt support, as BigInt
is required for marshalling C-side int64 into and out of JS.
@ -116,10 +125,12 @@
the `free(3)`-compatible routine for the WASM
environment. Defaults to `"free"`.
- `wasmfsOpfsDir`[^1]: if the environment supports persistent storage, this
directory names the "mount point" for that directory. It must be prefixed
by `/` and may currently contain only a single directory-name part. Using
the root directory name is not supported by any current persistent backend.
- `wasmfsOpfsDir`[^1]: if the environment supports persistent
storage, this directory names the "mount point" for that
directory. It must be prefixed by `/` and may contain only a
single directory-name part. Using the root directory name is not
supported by any current persistent backend. This setting is
only used in WASMFS-enabled builds.
[^1] = This property may optionally be a function, in which case this
@ -388,8 +399,22 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
exceptions.
*/
class WasmAllocError extends Error {
/**
If called with 2 arguments and the 2nd one is an object, it
behaves like the Error constructor, else it concatenates all
arguments together with a single space between each to
construct an error message string. As a special case, if
called with no arguments then it uses a default error
message.
*/
constructor(...args){
super(...args);
if(2===args.length && 'object'===typeof args){
super(...args);
}else if(args.length){
super(args.join(' '));
}else{
super("Allocation failed.");
}
this.name = 'WasmAllocError';
}
};
@ -699,21 +724,33 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
API NOT throw and must instead return SQLITE_NOMEM (or
equivalent, depending on the context).
That said, very few cases in the API can result in
Very few cases in the sqlite3 JS APIs can result in
client-defined functions propagating exceptions via the C-style
API. Most notably, this applies ot User-defined SQL Functions
(UDFs) registered via sqlite3_create_function_v2(). For that
specific case it is recommended that all UDF creation be
funneled through a utility function and that a wrapper function
be added around the UDF which catches any exception and sets
the error state to OOM. (The overall complexity of registering
UDFs essentially requires a helper for doing so!)
API. Most notably, this applies to WASM-bound JS functions
which are created directly by clients and passed on _as WASM
function pointers_ to functions such as
sqlite3_create_function_v2(). Such bindings created
transparently by this API will automatically use wrappers which
catch exceptions and convert them to appropriate error codes.
For cases where non-throwing allocation is required, use
sqlite3.wasm.alloc.impl(), which is direct binding of the
underlying C-level allocator.
Design note: this function is not named "malloc" primarily
because Emscripten uses that name and we wanted to avoid any
confusion early on in this code's development, when it still
had close ties to Emscripten's glue code.
*/
alloc: undefined/*installed later*/,
/**
The API's one single point of access to the WASM-side memory
deallocator. Works like free(3) (and is likely bound to
free()).
Design note: this function is not named "free" for the same
reason that this.alloc() is not called this.malloc().
*/
dealloc: undefined/*installed later*/
@ -741,7 +778,9 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
wasm.allocFromTypedArray = function(srcTypedArray){
affirmBindableTypedArray(srcTypedArray);
const pRet = wasm.alloc(srcTypedArray.byteLength || 1);
wasm.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet);
wasm.heapForSize(srcTypedArray.constructor).set(
srcTypedArray.byteLength ? srcTypedArray : [0], pRet
);
return pRet;
};
@ -752,13 +791,13 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function.");
}
wasm.alloc = function(n){
const m = wasm.exports[keyAlloc](n);
if(!m) throw new WasmAllocError("Failed to allocate "+n+" bytes.");
wasm.alloc = function f(n){
const m = f.impl(n);
if(!m) throw new WasmAllocError("Failed to allocate",n," bytes.");
return m;
};
wasm.dealloc = (m)=>wasm.exports[keyDealloc](m);
wasm.alloc.impl = wasm.exports[keyAlloc];
wasm.dealloc = wasm.exports[keyDealloc];
/**
Reports info about compile-time options using

84
ext/wasm/api/sqlite3-opfs-async-proxy.js
View File

@ -44,6 +44,7 @@ if(self.window === self){
this API.
*/
const state = Object.create(null);
/**
verbose:
@ -96,13 +97,27 @@ metrics.dump = ()=>{
};
/**
Map of sqlite3_file pointers (integers) to metadata related to a
given OPFS file handles. The pointers are, in this side of the
interface, opaque file handle IDs provided by the synchronous
part of this constellation. Each value is an object with a structure
demonstrated in the xOpen() impl.
__openFiles is a map of sqlite3_file pointers (integers) to
metadata related to a given OPFS file handles. The pointers are, in
this side of the interface, opaque file handle IDs provided by the
synchronous part of this constellation. Each value is an object
with a structure demonstrated in the xOpen() impl.
*/
const __openFiles = Object.create(null);
/**
__autoLocks is a Set of sqlite3_file pointers (integers) which were
"auto-locked". i.e. those for which we obtained a sync access
handle without an explicit xLock() call. Such locks will be
released during db connection idle time, whereas a sync access
handle obtained via xLock(), or subsequently xLock()'d after
auto-acquisition, will not be released until xUnlock() is called.
Maintenance reminder: if we relinquish auto-locks at the end of the
operation which acquires them, we pay a massive performance
penalty: speedtest1 benchmarks take up to 4x as long. By delaying
the lock release until idle time, the hit is negligible.
*/
const __autoLocks = new Set();
/**
Expects an OPFS file path. It gets resolved, such that ".."
@ -191,6 +206,10 @@ const getSyncHandle = async (fh)=>{
}
}
log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
if(!fh.xLock){
__autoLocks.add(fh.fid);
log("Auto-locked",fh.fid,fh.filenameAbs);
}
}
return fh.syncHandle;
};
@ -210,10 +229,30 @@ const closeSyncHandle = async (fh)=>{
log("Closing sync handle for",fh.filenameAbs);
const h = fh.syncHandle;
delete fh.syncHandle;
delete fh.xLock;
__autoLocks.delete(fh.fid);
return h.close();
}
};
/**
A proxy for closeSyncHandle() which is guaranteed to not throw.
This function is part of a lock/unlock step in functions which
require a sync access handle but may be called without xLock()
having been called first. Such calls need to release that
handle to avoid locking the file for all of time. This is an
_attempt_ at reducing cross-tab contention but it may prove
to be more of a problem than a solution and may need to be
removed.
*/
const closeSyncHandleNoThrow = async (fh)=>{
try{await closeSyncHandle(fh)}
catch(e){
warn("closeSyncHandleNoThrow() ignoring:",e,fh);
}
};
/**
Stores the given value at state.sabOPView[state.opIds.rc] and then
Atomics.notify()'s it.
@ -342,9 +381,10 @@ const vfsAsyncImpls = {
xClose: async function(fid/*sqlite3_file pointer*/){
const opName = 'xClose';
mTimeStart(opName);
__autoLocks.delete(fid);
const fh = __openFiles[fid];
let rc = 0;
wTimeStart('xClose');
wTimeStart(opName);
if(fh){
delete __openFiles[fid];
await closeSyncHandle(fh);
@ -422,12 +462,17 @@ const vfsAsyncImpls = {
mTimeStart('xLock');
const fh = __openFiles[fid];
let rc = 0;
const oldLockType = fh.xLock;
fh.xLock = lockType;
if( !fh.syncHandle ){
wTimeStart('xLock');
try { await getSyncHandle(fh) }
catch(e){
try {
await getSyncHandle(fh);
__autoLocks.delete(fid);
}catch(e){
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_LOCK;
fh.xLock = oldLockType;
}
wTimeEnd();
}
@ -461,6 +506,7 @@ const vfsAsyncImpls = {
*/
wTimeEnd();
__openFiles[fid] = Object.assign(Object.create(null),{
fid: fid,
filenameAbs: filename,
filenamePart: filenamePart,
dirHandle: hDir,
@ -610,7 +656,7 @@ const initS11n = ()=>{
default: toss("Invalid type ID:",tid);
}
};
state.s11n.deserialize = function(){
state.s11n.deserialize = function(clear=false){
++metrics.s11n.deserialize.count;
const t = performance.now();
const argc = viewU8[0];
@ -635,6 +681,7 @@ const initS11n = ()=>{
rc.push(v);
}
}
if(clear) viewU8[0] = 0;
//log("deserialize:",argc, rc);
metrics.s11n.deserialize.time += performance.now() - t;
return rc;
@ -701,21 +748,30 @@ const waitLoop = async function f(){
We need to wake up periodically to give the thread a chance
to do other things.
*/
const waitTime = 1000;
const waitTime = 500;
while(!flagAsyncShutdown){
try {
if('timed-out'===Atomics.wait(
state.sabOPView, state.opIds.whichOp, 0, waitTime
)){
if(__autoLocks.size){
/* Release all auto-locks. */
for(const fid of __autoLocks){
const fh = __openFiles[fid];
await closeSyncHandleNoThrow(fh);
log("Auto-unlocked",fid,fh.filenameAbs);
}
}
continue;
}
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
const args = state.s11n.deserialize() || [];
state.s11n.serialize(/* clear s11n to keep the caller from
confusing this with an exception string
written by the upcoming operation */);
const args = state.s11n.deserialize(
true /* clear s11n to keep the caller from confusing this with
an exception string written by the upcoming
operation */
) || [];
//warn("waitLoop() whichOp =",opId, hnd, args);
if(hnd.f) await hnd.f(...args);
else error("Missing callback for opId",opId);

5
ext/wasm/api/sqlite3-wasm.c
View File

@ -67,7 +67,7 @@
** larger cache benefits the larger workloads. Speed differences
** between 2x and nearly 3x have been measured with ample page cache.
*/
# define SQLITE_DEFAULT_CACHE_SIZE -16777216
# define SQLITE_DEFAULT_CACHE_SIZE -16384
#endif
#if 0 && !defined(SQLITE_DEFAULT_PAGE_SIZE)
/* TODO: experiment with this. */
@ -1108,9 +1108,6 @@ int sqlite3_wasm_init_wasmfs(const char *zMountPoint){
/** It's not enough to instantiate the backend. We have to create a
mountpoint in the VFS and attach the backend to it. */
if( pOpfs && 0!=access(zMountPoint, F_OK) ){
/* mkdir() simply hangs when called from fiddle app. Cause is
not yet determined but the hypothesis is an init-order
issue. */
/* Note that this check and is not robust but it will
hypothetically suffice for the transient wasm-based virtual
filesystem we're currently running in. */

2
ext/wasm/module-symbols.html
View File

@ -189,7 +189,7 @@
<div id='list-compile-options' class='pseudolist wide2'></div>
</div><!-- .initially-hidden -->
<script src="jswasm/sqlite3.js">/* This tag MUST be in side the
<script src="jswasm/sqlite3.js">/* This tag MUST be inside the
fossil-doc block so that this part can work without modification in
the wasm docs repo. */</script>
<script>(async function(){

7
ext/wasm/speedtest1-worker.html
View File

@ -169,7 +169,11 @@
return str+a.join('&nbsp;');
};
const W = new Worker("speedtest1-worker.js?sqlite3.dir=jswasm");
const urlParams = new URL(self.location.href).searchParams;
const W = new Worker(
"speedtest1-worker.js?sqlite3.dir=jswasm"+
(urlParams.has('opfs-verbose') ? '&opfs-verbose' : '')
);
const mPost = function(msgType,payload){
W.postMessage({type: msgType, data: payload});
};
@ -179,7 +183,6 @@
const eLinkMainThread = E('#link-main-thread');
const eLinkWasmfs = E('#link-wasmfs');
const eLinkKvvfs = E('#link-kvvfs');
const urlParams = new URL(self.location.href).searchParams;
const getSelectedFlags = ()=>{
const f = Array.prototype.map.call(eFlags.selectedOptions, (v)=>v.value);
[

14
ext/wasm/tester1.js
View File

@ -342,8 +342,20 @@
throw new sqlite3.WasmAllocError;
}catch(e){
T.assert(e instanceof Error)
.assert(e instanceof sqlite3.WasmAllocError);
.assert(e instanceof sqlite3.WasmAllocError)
.assert("Allocation failed." === e.message);
}
try {
throw new sqlite3.WasmAllocError("test",{
cause: 3
});
}catch(e){
T.assert(3 === e.cause)
.assert("test" === e.message);
}
try {throw new sqlite3.WasmAllocError("test","ing",".")}
catch(e){T.assert("test ing ." === e.message)}
try{ throw new sqlite3.SQLite3Error(capi.SQLITE_SCHEMA) }
catch(e){ T.assert('SQLITE_SCHEMA' === e.message) }
try{ sqlite3.SQLite3Error.toss(capi.SQLITE_CORRUPT,{cause: true}) }